257 lines
7.9 KiB
Go
257 lines
7.9 KiB
Go
package backupbinlog
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"github.com/alibaba/polardbx-operator/pkg/binlogtool/binlog"
|
|
binlogEvent "github.com/alibaba/polardbx-operator/pkg/binlogtool/binlog/event"
|
|
"github.com/alibaba/polardbx-operator/pkg/binlogtool/binlog/spec"
|
|
. "github.com/alibaba/polardbx-operator/pkg/hpfs/common"
|
|
"github.com/alibaba/polardbx-operator/pkg/hpfs/config"
|
|
"github.com/alibaba/polardbx-operator/pkg/hpfs/filestream"
|
|
"github.com/google/uuid"
|
|
"io"
|
|
"k8s.io/apimachinery/pkg/util/net"
|
|
"os"
|
|
"strconv"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
const (
|
|
BufferSizeBytes = 8 << 10 // 8KB
|
|
FilestreamIp = "127.0.0.1"
|
|
BinlogFilepathFormat = "%s/%s/%s/%s/%s/%s/%s/%s/%s/binlog-file/%s"
|
|
BinlogMetaFilepathFormat = "%s/%s/%s/%s/%s/%s/%s/%s/%s/binlog-meta/%s"
|
|
BatchSize = 1000
|
|
)
|
|
|
|
type Callback func(watcher *Watcher, file *BinlogFile) bool
|
|
|
|
var filestreamPort int
|
|
|
|
func SetLocalFilestreamSeverPort(port int) {
|
|
filestreamPort = port
|
|
}
|
|
|
|
//BeforeUpload print some info. set RequestId
|
|
func BeforeUpload(w *Watcher, binlogFile *BinlogFile) bool {
|
|
w.uploadLogger = w.logger.WithValues("trace", uuid.New().String(), "filepath", binlogFile.Filepath)
|
|
infoJson, _ := json.Marshal(binlogFile)
|
|
fileInfo, err := os.Stat(binlogFile.Filepath)
|
|
if err != nil {
|
|
w.uploadLogger.Error(err, "failed to stat file")
|
|
return false
|
|
}
|
|
binlogFile.FileLastModifiedAt = fileInfo.ModTime()
|
|
binlogFile.Size = fileInfo.Size()
|
|
w.uploadLogger.Info("BeforeUpload", "binlogFile", string(infoJson))
|
|
return true
|
|
}
|
|
|
|
//AfterUpload print some info. set RequestId
|
|
func AfterUpload(w *Watcher, binlogFile *BinlogFile) bool {
|
|
w.uploadLogger = nil
|
|
return true
|
|
}
|
|
|
|
//FetchStartIndex parse start index from the beginning of the binlog file
|
|
func FetchStartIndex(w *Watcher, binlogFile *BinlogFile) bool {
|
|
logger := w.uploadLogger.WithValues("action", "FetchStartIndex")
|
|
logger.Info("begin")
|
|
f, err := os.OpenFile(binlogFile.Filepath, os.O_RDONLY, os.ModePerm)
|
|
if err != nil {
|
|
logger.Error(err, "failed to open file")
|
|
return false
|
|
}
|
|
defer f.Close()
|
|
headBytes, err := ReadBytes(f, BufferSizeBytes)
|
|
if len(headBytes) > 0 {
|
|
binlogFile.StartIndex, binlogFile.EventTimestamp, err = GetBinlogFileBeginInfo(headBytes, binlogFile.Filename, binlogFile.BinlogChecksum)
|
|
if err != nil {
|
|
logger.Error(err, "failed to get binlog file begin info")
|
|
return false
|
|
}
|
|
logger.Info("success", "startIndex", binlogFile.StartIndex)
|
|
return true
|
|
}
|
|
logger.Error(err, "failed")
|
|
return false
|
|
}
|
|
|
|
func GetBinlogFileBeginInfo(beginBytes []byte, filename string, binlogChecksum string) (uint64, uint64, error) {
|
|
opts := []binlog.LogEventScannerOption{
|
|
binlog.WithBinlogFile(filename),
|
|
binlog.WithChecksumAlgorithm(binlogChecksum),
|
|
binlog.WithLogEventHeaderFilter(func(header binlogEvent.LogEventHeader) bool {
|
|
return header.EventTypeCode() == spec.PREVIOUS_CONSENSUS_INDEX_LOG_EVENT
|
|
}),
|
|
}
|
|
scanner, err := binlog.NewLogEventScanner(bufio.NewReader(bytes.NewReader(beginBytes)), opts...)
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
_, ev, err := scanner.Next()
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
consensusEvent, ok := ev.EventData().(*binlogEvent.PreviousConsensusIndexEvent)
|
|
if !ok {
|
|
return 0, 0, err
|
|
}
|
|
return consensusEvent.Index, uint64(ev.EventHeader().EventTimestamp()), nil
|
|
}
|
|
|
|
// Upload read the file only once. do 1.compute sha256 2.get start consensus log index 3. upload the file finally.
|
|
func Upload(w *Watcher, binlogFile *BinlogFile) bool {
|
|
logger := w.uploadLogger.WithValues("action", "Upload")
|
|
if binlogFile.SinkType == config.SinkTypeNone {
|
|
w.uploadLogger.Info("skip")
|
|
return true
|
|
}
|
|
logger.Info("begin")
|
|
f, err := os.OpenFile(binlogFile.Filepath, os.O_RDONLY, os.ModePerm)
|
|
if err != nil {
|
|
logger.Error(err, "failed to open file")
|
|
return false
|
|
}
|
|
defer f.Close()
|
|
buf := make([]byte, BufferSizeBytes)
|
|
hash := sha256.New()
|
|
reader, writer := io.Pipe()
|
|
defer reader.Close()
|
|
defer writer.Close()
|
|
|
|
var waitGroup sync.WaitGroup
|
|
var errValue atomic.Value
|
|
binlogFile.CreatedAt = time.Now()
|
|
binlogFile.UpdatedAt = binlogFile.CreatedAt
|
|
go func() {
|
|
defer reader.Close()
|
|
waitGroup.Add(1)
|
|
defer waitGroup.Done()
|
|
// upload mock
|
|
err := uploadBinlogFile(reader, binlogFile)
|
|
if err != nil {
|
|
logger.Error(err, "failed to upload remote")
|
|
errValue.Store(err)
|
|
}
|
|
}()
|
|
|
|
for {
|
|
cnt, err := f.Read(buf)
|
|
if err != nil {
|
|
if net.IsProbableEOF(err) {
|
|
writer.Close()
|
|
break
|
|
}
|
|
logger.Error(err, "failed to read file")
|
|
return false
|
|
}
|
|
if cnt > 0 {
|
|
writer.Write(buf[:cnt])
|
|
hash.Write(buf[:cnt])
|
|
}
|
|
}
|
|
waitGroup.Wait()
|
|
if errValue.Load() != nil {
|
|
return false
|
|
}
|
|
hashBytes := hash.Sum(nil)
|
|
binlogFile.Sha256 = hex.EncodeToString(hashBytes)
|
|
err = uploadBinlogFileMeta(binlogFile)
|
|
if err != nil {
|
|
logger.Error(err, "failed to upload binlog meta")
|
|
return false
|
|
}
|
|
logger.Info("success", "sha256", binlogFile.Sha256, "size", binlogFile.Size)
|
|
return true
|
|
}
|
|
|
|
//UploadRemote by filestream client
|
|
func uploadBinlogFile(reader io.Reader, binlogFile *BinlogFile) error {
|
|
//upload binlogfile meta
|
|
client := filestream.NewFileClient(FilestreamIp, filestreamPort, nil)
|
|
action := filestream.UploadOss
|
|
if binlogFile.SinkType == config.SinkTypeSftp {
|
|
action = filestream.UploadSsh
|
|
}
|
|
//upload binlogfile
|
|
binlogFileMetadata := filestream.ActionMetadata{
|
|
Action: filestream.Action(action),
|
|
Filepath: getBinlogFilepath(binlogFile),
|
|
RequestId: uuid.New().String(),
|
|
Sink: binlogFile.SinkName,
|
|
OssBufferSize: strconv.FormatInt(binlogFile.Size, 10),
|
|
}
|
|
uploadedLen, err := client.Upload(reader, binlogFileMetadata)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if uploadedLen != binlogFile.Size {
|
|
return fmt.Errorf("not the same len contentSize=%d, uploadSize=%d", binlogFile.Size, uploadedLen)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func uploadBinlogFileMeta(binlogFile *BinlogFile) error {
|
|
//upload binlogfile meta
|
|
client := filestream.NewFileClient(FilestreamIp, filestreamPort, nil)
|
|
action := filestream.UploadOss
|
|
if binlogFile.SinkType == config.SinkTypeSftp {
|
|
action = filestream.UploadSsh
|
|
}
|
|
binlogMetaJsonBytes, _ := json.Marshal(binlogFile)
|
|
binlogMetaFileMetadata := filestream.ActionMetadata{
|
|
Action: filestream.Action(action),
|
|
Filepath: getBinlogMetaFilepath(binlogFile),
|
|
RequestId: uuid.New().String(),
|
|
Sink: binlogFile.SinkName,
|
|
OssBufferSize: fmt.Sprintf("%d", len(binlogMetaJsonBytes)),
|
|
}
|
|
reader := bytes.NewReader(binlogMetaJsonBytes)
|
|
uploadedLen, err := client.Upload(reader, binlogMetaFileMetadata)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
jsonByteLen := int64(len(binlogMetaJsonBytes))
|
|
if uploadedLen != jsonByteLen {
|
|
return fmt.Errorf("not the same len contentSize=%d, uploadSize=%d", jsonByteLen, uploadedLen)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func getBatchName(num int64) string {
|
|
times := num / BatchSize
|
|
return fmt.Sprintf("%d_%d", times*BatchSize, (times+1)*BatchSize)
|
|
}
|
|
|
|
func getBinlogFilepath(binlogFile *BinlogFile) string {
|
|
batchName := getBatchName(binlogFile.Num)
|
|
return fmt.Sprintf(BinlogFilepathFormat, config.GetBinlogStoragePathPrefix(), binlogFile.Namespace, binlogFile.PxcName, binlogFile.PxcUid, binlogFile.XStoreName, binlogFile.XStoreUid, binlogFile.PodName, binlogFile.Version, batchName, binlogFile.Filename)
|
|
}
|
|
|
|
func getBinlogMetaFilepath(binlogFile *BinlogFile) string {
|
|
batchName := getBatchName(binlogFile.Num)
|
|
filename := binlogFile.Filename + ".txt"
|
|
return fmt.Sprintf(BinlogMetaFilepathFormat, config.GetBinlogStoragePathPrefix(), binlogFile.Namespace, binlogFile.PxcName, binlogFile.PxcUid, binlogFile.XStoreName, binlogFile.XStoreUid, binlogFile.PodName, binlogFile.Version, batchName, filename)
|
|
}
|
|
|
|
// RecordUpload add an upload record into db
|
|
func RecordUpload(w *Watcher, binlogFile *BinlogFile) bool {
|
|
logger := w.uploadLogger.WithValues("action", "RecordUpload")
|
|
logger.Info("begin")
|
|
err := AddRecord(w.db, *binlogFile)
|
|
if err != nil {
|
|
logger.Error(err, "failed to record upload file")
|
|
//ignore the error
|
|
}
|
|
logger.Info("success")
|
|
return true
|
|
}
|