polardbxoperator/pkg/hpfs/remote/aliyun_oss.go

702 lines
18 KiB
Go

/*
Copyright 2021 Alibaba Group Holding Limited.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package remote
import (
"bytes"
"context"
"encoding/json"
"fmt"
"github.com/alibaba/polardbx-operator/pkg/hpfs/common"
polarxIo "github.com/alibaba/polardbx-operator/pkg/util/io"
polarxPath "github.com/alibaba/polardbx-operator/pkg/util/path"
"github.com/aliyun/aliyun-oss-go-sdk/oss"
"github.com/eapache/queue"
"io"
"os"
"path/filepath"
"strconv"
"sync/atomic"
"time"
)
const (
LimitedReaderSize = 1 << 20 * 600 //600MB
MaxPartSize = (1 << 30) * 5 //5GB
)
func init() {
MustRegisterFileService("aliyun-oss", &aliyunOssFs{})
}
type aliyunOssFs struct{}
func (o *aliyunOssFs) newClient(ossCtx *aliyunOssContext) (*oss.Client, error) {
return oss.New(ossCtx.endpoint, ossCtx.accessKey, ossCtx.accessSecret, oss.Timeout(10, 3600*2))
}
func (o *aliyunOssFs) DeleteFile(ctx context.Context, path string, auth, params map[string]string) error {
ossCtx, err := newAliyunOssContext(ctx, auth, params)
if err != nil {
return err
}
client, err := o.newClient(ossCtx)
if err != nil {
return fmt.Errorf("failed to create oss client: %w", err)
}
bucket, err := client.Bucket(ossCtx.bucket)
if err != nil {
return fmt.Errorf("failed to open oss bucket: %w", err)
}
return bucket.DeleteObject(path)
}
func (o aliyunOssFs) DeleteExpiredFile(ctx context.Context, path string, auth, params map[string]string) (FileTask, error) {
ossCtx, err := newAliyunOssContext(ctx, auth, params)
if err != nil {
return nil, err
}
client, err := o.newClient(ossCtx)
if err != nil {
return nil, fmt.Errorf("failed to create oss client: %w", err)
}
bucket, err := client.Bucket(ossCtx.bucket)
if err != nil {
return nil, fmt.Errorf("failed to open oss bucket: %w", err)
}
ft := newFileTask(ctx)
go func() {
err := o.ListFileWithDeadline(path, bucket, ossCtx.deadline, func(objs []string) error {
_, err := bucket.DeleteObjects(objs)
if err != nil {
return err
}
for _, obj := range objs {
if val, ok := ctx.Value(common.AffectedFiles).(*[]string); ok {
*val = append(*val, obj)
}
}
return nil
})
ft.complete(err)
}()
return ft, nil
}
type ossProgressListener4FileTask struct {
*fileTask
}
func (l *ossProgressListener4FileTask) ProgressChanged(event *oss.ProgressEvent) {
l.progress = int32(event.ConsumedBytes * 100 / event.TotalBytes)
}
func (o *aliyunOssFs) UploadFileNormally(ctx context.Context, reader io.Reader, path string, auth, params map[string]string) (FileTask, error) {
ossCtx, err := newAliyunOssContext(ctx, auth, params)
if err != nil {
return nil, err
}
client, err := o.newClient(ossCtx)
if err != nil {
return nil, fmt.Errorf("failed to create oss client: %w", err)
}
bucket, err := client.Bucket(ossCtx.bucket)
if err != nil {
return nil, fmt.Errorf("failed to open oss bucket: %w", err)
}
ft := newFileTask(ctx)
go func() {
opts := []oss.Option{
oss.Progress(&ossProgressListener4FileTask{fileTask: ft}),
}
// Expires at specified time.
if ossCtx.retentionTime > 0 {
opts = append(opts, oss.Expires(time.Now().Add(ossCtx.retentionTime)))
}
ft.complete(bucket.PutObject(path, reader, opts...))
}()
return ft, nil
}
func (o *aliyunOssFs) UploadFile(ctx context.Context, reader io.Reader, path string, auth, params map[string]string) (FileTask, error) {
ossCtx, err := newAliyunOssContext(ctx, auth, params)
var limitReaderSize int64 = LimitedReaderSize
val, ok := params["limit_reader_size"]
if ok && val != "" {
parsedVal, err := strconv.ParseInt(val, 10, 64)
if err != nil {
panic(err)
}
limitReaderSize = parsedVal
}
if err != nil {
return nil, err
}
client, err := o.newClient(ossCtx)
if err != nil {
return nil, fmt.Errorf("failed to create oss client: %w", err)
}
bucket, err := client.Bucket(ossCtx.bucket)
if err != nil {
return nil, fmt.Errorf("failed to open oss bucket: %w", err)
}
ft := newFileTask(ctx)
go func() {
opts := []oss.Option{
oss.Progress(&ossProgressListener4FileTask{fileTask: ft}),
}
// Expires at specified time.
if ossCtx.retentionTime > 0 {
opts = append(opts, oss.Expires(time.Now().Add(ossCtx.retentionTime)))
}
if err != nil {
ft.complete(err)
return
}
var partIndex int = 1
imur, err := bucket.InitiateMultipartUpload(path, opts...)
if err != nil {
ft.complete(err)
return
}
//complete := false
defer func() {
bucket.AbortMultipartUpload(imur)
}()
pipeReader, pipeWriter := io.Pipe()
defer pipeReader.Close()
var actualSize int64
go func() {
defer pipeWriter.Close()
buff := make([]byte, ossCtx.bufferSize)
written, err := io.CopyBuffer(pipeWriter, reader, buff)
if written > 0 {
if written%limitReaderSize != 0 {
toFillBytes := limitReaderSize - (written % limitReaderSize)
buff = make([]byte, ossCtx.bufferSize)
for {
if toFillBytes == 0 {
break
}
if toFillBytes > ossCtx.bufferSize {
pipeWriter.Write(buff)
toFillBytes -= ossCtx.bufferSize
} else {
pipeWriter.Write(buff[:toFillBytes])
toFillBytes -= toFillBytes
}
}
}
atomic.StoreInt64(&actualSize, written)
}
if err != nil {
ft.complete(err)
return
}
}()
var uploadedLen int64
parts := make([]oss.UploadPart, 0)
emptyBytes := make([]byte, 0)
for {
_, err := pipeReader.Read(emptyBytes)
if err != nil {
break
}
limitedReader := io.LimitReader(pipeReader, limitReaderSize)
uploadPart, err := bucket.UploadPart(imur, limitedReader, limitReaderSize, partIndex, opts...)
partIndex++
uploadedLen += limitReaderSize
if err != nil {
break
}
parts = append(parts, uploadPart)
}
if len(parts) > 0 {
_, err = bucket.CompleteMultipartUpload(imur, parts, opts...)
if err != nil {
ft.complete(err)
return
}
totalSize := atomic.LoadInt64(&actualSize)
SetTags(bucket, path, actualSize)
ft.complete(nil)
var copyPosition int64
pageNumber := 1
copiedParts := make([]oss.UploadPart, 0)
if totalSize%limitReaderSize != 0 {
imur, err = bucket.InitiateMultipartUpload(path, opts...)
if err != nil {
return
}
for {
partSize := totalSize - copyPosition
if partSize >= MaxPartSize {
partSize = MaxPartSize
}
copiedUploadPart, err := bucket.UploadPartCopy(imur, bucket.BucketName, path, copyPosition, partSize, pageNumber, opts...)
if err != nil {
return
}
pageNumber++
copiedParts = append(copiedParts, copiedUploadPart)
copyPosition += partSize
if copyPosition == totalSize {
_, err = bucket.CompleteMultipartUpload(imur, copiedParts, opts...)
if err != nil {
return
}
SetTags(bucket, path, actualSize)
break
}
}
}
} else {
ft.complete(nil)
}
}()
return ft, nil
}
func SetTags(bucket *oss.Bucket, objKey string, actualSize int64) {
uploaderTag := oss.Tag{
Key: "uploader",
Value: "hpfs",
}
sizeTag := oss.Tag{
Key: "size",
Value: strconv.FormatInt(actualSize, 10),
}
tagging := oss.Tagging{
Tags: []oss.Tag{uploaderTag, sizeTag},
}
bucket.PutObjectTagging(objKey, tagging)
}
func GetActualSizeFromTags(bucket *oss.Bucket, objKey string) int64 {
hpfsUpload := false
var size int64 = -1
tagResult, err := bucket.GetObjectTagging(objKey)
if err != nil {
return size
}
if tagResult.Tags != nil && len(tagResult.Tags) > 2 {
sizeStr := ""
for _, tag := range tagResult.Tags {
if tag.Key == "uploader" && tag.Value == "hpfs" {
hpfsUpload = true
}
if tag.Key == "size" {
sizeStr = tag.Value
}
}
if hpfsUpload && sizeStr != "" {
size, err = strconv.ParseInt(sizeStr, 10, 64)
if err != nil {
size = -1
}
}
}
if !hpfsUpload {
size = -1
}
return size
}
func (o *aliyunOssFs) UploadFileTmpFile(ctx context.Context, reader io.Reader, path string, auth, params map[string]string) (FileTask, error) {
ossCtx, err := newAliyunOssContext(ctx, auth, params)
if err != nil {
return nil, err
}
if !ossCtx.useTmpFile {
return o.UploadFileNormally(ctx, reader, path, auth, params)
}
client, err := o.newClient(ossCtx)
if err != nil {
return nil, fmt.Errorf("failed to create oss client: %w", err)
}
bucket, err := client.Bucket(ossCtx.bucket)
if err != nil {
return nil, fmt.Errorf("failed to open oss bucket: %w", err)
}
ft := newFileTask(ctx)
go func() {
opts := []oss.Option{
oss.Progress(&ossProgressListener4FileTask{fileTask: ft}),
}
// Expires at specified time.
if ossCtx.retentionTime > 0 {
opts = append(opts, oss.Expires(time.Now().Add(ossCtx.retentionTime)))
}
imur, err := bucket.InitiateMultipartUpload(path, opts...)
complete := false
if err != nil {
ft.complete(err)
return
}
defer func() {
if !complete {
bucket.AbortMultipartUpload(imur, opts...)
}
}()
parts := make([]oss.UploadPart, 0)
tmpFileChan := make(chan string)
tmpFileDir := filepath.Join("/tmp/oss", ossCtx.bucket)
err = os.MkdirAll(tmpFileDir, os.ModePerm)
if err != nil {
ft.complete(err)
return
}
defer func() {
os.RemoveAll(tmpFileDir)
}()
go func() {
defer close(tmpFileChan)
fileIndex := 0
var writeTmpFile *os.File
defer func() {
if writeTmpFile != nil {
writeTmpFile.Close()
}
}()
for {
tempFilePath := filepath.Join(tmpFileDir, strconv.FormatInt(int64(fileIndex), 10))
writeTmpFile, err = os.OpenFile(tempFilePath, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
fileIndex++
if err != nil {
ft.complete(err)
return
}
copiedNum, _ := io.CopyN(writeTmpFile, reader, ossCtx.bufferSize)
writeTmpFile.Close()
writeTmpFile = nil
if copiedNum > 0 {
tmpFileChan <- tempFilePath
}
if copiedNum < ossCtx.bufferSize {
return
}
}
}()
var readTmpFile *os.File
defer func() {
if readTmpFile != nil {
readTmpFile.Close()
}
}()
for {
tempFilePath, ok := <-tmpFileChan
if !ok {
break
}
fileInfo, err := os.Stat(tempFilePath)
if err != nil {
ft.complete(err)
return
}
readTmpFile, err = os.OpenFile(tempFilePath, os.O_RDWR, 0644)
if err != nil {
ft.complete(err)
return
}
uploadPart, err := bucket.UploadPart(imur, readTmpFile, fileInfo.Size(), len(parts)+1, opts...)
readTmpFile.Close()
readTmpFile = nil
os.Remove(tempFilePath)
if err != nil {
ft.complete(err)
return
}
parts = append(parts, uploadPart)
}
_, err = bucket.CompleteMultipartUpload(imur, parts, opts...)
if err != nil {
complete = true
}
ft.complete(err)
}()
return ft, nil
}
func (o *aliyunOssFs) DownloadFile(ctx context.Context, writer io.Writer, path string, auth, params map[string]string) (FileTask, error) {
ossCtx, err := newAliyunOssContext(ctx, auth, params)
if err != nil {
return nil, err
}
client, err := o.newClient(ossCtx)
if err != nil {
return nil, fmt.Errorf("failed to create oss client: %w", err)
}
bucket, err := client.Bucket(ossCtx.bucket)
if err != nil {
return nil, fmt.Errorf("failed to open oss bucket: %w", err)
}
ft := newFileTask(ctx)
go func() {
var bytesCount int64
if ossCtx.writeLen {
r, err := bucket.GetObjectMeta(path)
if err != nil {
ft.complete(fmt.Errorf("failed to get object meta: %w", err))
return
}
contentLength := r.Get("Content-Length")
bytesCount, _ = strconv.ParseInt(contentLength, 10, 64)
actualSize := GetActualSizeFromTags(bucket, path)
if actualSize != -1 {
bytesCount = actualSize
}
polarxIo.WriteUint64(writer, uint64(bytesCount))
}
r, err := bucket.GetObject(path)
if err != nil {
ft.complete(fmt.Errorf("failed to get object: %w", err))
return
}
if ossCtx.writeLen {
_, err = io.CopyN(writer, r, bytesCount)
} else {
_, err = io.Copy(writer, r)
}
if err != nil {
ft.complete(fmt.Errorf("failed to copy content: %w", err))
return
}
ft.complete(nil)
}()
return ft, nil
}
func (o *aliyunOssFs) ListFiles(ctx context.Context, writer io.Writer, path string, auth, params map[string]string) (FileTask, error) {
ossCtx, err := newAliyunOssContext(ctx, auth, params)
if err != nil {
return nil, err
}
client, err := o.newClient(ossCtx)
if err != nil {
return nil, fmt.Errorf("failed to create oss client: %w", err)
}
bucket, err := client.Bucket(ossCtx.bucket)
if err != nil {
return nil, fmt.Errorf("failed to open oss bucket: %w", err)
}
ft := newFileTask(ctx)
go func() {
// list entries in path use oss sdk
entryNames := make([]string, 0)
marker := ""
prefix := oss.Prefix(path)
delimiter := oss.Delimiter("/")
for {
lsRes, err := bucket.ListObjects(oss.Marker(marker), prefix, delimiter)
if err != nil {
ft.complete(fmt.Errorf("failed to list oss objects in path %s: %w", path, err))
return
}
for _, object := range lsRes.Objects { // file
entryNames = append(entryNames, polarxPath.GetBaseNameFromPath(object.Key))
}
for _, dir := range lsRes.CommonPrefixes { // subdirectory
entryNames = append(entryNames, polarxPath.GetBaseNameFromPath(dir))
}
if lsRes.IsTruncated {
marker = lsRes.NextMarker
} else {
break
}
}
// parse entry slice and send response
encodedEntryNames, err := json.Marshal(entryNames)
if err != nil {
ft.complete(fmt.Errorf("failed to encode entry name slice,: %w", err))
return
}
if ossCtx.writeLen {
bytesCount := int64(len(encodedEntryNames))
err := polarxIo.WriteUint64(writer, uint64(bytesCount))
if err != nil {
ft.complete(fmt.Errorf("failed to send content bytes count: %w", err))
return
}
_, err = io.CopyN(writer, bytes.NewReader(encodedEntryNames), bytesCount)
} else {
_, err = io.Copy(writer, bytes.NewReader(encodedEntryNames))
}
if err != nil {
ft.complete(fmt.Errorf("failed to copy content: %w", err))
return
}
ft.complete(nil)
}()
return ft, nil
}
func (o *aliyunOssFs) ListFileWithDeadline(path string, bucket *oss.Bucket, deadline int64, callback func([]string) error) error {
marker := ""
delimiter := oss.Delimiter("/")
fileQueue := queue.New()
fileQueue.Add([]string{path, marker})
for fileQueue.Length() != 0 {
element := fileQueue.Remove().([]string)
lsRes, err := bucket.ListObjects(oss.Marker(element[1]), oss.Prefix(element[0]), delimiter)
if err != nil {
return err
}
if lsRes.IsTruncated {
fileQueue.Add([]string{element[0], lsRes.NextMarker})
}
for _, commonPrefix := range lsRes.CommonPrefixes {
fileQueue.Add([]string{commonPrefix, ""})
}
objs := make([]string, 0)
for _, obj := range lsRes.Objects {
if obj.LastModified.Unix() < deadline {
// delete it
objs = append(objs, obj.Key)
}
}
if len(objs) > 0 {
err := callback(objs)
if err != nil {
return err
}
}
}
return nil
}
func (o *aliyunOssFs) ListAllFiles(ctx context.Context, path string, auth, params map[string]string) (FileTask, error) {
ossCtx, err := newAliyunOssContext(ctx, auth, params)
if err != nil {
return nil, err
}
client, err := o.newClient(ossCtx)
if err != nil {
return nil, fmt.Errorf("failed to create oss client: %w", err)
}
bucket, err := client.Bucket(ossCtx.bucket)
if err != nil {
return nil, fmt.Errorf("failed to open oss bucket: %w", err)
}
ft := newFileTask(ctx)
go func() {
err := o.ListFileWithDeadline(path, bucket, ossCtx.deadline, func(objs []string) error {
for _, obj := range objs {
if val, ok := ctx.Value(common.AffectedFiles).(*[]string); ok {
*val = append(*val, obj)
}
}
return nil
})
ft.complete(err)
}()
return ft, nil
}
type aliyunOssContext struct {
ctx context.Context
endpoint string
accessKey string
accessSecret string
bucket string
retentionTime time.Duration
writeLen bool
bufferSize int64
useTmpFile bool
deadline int64
}
func newAliyunOssContext(ctx context.Context, auth, params map[string]string) (*aliyunOssContext, error) {
var writeLen bool
if val, ok := params["write_len"]; ok {
toWriteLenVal, err := strconv.ParseBool(val)
if err != nil {
return nil, err
}
writeLen = toWriteLenVal
}
var bufferSize int64 = 1 << 20 * 50 //50MB
if val, ok := params["buffer_size"]; ok {
toBufferSize, err := strconv.ParseInt(val, 10, 64)
if err != nil {
return nil, err
}
bufferSize = toBufferSize
}
var useTmpFile bool = true
if val, ok := params["use_tmp_file"]; ok {
toUseTmpFile, err := strconv.ParseBool(val)
if err != nil {
return nil, err
}
useTmpFile = toUseTmpFile
}
var deadline int64 = 0
if val, ok := params["deadline"]; ok {
parsedDeadline, err := strconv.ParseInt(val, 10, 64)
if err != nil {
return nil, err
}
deadline = parsedDeadline
}
ossCtx := &aliyunOssContext{
ctx: ctx,
endpoint: auth["endpoint"],
accessKey: auth["access_key"],
accessSecret: auth["access_secret"],
bucket: params["bucket"],
writeLen: writeLen,
bufferSize: bufferSize,
useTmpFile: useTmpFile,
deadline: deadline,
}
if t, ok := params["retention-time"]; ok {
d, err := time.ParseDuration(t)
if err != nil {
return nil, fmt.Errorf("format error for retention-time: %w", err)
}
ossCtx.retentionTime = d
} else {
ossCtx.retentionTime = 0
}
return ossCtx, nil
}