init
This commit is contained in:
427
pkg/filesystem/driver/cos/handler.go
Normal file
427
pkg/filesystem/driver/cos/handler.go
Normal file
@ -0,0 +1,427 @@
|
||||
package cos
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/sha1"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/request"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/util"
|
||||
"github.com/google/go-querystring/query"
|
||||
cossdk "github.com/tencentyun/cos-go-sdk-v5"
|
||||
)
|
||||
|
||||
// UploadPolicy 腾讯云COS上传策略
|
||||
type UploadPolicy struct {
|
||||
Expiration string `json:"expiration"`
|
||||
Conditions []interface{} `json:"conditions"`
|
||||
}
|
||||
|
||||
// MetaData 文件元信息
|
||||
type MetaData struct {
|
||||
Size uint64
|
||||
CallbackKey string
|
||||
CallbackURL string
|
||||
}
|
||||
|
||||
type urlOption struct {
|
||||
Speed int `url:"x-cos-traffic-limit,omitempty"`
|
||||
ContentDescription string `url:"response-content-disposition,omitempty"`
|
||||
}
|
||||
|
||||
// Driver 腾讯云COS适配器模板
|
||||
type Driver struct {
|
||||
Policy *model.Policy
|
||||
Client *cossdk.Client
|
||||
HTTPClient request.Client
|
||||
}
|
||||
|
||||
// List 列出COS文件
|
||||
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
|
||||
// 初始化列目录参数
|
||||
opt := &cossdk.BucketGetOptions{
|
||||
Prefix: strings.TrimPrefix(base, "/"),
|
||||
EncodingType: "",
|
||||
MaxKeys: 1000,
|
||||
}
|
||||
// 是否为递归列出
|
||||
if !recursive {
|
||||
opt.Delimiter = "/"
|
||||
}
|
||||
// 手动补齐结尾的slash
|
||||
if opt.Prefix != "" {
|
||||
opt.Prefix += "/"
|
||||
}
|
||||
|
||||
var (
|
||||
marker string
|
||||
objects []cossdk.Object
|
||||
commons []string
|
||||
)
|
||||
|
||||
for {
|
||||
res, _, err := handler.Client.Bucket.Get(ctx, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
objects = append(objects, res.Contents...)
|
||||
commons = append(commons, res.CommonPrefixes...)
|
||||
// 如果本次未列取完，则继续使用marker获取结果
|
||||
marker = res.NextMarker
|
||||
// marker 为空时结果列取完毕，跳出
|
||||
if marker == "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// 处理列取结果
|
||||
res := make([]response.Object, 0, len(objects)+len(commons))
|
||||
// 处理目录
|
||||
for _, object := range commons {
|
||||
rel, err := filepath.Rel(opt.Prefix, object)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res = append(res, response.Object{
|
||||
Name: path.Base(object),
|
||||
RelativePath: filepath.ToSlash(rel),
|
||||
Size: 0,
|
||||
IsDir: true,
|
||||
LastModify: time.Now(),
|
||||
})
|
||||
}
|
||||
// 处理文件
|
||||
for _, object := range objects {
|
||||
rel, err := filepath.Rel(opt.Prefix, object.Key)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
res = append(res, response.Object{
|
||||
Name: path.Base(object.Key),
|
||||
Source: object.Key,
|
||||
RelativePath: filepath.ToSlash(rel),
|
||||
Size: uint64(object.Size),
|
||||
IsDir: false,
|
||||
LastModify: time.Now(),
|
||||
})
|
||||
}
|
||||
|
||||
return res, nil
|
||||
|
||||
}
|
||||
|
||||
// CORS 创建跨域策略
|
||||
func (handler Driver) CORS() error {
|
||||
_, err := handler.Client.Bucket.PutCORS(context.Background(), &cossdk.BucketPutCORSOptions{
|
||||
Rules: []cossdk.BucketCORSRule{{
|
||||
AllowedMethods: []string{
|
||||
"GET",
|
||||
"POST",
|
||||
"PUT",
|
||||
"DELETE",
|
||||
"HEAD",
|
||||
},
|
||||
AllowedOrigins: []string{"*"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
MaxAgeSeconds: 3600,
|
||||
ExposeHeaders: []string{},
|
||||
}},
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Get 获取文件
|
||||
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
|
||||
// 获取文件源地址
|
||||
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 获取文件数据流
|
||||
resp, err := handler.HTTPClient.Request(
|
||||
"GET",
|
||||
downloadURL,
|
||||
nil,
|
||||
request.WithContext(ctx),
|
||||
request.WithTimeout(time.Duration(0)),
|
||||
).CheckHTTPResponse(200).GetRSCloser()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp.SetFirstFakeChunk()
|
||||
|
||||
// 尝试自主获取文件大小
|
||||
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
|
||||
resp.SetContentLength(int64(file.Size))
|
||||
}
|
||||
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Put 将文件流保存到指定目录
|
||||
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
|
||||
defer file.Close()
|
||||
|
||||
opt := &cossdk.ObjectPutOptions{}
|
||||
_, err := handler.Client.Object.Put(ctx, file.Info().SavePath, file, opt)
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete 删除一个或多个文件，
|
||||
// 返回未删除的文件，及遇到的最后一个错误
|
||||
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
|
||||
obs := []cossdk.Object{}
|
||||
for _, v := range files {
|
||||
obs = append(obs, cossdk.Object{Key: v})
|
||||
}
|
||||
opt := &cossdk.ObjectDeleteMultiOptions{
|
||||
Objects: obs,
|
||||
Quiet: true,
|
||||
}
|
||||
|
||||
res, _, err := handler.Client.Object.DeleteMulti(context.Background(), opt)
|
||||
if err != nil {
|
||||
return files, err
|
||||
}
|
||||
|
||||
// 整理删除结果
|
||||
failed := make([]string, 0, len(files))
|
||||
for _, v := range res.Errors {
|
||||
failed = append(failed, v.Key)
|
||||
}
|
||||
|
||||
if len(failed) == 0 {
|
||||
return failed, nil
|
||||
}
|
||||
|
||||
return failed, errors.New("delete failed")
|
||||
}
|
||||
|
||||
// Thumb 获取文件缩略图
|
||||
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
|
||||
// quick check by extension name
|
||||
// https://cloud.tencent.com/document/product/436/44893
|
||||
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "heif", "heic"}
|
||||
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
|
||||
supported = handler.Policy.OptionsSerialized.ThumbExts
|
||||
}
|
||||
|
||||
if !util.IsInExtensionList(supported, file.Name) || file.Size > (32<<(10*2)) {
|
||||
return nil, driver.ErrorThumbNotSupported
|
||||
}
|
||||
|
||||
var (
|
||||
thumbSize = [2]uint{400, 300}
|
||||
ok = false
|
||||
)
|
||||
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
|
||||
return nil, errors.New("failed to get thumbnail size")
|
||||
}
|
||||
|
||||
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
|
||||
|
||||
thumbParam := fmt.Sprintf("imageMogr2/thumbnail/%dx%d/quality/%d", thumbSize[0], thumbSize[1], thumbEncodeQuality)
|
||||
|
||||
source, err := handler.signSourceURL(
|
||||
ctx,
|
||||
file.SourceName,
|
||||
int64(model.GetIntSetting("preview_timeout", 60)),
|
||||
&urlOption{},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
thumbURL, _ := url.Parse(source)
|
||||
thumbQuery := thumbURL.Query()
|
||||
thumbQuery.Add(thumbParam, "")
|
||||
thumbURL.RawQuery = thumbQuery.Encode()
|
||||
|
||||
return &response.ContentResponse{
|
||||
Redirect: true,
|
||||
URL: thumbURL.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Source 获取外链URL
|
||||
func (handler Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
|
||||
// 尝试从上下文获取文件名
|
||||
fileName := ""
|
||||
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
|
||||
fileName = file.Name
|
||||
}
|
||||
|
||||
// 添加各项设置
|
||||
options := urlOption{}
|
||||
if speed > 0 {
|
||||
if speed < 819200 {
|
||||
speed = 819200
|
||||
}
|
||||
if speed > 838860800 {
|
||||
speed = 838860800
|
||||
}
|
||||
options.Speed = speed
|
||||
}
|
||||
if isDownload {
|
||||
options.ContentDescription = "attachment; filename=\"" + url.PathEscape(fileName) + "\""
|
||||
}
|
||||
|
||||
return handler.signSourceURL(ctx, path, ttl, &options)
|
||||
}
|
||||
|
||||
func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, options *urlOption) (string, error) {
|
||||
cdnURL, err := url.Parse(handler.Policy.BaseURL)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 公有空间不需要签名
|
||||
if !handler.Policy.IsPrivate {
|
||||
file, err := url.Parse(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 非签名URL不支持设置响应header
|
||||
options.ContentDescription = ""
|
||||
|
||||
optionQuery, err := query.Values(*options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
file.RawQuery = optionQuery.Encode()
|
||||
sourceURL := cdnURL.ResolveReference(file)
|
||||
|
||||
return sourceURL.String(), nil
|
||||
}
|
||||
|
||||
presignedURL, err := handler.Client.Object.GetPresignedURL(ctx, http.MethodGet, path,
|
||||
handler.Policy.AccessKey, handler.Policy.SecretKey, time.Duration(ttl)*time.Second, options)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// 将最终生成的签名URL域名换成用户自定义的加速域名（如果有）
|
||||
presignedURL.Host = cdnURL.Host
|
||||
presignedURL.Scheme = cdnURL.Scheme
|
||||
|
||||
return presignedURL.String(), nil
|
||||
}
|
||||
|
||||
// Token 获取上传策略和认证Token
|
||||
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
|
||||
// 生成回调地址
|
||||
siteURL := model.GetSiteURL()
|
||||
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + uploadSession.Key)
|
||||
apiURL := siteURL.ResolveReference(apiBaseURI).String()
|
||||
|
||||
// 上传策略
|
||||
savePath := file.Info().SavePath
|
||||
startTime := time.Now()
|
||||
endTime := startTime.Add(time.Duration(ttl) * time.Second)
|
||||
keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix())
|
||||
postPolicy := UploadPolicy{
|
||||
Expiration: endTime.UTC().Format(time.RFC3339),
|
||||
Conditions: []interface{}{
|
||||
map[string]string{"bucket": handler.Policy.BucketName},
|
||||
map[string]string{"$key": savePath},
|
||||
map[string]string{"x-cos-meta-callback": apiURL},
|
||||
map[string]string{"x-cos-meta-key": uploadSession.Key},
|
||||
map[string]string{"q-sign-algorithm": "sha1"},
|
||||
map[string]string{"q-ak": handler.Policy.AccessKey},
|
||||
map[string]string{"q-sign-time": keyTime},
|
||||
},
|
||||
}
|
||||
|
||||
if handler.Policy.MaxSize > 0 {
|
||||
postPolicy.Conditions = append(postPolicy.Conditions,
|
||||
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
|
||||
}
|
||||
|
||||
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, savePath)
|
||||
if err == nil {
|
||||
res.SessionID = uploadSession.Key
|
||||
res.Callback = apiURL
|
||||
res.UploadURLs = []string{handler.Policy.Server}
|
||||
}
|
||||
|
||||
return res, err
|
||||
|
||||
}
|
||||
|
||||
// 取消上传凭证
|
||||
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Meta 获取文件信息
|
||||
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
|
||||
res, err := handler.Client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &MetaData{
|
||||
Size: uint64(res.ContentLength),
|
||||
CallbackKey: res.Header.Get("x-cos-meta-key"),
|
||||
CallbackURL: res.Header.Get("x-cos-meta-callback"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string, savePath string) (*serializer.UploadCredential, error) {
|
||||
// 编码上传策略
|
||||
policyJSON, err := json.Marshal(policy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
|
||||
|
||||
// 签名上传策略
|
||||
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
|
||||
_, err = io.WriteString(hmacSign, keyTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signKey := fmt.Sprintf("%x", hmacSign.Sum(nil))
|
||||
|
||||
sha1Sign := sha1.New()
|
||||
_, err = sha1Sign.Write(policyJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stringToSign := fmt.Sprintf("%x", sha1Sign.Sum(nil))
|
||||
|
||||
// 最终签名
|
||||
hmacFinalSign := hmac.New(sha1.New, []byte(signKey))
|
||||
_, err = hmacFinalSign.Write([]byte(stringToSign))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
signature := hmacFinalSign.Sum(nil)
|
||||
|
||||
return &serializer.UploadCredential{
|
||||
Policy: policyEncoded,
|
||||
Path: savePath,
|
||||
AccessKey: handler.Policy.AccessKey,
|
||||
Credential: fmt.Sprintf("%x", signature),
|
||||
KeyTime: keyTime,
|
||||
}, nil
|
||||
}
|
||||
134
pkg/filesystem/driver/cos/scf.go
Normal file
134
pkg/filesystem/driver/cos/scf.go
Normal file
@ -0,0 +1,134 @@
|
||||
package cos
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
model "github.com/cloudreve/Cloudreve/v3/models"
|
||||
"github.com/cloudreve/Cloudreve/v3/pkg/hashid"
|
||||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
|
||||
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
|
||||
scf "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/scf/v20180416"
|
||||
)
|
||||
|
||||
const scfFunc = `# -*- coding: utf8 -*-
|
||||
# SCF配置COS触发，向 Cloudreve 发送回调
|
||||
from qcloud_cos_v5 import CosConfig
|
||||
from qcloud_cos_v5 import CosS3Client
|
||||
from qcloud_cos_v5 import CosServiceError
|
||||
from qcloud_cos_v5 import CosClientError
|
||||
import sys
|
||||
import logging
|
||||
import requests
|
||||
|
||||
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
def main_handler(event, context):
|
||||
logger.info("start main handler")
|
||||
for record in event['Records']:
|
||||
try:
|
||||
if "x-cos-meta-callback" not in record['cos']['cosObject']['meta']:
|
||||
logger.info("Cannot find callback URL, skiped.")
|
||||
return 'Success'
|
||||
callback = record['cos']['cosObject']['meta']['x-cos-meta-callback']
|
||||
key = record['cos']['cosObject']['key']
|
||||
logger.info("Callback URL is " + callback)
|
||||
|
||||
r = requests.get(callback)
|
||||
print(r.text)
|
||||
|
||||
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print('Error getting object {} callback url. '.format(key))
|
||||
raise e
|
||||
return "Fail"
|
||||
|
||||
return "Success"
|
||||
`
|
||||
|
||||
// CreateSCF 创建回调云函数
|
||||
func CreateSCF(policy *model.Policy, region string) error {
|
||||
// 初始化客户端
|
||||
credential := common.NewCredential(
|
||||
policy.AccessKey,
|
||||
policy.SecretKey,
|
||||
)
|
||||
cpf := profile.NewClientProfile()
|
||||
client, err := scf.NewClient(credential, region, cpf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// 创建回调代码数据
|
||||
buff := &bytes.Buffer{}
|
||||
bs64 := base64.NewEncoder(base64.StdEncoding, buff)
|
||||
zipWriter := zip.NewWriter(bs64)
|
||||
header := zip.FileHeader{
|
||||
Name: "callback.py",
|
||||
Method: zip.Deflate,
|
||||
}
|
||||
writer, err := zipWriter.CreateHeader(&header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = io.Copy(writer, strings.NewReader(scfFunc))
|
||||
zipWriter.Close()
|
||||
|
||||
// 创建云函数
|
||||
req := scf.NewCreateFunctionRequest()
|
||||
funcName := "cloudreve_" + hashid.HashID(policy.ID, hashid.PolicyID) + strconv.FormatInt(time.Now().Unix(), 10)
|
||||
zipFileBytes, _ := ioutil.ReadAll(buff)
|
||||
zipFileStr := string(zipFileBytes)
|
||||
codeSource := "ZipFile"
|
||||
handler := "callback.main_handler"
|
||||
desc := "Cloudreve 用回调函数"
|
||||
timeout := int64(60)
|
||||
runtime := "Python3.6"
|
||||
req.FunctionName = &funcName
|
||||
req.Code = &scf.Code{
|
||||
ZipFile: &zipFileStr,
|
||||
}
|
||||
req.Handler = &handler
|
||||
req.Description = &desc
|
||||
req.Timeout = &timeout
|
||||
req.Runtime = &runtime
|
||||
req.CodeSource = &codeSource
|
||||
|
||||
_, err = client.CreateFunction(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(5) * time.Second)
|
||||
|
||||
// 创建触发器
|
||||
server, _ := url.Parse(policy.Server)
|
||||
triggerType := "cos"
|
||||
triggerDesc := `{"event":"cos:ObjectCreated:Post","filter":{"Prefix":"","Suffix":""}}`
|
||||
enable := "OPEN"
|
||||
|
||||
trigger := scf.NewCreateTriggerRequest()
|
||||
trigger.FunctionName = &funcName
|
||||
trigger.TriggerName = &server.Host
|
||||
trigger.Type = &triggerType
|
||||
trigger.TriggerDesc = &triggerDesc
|
||||
trigger.Enable = &enable
|
||||
|
||||
_, err = client.CreateTrigger(trigger)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user