earthjasonlin/CloudrevePlus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

init

Browse Source
This commit is contained in:
Zichao Lin
2024-02-25 08:30:34 +08:00
commit 4947f39e74
273 changed files with 45396 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

427
pkg/filesystem/driver/cos/handler.go Normal file
View File

@ -0,0 +1,427 @@
package cos
import (
"context"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"path/filepath"
"strings"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/google/go-querystring/query"
cossdk "github.com/tencentyun/cos-go-sdk-v5"
)
// UploadPolicy 腾讯云COS上传策略
type UploadPolicy struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
// MetaData 文件元信息
type MetaData struct {
Size uint64
CallbackKey string
CallbackURL string
}
type urlOption struct {
Speed int `url:"x-cos-traffic-limit,omitempty"`
ContentDescription string `url:"response-content-disposition,omitempty"`
}
// Driver 腾讯云COS适配器模板
type Driver struct {
Policy *model.Policy
Client *cossdk.Client
HTTPClient request.Client
}
// List 列出COS文件
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化列目录参数
opt := &cossdk.BucketGetOptions{
Prefix: strings.TrimPrefix(base, "/"),
EncodingType: "",
MaxKeys: 1000,
}
// 是否为递归列出
if !recursive {
opt.Delimiter = "/"
}
// 手动补齐结尾的slash
if opt.Prefix != "" {
opt.Prefix += "/"
}
var (
marker string
objects []cossdk.Object
commons []string
)
for {
res, _, err := handler.Client.Bucket.Get(ctx, opt)
if err != nil {
return nil, err
}
objects = append(objects, res.Contents...)
commons = append(commons, res.CommonPrefixes...)
// 如果本次未列取完，则继续使用marker获取结果
marker = res.NextMarker
// marker 为空时结果列取完毕，跳出
if marker == "" {
break
}
}
// 处理列取结果
res := make([]response.Object, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(opt.Prefix, object)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(opt.Prefix, object.Key)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: uint64(object.Size),
IsDir: false,
LastModify: time.Now(),
})
}
return res, nil
}
// CORS 创建跨域策略
func (handler Driver) CORS() error {
_, err := handler.Client.Bucket.PutCORS(context.Background(), &cossdk.BucketPutCORSOptions{
Rules: []cossdk.BucketCORSRule{{
AllowedMethods: []string{
"GET",
"POST",
"PUT",
"DELETE",
"HEAD",
},
AllowedOrigins: []string{"*"},
AllowedHeaders: []string{"*"},
MaxAgeSeconds: 3600,
ExposeHeaders: []string{},
}},
})
return err
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
// 获取文件数据流
resp, err := handler.HTTPClient.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithTimeout(time.Duration(0)),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试自主获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
opt := &cossdk.ObjectPutOptions{}
_, err := handler.Client.Object.Put(ctx, file.Info().SavePath, file, opt)
return err
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
obs := []cossdk.Object{}
for _, v := range files {
obs = append(obs, cossdk.Object{Key: v})
}
opt := &cossdk.ObjectDeleteMultiOptions{
Objects: obs,
Quiet: true,
}
res, _, err := handler.Client.Object.DeleteMulti(context.Background(), opt)
if err != nil {
return files, err
}
// 整理删除结果
failed := make([]string, 0, len(files))
for _, v := range res.Errors {
failed = append(failed, v.Key)
}
if len(failed) == 0 {
return failed, nil
}
return failed, errors.New("delete failed")
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
// https://cloud.tencent.com/document/product/436/44893
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "heif", "heic"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) || file.Size > (32<<(10*2)) {
return nil, driver.ErrorThumbNotSupported
}
var (
thumbSize = [2]uint{400, 300}
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("failed to get thumbnail size")
}
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
thumbParam := fmt.Sprintf("imageMogr2/thumbnail/%dx%d/quality/%d", thumbSize[0], thumbSize[1], thumbEncodeQuality)
source, err := handler.signSourceURL(
ctx,
file.SourceName,
int64(model.GetIntSetting("preview_timeout", 60)),
&urlOption{},
)
if err != nil {
return nil, err
}
thumbURL, _ := url.Parse(source)
thumbQuery := thumbURL.Query()
thumbQuery.Add(thumbParam, "")
thumbURL.RawQuery = thumbQuery.Encode()
return &response.ContentResponse{
Redirect: true,
URL: thumbURL.String(),
}, nil
}
// Source 获取外链URL
func (handler Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := ""
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
fileName = file.Name
}
// 添加各项设置
options := urlOption{}
if speed > 0 {
if speed < 819200 {
speed = 819200
}
if speed > 838860800 {
speed = 838860800
}
options.Speed = speed
}
if isDownload {
options.ContentDescription = "attachment; filename=\"" + url.PathEscape(fileName) + "\""
}
return handler.signSourceURL(ctx, path, ttl, &options)
}
func (handler Driver) signSourceURL(ctx context.Context, path string, ttl int64, options *urlOption) (string, error) {
cdnURL, err := url.Parse(handler.Policy.BaseURL)
if err != nil {
return "", err
}
// 公有空间不需要签名
if !handler.Policy.IsPrivate {
file, err := url.Parse(path)
if err != nil {
return "", err
}
// 非签名URL不支持设置响应header
options.ContentDescription = ""
optionQuery, err := query.Values(*options)
if err != nil {
return "", err
}
file.RawQuery = optionQuery.Encode()
sourceURL := cdnURL.ResolveReference(file)
return sourceURL.String(), nil
}
presignedURL, err := handler.Client.Object.GetPresignedURL(ctx, http.MethodGet, path,
handler.Policy.AccessKey, handler.Policy.SecretKey, time.Duration(ttl)*time.Second, options)
if err != nil {
return "", err
}
// 将最终生成的签名URL域名换成用户自定义的加速域名（如果有）
presignedURL.Host = cdnURL.Host
presignedURL.Scheme = cdnURL.Scheme
return presignedURL.String(), nil
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/cos/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI).String()
// 上传策略
savePath := file.Info().SavePath
startTime := time.Now()
endTime := startTime.Add(time.Duration(ttl) * time.Second)
keyTime := fmt.Sprintf("%d;%d", startTime.Unix(), endTime.Unix())
postPolicy := UploadPolicy{
Expiration: endTime.UTC().Format(time.RFC3339),
Conditions: []interface{}{
map[string]string{"bucket": handler.Policy.BucketName},
map[string]string{"$key": savePath},
map[string]string{"x-cos-meta-callback": apiURL},
map[string]string{"x-cos-meta-key": uploadSession.Key},
map[string]string{"q-sign-algorithm": "sha1"},
map[string]string{"q-ak": handler.Policy.AccessKey},
map[string]string{"q-sign-time": keyTime},
},
}
if handler.Policy.MaxSize > 0 {
postPolicy.Conditions = append(postPolicy.Conditions,
[]interface{}{"content-length-range", 0, handler.Policy.MaxSize})
}
res, err := handler.getUploadCredential(ctx, postPolicy, keyTime, savePath)
if err == nil {
res.SessionID = uploadSession.Key
res.Callback = apiURL
res.UploadURLs = []string{handler.Policy.Server}
}
return res, err
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}
// Meta 获取文件信息
func (handler Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
res, err := handler.Client.Object.Head(ctx, path, &cossdk.ObjectHeadOptions{})
if err != nil {
return nil, err
}
return &MetaData{
Size: uint64(res.ContentLength),
CallbackKey: res.Header.Get("x-cos-meta-key"),
CallbackURL: res.Header.Get("x-cos-meta-callback"),
}, nil
}
func (handler Driver) getUploadCredential(ctx context.Context, policy UploadPolicy, keyTime string, savePath string) (*serializer.UploadCredential, error) {
// 编码上传策略
policyJSON, err := json.Marshal(policy)
if err != nil {
return nil, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
// 签名上传策略
hmacSign := hmac.New(sha1.New, []byte(handler.Policy.SecretKey))
_, err = io.WriteString(hmacSign, keyTime)
if err != nil {
return nil, err
}
signKey := fmt.Sprintf("%x", hmacSign.Sum(nil))
sha1Sign := sha1.New()
_, err = sha1Sign.Write(policyJSON)
if err != nil {
return nil, err
}
stringToSign := fmt.Sprintf("%x", sha1Sign.Sum(nil))
// 最终签名
hmacFinalSign := hmac.New(sha1.New, []byte(signKey))
_, err = hmacFinalSign.Write([]byte(stringToSign))
if err != nil {
return nil, err
}
signature := hmacFinalSign.Sum(nil)
return &serializer.UploadCredential{
Policy: policyEncoded,
Path: savePath,
AccessKey: handler.Policy.AccessKey,
Credential: fmt.Sprintf("%x", signature),
KeyTime: keyTime,
}, nil
}

134
pkg/filesystem/driver/cos/scf.go Normal file
View File

@ -0,0 +1,134 @@
package cos
import (
"archive/zip"
"bytes"
"encoding/base64"
"io"
"io/ioutil"
"net/url"
"strconv"
"strings"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/hashid"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
scf "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/scf/v20180416"
)
const scfFunc = `# -*- coding: utf8 -*-
# SCF配置COS触发，向 Cloudreve 发送回调
from qcloud_cos_v5 import CosConfig
from qcloud_cos_v5 import CosS3Client
from qcloud_cos_v5 import CosServiceError
from qcloud_cos_v5 import CosClientError
import sys
import logging
import requests
logging.basicConfig(level=logging.INFO, stream=sys.stdout)
logger = logging.getLogger()
def main_handler(event, context):
logger.info("start main handler")
for record in event['Records']:
try:
if "x-cos-meta-callback" not in record['cos']['cosObject']['meta']:
logger.info("Cannot find callback URL, skiped.")
return 'Success'
callback = record['cos']['cosObject']['meta']['x-cos-meta-callback']
key = record['cos']['cosObject']['key']
logger.info("Callback URL is " + callback)
r = requests.get(callback)
print(r.text)
except Exception as e:
print(e)
print('Error getting object {} callback url. '.format(key))
raise e
return "Fail"
return "Success"
`
// CreateSCF 创建回调云函数
func CreateSCF(policy *model.Policy, region string) error {
// 初始化客户端
credential := common.NewCredential(
policy.AccessKey,
policy.SecretKey,
)
cpf := profile.NewClientProfile()
client, err := scf.NewClient(credential, region, cpf)
if err != nil {
return err
}
// 创建回调代码数据
buff := &bytes.Buffer{}
bs64 := base64.NewEncoder(base64.StdEncoding, buff)
zipWriter := zip.NewWriter(bs64)
header := zip.FileHeader{
Name: "callback.py",
Method: zip.Deflate,
}
writer, err := zipWriter.CreateHeader(&header)
if err != nil {
return err
}
_, err = io.Copy(writer, strings.NewReader(scfFunc))
zipWriter.Close()
// 创建云函数
req := scf.NewCreateFunctionRequest()
funcName := "cloudreve_" + hashid.HashID(policy.ID, hashid.PolicyID) + strconv.FormatInt(time.Now().Unix(), 10)
zipFileBytes, _ := ioutil.ReadAll(buff)
zipFileStr := string(zipFileBytes)
codeSource := "ZipFile"
handler := "callback.main_handler"
desc := "Cloudreve 用回调函数"
timeout := int64(60)
runtime := "Python3.6"
req.FunctionName = &funcName
req.Code = &scf.Code{
ZipFile: &zipFileStr,
}
req.Handler = &handler
req.Description = &desc
req.Timeout = &timeout
req.Runtime = &runtime
req.CodeSource = &codeSource
_, err = client.CreateFunction(req)
if err != nil {
return err
}
time.Sleep(time.Duration(5) * time.Second)
// 创建触发器
server, _ := url.Parse(policy.Server)
triggerType := "cos"
triggerDesc := `{"event":"cos:ObjectCreated:Post","filter":{"Prefix":"","Suffix":""}}`
enable := "OPEN"
trigger := scf.NewCreateTriggerRequest()
trigger.FunctionName = &funcName
trigger.TriggerName = &server.Host
trigger.Type = &triggerType
trigger.TriggerDesc = &triggerDesc
trigger.Enable = &enable
_, err = client.CreateTrigger(trigger)
if err != nil {
return err
}
return nil
}

73
pkg/filesystem/driver/googledrive/client.go Normal file
View File

@ -0,0 +1,73 @@
package googledrive
import (
"errors"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"google.golang.org/api/drive/v3"
)
// Client Google Drive client
type Client struct {
Endpoints *Endpoints
Policy *model.Policy
Credential *Credential
ClientID string
ClientSecret string
Redirect string
Request request.Client
ClusterController cluster.Controller
}
// Endpoints OneDrive客户端相关设置
type Endpoints struct {
UserConsentEndpoint string // OAuth认证的基URL
TokenEndpoint string // OAuth token 基URL
EndpointURL string // 接口请求的基URL
}
const (
TokenCachePrefix = "googledrive_"
oauthEndpoint = "https://oauth2.googleapis.com/token"
userConsentBase = "https://accounts.google.com/o/oauth2/auth"
v3DriveEndpoint = "https://www.googleapis.com/drive/v3"
)
var (
// Defualt required scopes
RequiredScope = []string{
drive.DriveScope,
"openid",
"profile",
"https://www.googleapis.com/auth/userinfo.profile",
}
// ErrInvalidRefreshToken 上传策略无有效的RefreshToken
ErrInvalidRefreshToken = errors.New("no valid refresh token in this policy")
)
// NewClient 根据存储策略获取新的client
func NewClient(policy *model.Policy) (*Client, error) {
client := &Client{
Endpoints: &Endpoints{
TokenEndpoint: oauthEndpoint,
UserConsentEndpoint: userConsentBase,
EndpointURL: v3DriveEndpoint,
},
Credential: &Credential{
RefreshToken: policy.AccessKey,
},
Policy: policy,
ClientID: policy.BucketName,
ClientSecret: policy.SecretKey,
Redirect: policy.OptionsSerialized.OauthRedirect,
Request: request.NewClient(),
ClusterController: cluster.DefaultController,
}
return client, nil
}

65
pkg/filesystem/driver/googledrive/handler.go Normal file
View File

@ -0,0 +1,65 @@
package googledrive
import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Driver Google Drive 适配器
type Driver struct {
Policy *model.Policy
HTTPClient request.Client
}
// NewDriver 从存储策略初始化新的Driver实例
func NewDriver(policy *model.Policy) (driver.Handler, error) {
return &Driver{
Policy: policy,
HTTPClient: request.NewClient(),
}, nil
}
func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
//TODO implement me
panic("implement me")
}
func (d *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
//TODO implement me
panic("implement me")
}
func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
//TODO implement me
panic("implement me")
}

154
pkg/filesystem/driver/googledrive/oauth.go Normal file
View File

@ -0,0 +1,154 @@
package googledrive
import (
"context"
"encoding/json"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/oauth"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"io"
"net/http"
"net/url"
"strings"
"time"
)
// OAuthURL 获取OAuth认证页面URL
func (client *Client) OAuthURL(ctx context.Context, scope []string) string {
query := url.Values{
"client_id": {client.ClientID},
"scope": {strings.Join(scope, " ")},
"response_type": {"code"},
"redirect_uri": {client.Redirect},
"access_type": {"offline"},
"prompt": {"consent"},
}
u, _ := url.Parse(client.Endpoints.UserConsentEndpoint)
u.RawQuery = query.Encode()
return u.String()
}
// ObtainToken 通过code或refresh_token兑换token
func (client *Client) ObtainToken(ctx context.Context, code, refreshToken string) (*Credential, error) {
body := url.Values{
"client_id": {client.ClientID},
"redirect_uri": {client.Redirect},
"client_secret": {client.ClientSecret},
}
if code != "" {
body.Add("grant_type", "authorization_code")
body.Add("code", code)
} else {
body.Add("grant_type", "refresh_token")
body.Add("refresh_token", refreshToken)
}
strBody := body.Encode()
res := client.Request.Request(
"POST",
client.Endpoints.TokenEndpoint,
io.NopCloser(strings.NewReader(strBody)),
request.WithHeader(http.Header{
"Content-Type": {"application/x-www-form-urlencoded"}},
),
request.WithContentLength(int64(len(strBody))),
)
if res.Err != nil {
return nil, res.Err
}
respBody, err := res.GetResponse()
if err != nil {
return nil, err
}
var (
errResp OAuthError
credential Credential
decodeErr error
)
if res.Response.StatusCode != 200 {
decodeErr = json.Unmarshal([]byte(respBody), &errResp)
} else {
decodeErr = json.Unmarshal([]byte(respBody), &credential)
}
if decodeErr != nil {
return nil, decodeErr
}
if errResp.ErrorType != "" {
return nil, errResp
}
return &credential, nil
}
// UpdateCredential 更新凭证，并检查有效期
func (client *Client) UpdateCredential(ctx context.Context, isSlave bool) error {
if isSlave {
return client.fetchCredentialFromMaster(ctx)
}
oauth.GlobalMutex.Lock(client.Policy.ID)
defer oauth.GlobalMutex.Unlock(client.Policy.ID)
// 如果已存在凭证
if client.Credential != nil && client.Credential.AccessToken != "" {
// 检查已有凭证是否过期
if client.Credential.ExpiresIn > time.Now().Unix() {
// 未过期，不要更新
return nil
}
}
// 尝试从缓存中获取凭证
if cacheCredential, ok := cache.Get(TokenCachePrefix + client.ClientID); ok {
credential := cacheCredential.(Credential)
if credential.ExpiresIn > time.Now().Unix() {
client.Credential = &credential
return nil
}
}
// 获取新的凭证
if client.Credential == nil || client.Credential.RefreshToken == "" {
// 无有效的RefreshToken
util.Log().Error("Failed to refresh credential for policy %q, please login your Google account again.", client.Policy.Name)
return ErrInvalidRefreshToken
}
credential, err := client.ObtainToken(ctx, "", client.Credential.RefreshToken)
if err != nil {
return err
}
// 更新有效期为绝对时间戳
expires := credential.ExpiresIn - 60
credential.ExpiresIn = time.Now().Add(time.Duration(expires) * time.Second).Unix()
// refresh token for Google Drive does not expire in production
credential.RefreshToken = client.Credential.RefreshToken
client.Credential = credential
// 更新缓存
cache.Set(TokenCachePrefix+client.ClientID, *credential, int(expires))
return nil
}
func (client *Client) AccessToken() string {
return client.Credential.AccessToken
}
// UpdateCredential 更新凭证，并检查有效期
func (client *Client) fetchCredentialFromMaster(ctx context.Context) error {
res, err := client.ClusterController.GetPolicyOauthToken(client.Policy.MasterID, client.Policy.ID)
if err != nil {
return err
}
client.Credential = &Credential{AccessToken: res}
return nil
}

43
pkg/filesystem/driver/googledrive/types.go Normal file
View File

@ -0,0 +1,43 @@
package googledrive
import "encoding/gob"
// RespError 接口返回错误
type RespError struct {
APIError APIError `json:"error"`
}
// APIError 接口返回的错误内容
type APIError struct {
Code string `json:"code"`
Message string `json:"message"`
}
// Error 实现error接口
func (err RespError) Error() string {
return err.APIError.Message
}
// Credential 获取token时返回的凭证
type Credential struct {
ExpiresIn int64 `json:"expires_in"`
Scope string `json:"scope"`
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
UserID string `json:"user_id"`
}
// OAuthError OAuth相关接口的错误响应
type OAuthError struct {
ErrorType string `json:"error"`
ErrorDescription string `json:"error_description"`
}
// Error 实现error接口
func (err OAuthError) Error() string {
return err.ErrorDescription
}
func init() {
gob.Register(Credential{})
}

52
pkg/filesystem/driver/handler.go Normal file
View File

@ -0,0 +1,52 @@
package driver
import (
"context"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
var (
ErrorThumbNotExist = fmt.Errorf("thumb not exist")
ErrorThumbNotSupported = fmt.Errorf("thumb not supported")
)
// Handler 存储策略适配器
type Handler interface {
// 上传文件, dst为文件存储路径，size 为文件大小。上下文关闭
// 时，应取消上传并清理临时文件
Put(ctx context.Context, file fsctx.FileHeader) error
// 删除一个或多个给定路径的文件，返回删除失败的文件路径列表及错误
Delete(ctx context.Context, files []string) ([]string, error)
// 获取文件内容
Get(ctx context.Context, path string) (response.RSCloser, error)
// 获取缩略图，可直接在ContentResponse中返回文件数据流，也可指
// 定为重定向
// 如果缩略图不存在, 且需要 Cloudreve 代理生成并上传，应返回 ErrorThumbNotExist，生
// 成的缩略图文件存储规则与本机策略一致。
// 如果不支持此文件的缩略图，并且不希望后续继续请求此缩略图，应返回 ErrorThumbNotSupported
Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error)
// 获取外链/下载地址，
// url - 站点本身地址,
// isDownload - 是否直接下载
Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error)
// Token 获取有效期为ttl的上传凭证和签名
Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error)
// CancelToken 取消已经创建的有状态上传凭证
CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error
// List 递归列取远程端path路径下文件、目录，不包含path本身，
// 返回的对象路径以path作为起始根目录.
// recursive - 是否递归列出
List(ctx context.Context, path string, recursive bool) ([]response.Object, error)
}

292
pkg/filesystem/driver/local/handler.go Normal file
View File

@ -0,0 +1,292 @@
package local
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"os"
"path/filepath"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
const (
Perm = 0744
)
// Driver 本地策略适配器
type Driver struct {
Policy *model.Policy
}
// List 递归列取给定物理路径下所有文件
func (handler Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
var res []response.Object
// 取得起始路径
root := util.RelativePath(filepath.FromSlash(path))
// 开始遍历路径下的文件、目录
err := filepath.Walk(root,
func(path string, info os.FileInfo, err error) error {
// 跳过根目录
if path == root {
return nil
}
if err != nil {
util.Log().Warning("Failed to walk folder %q: %s", path, err)
return filepath.SkipDir
}
// 将遍历对象的绝对路径转换为相对路径
rel, err := filepath.Rel(root, path)
if err != nil {
return err
}
res = append(res, response.Object{
Name: info.Name(),
RelativePath: filepath.ToSlash(rel),
Source: path,
Size: uint64(info.Size()),
IsDir: info.IsDir(),
LastModify: info.ModTime(),
})
// 如果非递归，则不步入目录
if !recursive && info.IsDir() {
return filepath.SkipDir
}
return nil
})
return res, err
}
// Get 获取文件内容
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 打开文件
file, err := os.Open(util.RelativePath(path))
if err != nil {
util.Log().Debug("Failed to open file: %s", err)
return nil, err
}
return file, nil
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
fileInfo := file.Info()
dst := util.RelativePath(filepath.FromSlash(fileInfo.SavePath))
// 如果非 Overwrite，则检查是否有重名冲突
if fileInfo.Mode&fsctx.Overwrite != fsctx.Overwrite {
if util.Exists(dst) {
util.Log().Warning("File with the same name existed or unavailable: %s", dst)
return errors.New("file with the same name existed or unavailable")
}
}
// 如果目标目录不存在，创建
basePath := filepath.Dir(dst)
if !util.Exists(basePath) {
err := os.MkdirAll(basePath, Perm)
if err != nil {
util.Log().Warning("Failed to create directory: %s", err)
return err
}
}
var (
out *os.File
err error
)
openMode := os.O_CREATE | os.O_RDWR
if fileInfo.Mode&fsctx.Append == fsctx.Append {
openMode |= os.O_APPEND
} else {
openMode |= os.O_TRUNC
}
out, err = os.OpenFile(dst, openMode, Perm)
if err != nil {
util.Log().Warning("Failed to open or create file: %s", err)
return err
}
defer out.Close()
if fileInfo.Mode&fsctx.Append == fsctx.Append {
stat, err := out.Stat()
if err != nil {
util.Log().Warning("Failed to read file info: %s", err)
return err
}
if uint64(stat.Size()) < fileInfo.AppendStart {
return errors.New("size of unfinished uploaded chunks is not as expected")
} else if uint64(stat.Size()) > fileInfo.AppendStart {
out.Close()
if err := handler.Truncate(ctx, dst, fileInfo.AppendStart); err != nil {
return fmt.Errorf("failed to overwrite chunk: %w", err)
}
out, err = os.OpenFile(dst, openMode, Perm)
defer out.Close()
if err != nil {
util.Log().Warning("Failed to create or open file: %s", err)
return err
}
}
}
// 写入文件内容
_, err = io.Copy(out, file)
return err
}
func (handler Driver) Truncate(ctx context.Context, src string, size uint64) error {
util.Log().Warning("Truncate file %q to [%d].", src, size)
out, err := os.OpenFile(src, os.O_WRONLY, Perm)
if err != nil {
util.Log().Warning("Failed to open file: %s", err)
return err
}
defer out.Close()
return out.Truncate(int64(size))
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
deleteFailed := make([]string, 0, len(files))
var retErr error
for _, value := range files {
filePath := util.RelativePath(filepath.FromSlash(value))
if util.Exists(filePath) {
err := os.Remove(filePath)
if err != nil {
util.Log().Warning("Failed to delete file: %s", err)
retErr = err
deleteFailed = append(deleteFailed, value)
}
}
// 尝试删除文件的缩略图（如果有）
_ = os.Remove(util.RelativePath(value + model.GetSettingByNameWithDefault("thumb_file_suffix", "._thumb")))
}
return deleteFailed, retErr
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// Quick check thumb existence on master.
if conf.SystemConfig.Mode == "master" && file.MetadataSerialized[model.ThumbStatusMetadataKey] == model.ThumbStatusNotExist {
// Tell invoker to generate a thumb
return nil, driver.ErrorThumbNotExist
}
thumbFile, err := handler.Get(ctx, file.ThumbFile())
if err != nil {
if errors.Is(err, os.ErrNotExist) {
err = fmt.Errorf("thumb not exist: %w (%w)", err, driver.ErrorThumbNotExist)
}
return nil, err
}
return &response.ContentResponse{
Redirect: false,
Content: thumbFile,
}, nil
}
// Source 获取外链URL
func (handler Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
file, ok := ctx.Value(fsctx.FileModelCtx).(model.File)
if !ok {
return "", errors.New("failed to read file model context")
}
var baseURL *url.URL
// 是否启用了CDN
if handler.Policy.BaseURL != "" {
cdnURL, err := url.Parse(handler.Policy.BaseURL)
if err != nil {
return "", err
}
baseURL = cdnURL
}
var (
signedURI *url.URL
err error
)
if isDownload {
// 创建下载会话，将文件信息写入缓存
downloadSessionID := util.RandStringRunes(16)
err = cache.Set("download_"+downloadSessionID, file, int(ttl))
if err != nil {
return "", serializer.NewError(serializer.CodeCacheOperation, "Failed to create download session", err)
}
// 签名生成文件记录
signedURI, err = auth.SignURI(
auth.General,
fmt.Sprintf("/api/v3/file/download/%s", downloadSessionID),
ttl,
)
} else {
// 签名生成文件记录
signedURI, err = auth.SignURI(
auth.General,
fmt.Sprintf("/api/v3/file/get/%d/%s", file.ID, file.Name),
ttl,
)
}
if err != nil {
return "", serializer.NewError(serializer.CodeEncryptError, "Failed to sign url", err)
}
finalURL := signedURI.String()
if baseURL != nil {
finalURL = baseURL.ResolveReference(signedURI).String()
}
return finalURL, nil
}
// Token 获取上传策略和认证Token，本地策略直接返回空值
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
if util.Exists(uploadSession.SavePath) {
return nil, errors.New("placeholder file already exist")
}
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}

595
pkg/filesystem/driver/onedrive/api.go Normal file
View File

@ -0,0 +1,595 @@
package onedrive
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/conf"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
const (
// SmallFileSize 单文件上传接口最大尺寸
SmallFileSize uint64 = 4 * 1024 * 1024
// ChunkSize 服务端中转分片上传分片大小
ChunkSize uint64 = 10 * 1024 * 1024
// ListRetry 列取请求重试次数
ListRetry = 1
chunkRetrySleep = time.Second * 5
notFoundError = "itemNotFound"
)
// GetSourcePath 获取文件的绝对路径
func (info *FileInfo) GetSourcePath() string {
res, err := url.PathUnescape(info.ParentReference.Path)
if err != nil {
return ""
}
return strings.TrimPrefix(
path.Join(
strings.TrimPrefix(res, "/drive/root:"),
info.Name,
),
"/",
)
}
func (client *Client) getRequestURL(api string, opts ...Option) string {
options := newDefaultOption()
for _, o := range opts {
o.apply(options)
}
base, _ := url.Parse(client.Endpoints.EndpointURL)
if base == nil {
return ""
}
if options.useDriverResource {
base.Path = path.Join(base.Path, client.Endpoints.DriverResource, api)
} else {
base.Path = path.Join(base.Path, api)
}
return base.String()
}
// ListChildren 根据路径列取子对象
func (client *Client) ListChildren(ctx context.Context, path string) ([]FileInfo, error) {
var requestURL string
dst := strings.TrimPrefix(path, "/")
if dst == "" {
requestURL = client.getRequestURL("root/children")
} else {
requestURL = client.getRequestURL("root:/" + dst + ":/children")
}
res, err := client.requestWithStr(ctx, "GET", requestURL+"?$top=999999999", "", 200)
if err != nil {
retried := 0
if v, ok := ctx.Value(fsctx.RetryCtx).(int); ok {
retried = v
}
if retried < ListRetry {
retried++
util.Log().Debug("Failed to list path %q: %s, will retry in 5 seconds.", path, err)
time.Sleep(time.Duration(5) * time.Second)
return client.ListChildren(context.WithValue(ctx, fsctx.RetryCtx, retried), path)
}
return nil, err
}
var (
decodeErr error
fileInfo ListResponse
)
decodeErr = json.Unmarshal([]byte(res), &fileInfo)
if decodeErr != nil {
return nil, decodeErr
}
return fileInfo.Value, nil
}
// Meta 根据资源ID或文件路径获取文件元信息
func (client *Client) Meta(ctx context.Context, id string, path string) (*FileInfo, error) {
var requestURL string
if id != "" {
requestURL = client.getRequestURL("items/" + id)
} else {
dst := strings.TrimPrefix(path, "/")
requestURL = client.getRequestURL("root:/" + dst)
}
res, err := client.requestWithStr(ctx, "GET", requestURL+"?expand=thumbnails", "", 200)
if err != nil {
return nil, err
}
var (
decodeErr error
fileInfo FileInfo
)
decodeErr = json.Unmarshal([]byte(res), &fileInfo)
if decodeErr != nil {
return nil, decodeErr
}
return &fileInfo, nil
}
// CreateUploadSession 创建分片上传会话
func (client *Client) CreateUploadSession(ctx context.Context, dst string, opts ...Option) (string, error) {
options := newDefaultOption()
for _, o := range opts {
o.apply(options)
}
dst = strings.TrimPrefix(dst, "/")
requestURL := client.getRequestURL("root:/" + dst + ":/createUploadSession")
body := map[string]map[string]interface{}{
"item": {
"@microsoft.graph.conflictBehavior": options.conflictBehavior,
},
}
bodyBytes, _ := json.Marshal(body)
res, err := client.requestWithStr(ctx, "POST", requestURL, string(bodyBytes), 200)
if err != nil {
return "", err
}
var (
decodeErr error
uploadSession UploadSessionResponse
)
decodeErr = json.Unmarshal([]byte(res), &uploadSession)
if decodeErr != nil {
return "", decodeErr
}
return uploadSession.UploadURL, nil
}
// GetSiteIDByURL 通过 SharePoint 站点 URL 获取站点ID
func (client *Client) GetSiteIDByURL(ctx context.Context, siteUrl string) (string, error) {
siteUrlParsed, err := url.Parse(siteUrl)
if err != nil {
return "", err
}
hostName := siteUrlParsed.Hostname()
relativePath := strings.Trim(siteUrlParsed.Path, "/")
requestURL := client.getRequestURL(fmt.Sprintf("sites/%s:/%s", hostName, relativePath), WithDriverResource(false))
res, reqErr := client.requestWithStr(ctx, "GET", requestURL, "", 200)
if reqErr != nil {
return "", reqErr
}
var (
decodeErr error
siteInfo Site
)
decodeErr = json.Unmarshal([]byte(res), &siteInfo)
if decodeErr != nil {
return "", decodeErr
}
return siteInfo.ID, nil
}
// GetUploadSessionStatus 查询上传会话状态
func (client *Client) GetUploadSessionStatus(ctx context.Context, uploadURL string) (*UploadSessionResponse, error) {
res, err := client.requestWithStr(ctx, "GET", uploadURL, "", 200)
if err != nil {
return nil, err
}
var (
decodeErr error
uploadSession UploadSessionResponse
)
decodeErr = json.Unmarshal([]byte(res), &uploadSession)
if decodeErr != nil {
return nil, decodeErr
}
return &uploadSession, nil
}
// UploadChunk 上传分片
func (client *Client) UploadChunk(ctx context.Context, uploadURL string, content io.Reader, current *chunk.ChunkGroup) (*UploadSessionResponse, error) {
res, err := client.request(
ctx, "PUT", uploadURL, content,
request.WithContentLength(current.Length()),
request.WithHeader(http.Header{
"Content-Range": {current.RangeHeader()},
}),
request.WithoutHeader([]string{"Authorization", "Content-Type"}),
request.WithTimeout(0),
)
if err != nil {
return nil, fmt.Errorf("failed to upload OneDrive chunk #%d: %w", current.Index(), err)
}
if current.IsLast() {
return nil, nil
}
var (
decodeErr error
uploadRes UploadSessionResponse
)
decodeErr = json.Unmarshal([]byte(res), &uploadRes)
if decodeErr != nil {
return nil, decodeErr
}
return &uploadRes, nil
}
// Upload 上传文件
func (client *Client) Upload(ctx context.Context, file fsctx.FileHeader) error {
fileInfo := file.Info()
// 决定是否覆盖文件
overwrite := "fail"
if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
overwrite = "replace"
}
size := int(fileInfo.Size)
dst := fileInfo.SavePath
// 小文件，使用简单上传接口上传
if size <= int(SmallFileSize) {
_, err := client.SimpleUpload(ctx, dst, file, int64(size), WithConflictBehavior(overwrite))
return err
}
// 大文件，进行分片
// 创建上传会话
uploadURL, err := client.CreateUploadSession(ctx, dst, WithConflictBehavior(overwrite))
if err != nil {
return err
}
// Initial chunk groups
chunks := chunk.NewChunkGroup(file, client.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
_, err := client.UploadChunk(ctx, uploadURL, content, current)
return err
}
// upload chunks
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
}
return nil
}
// DeleteUploadSession 删除上传会话
func (client *Client) DeleteUploadSession(ctx context.Context, uploadURL string) error {
_, err := client.requestWithStr(ctx, "DELETE", uploadURL, "", 204)
if err != nil {
return err
}
return nil
}
// SimpleUpload 上传小文件到dst
func (client *Client) SimpleUpload(ctx context.Context, dst string, body io.Reader, size int64, opts ...Option) (*UploadResult, error) {
options := newDefaultOption()
for _, o := range opts {
o.apply(options)
}
dst = strings.TrimPrefix(dst, "/")
requestURL := client.getRequestURL("root:/" + dst + ":/content")
requestURL += ("?@microsoft.graph.conflictBehavior=" + options.conflictBehavior)
res, err := client.request(ctx, "PUT", requestURL, body, request.WithContentLength(int64(size)),
request.WithTimeout(0),
)
if err != nil {
return nil, err
}
var (
decodeErr error
uploadRes UploadResult
)
decodeErr = json.Unmarshal([]byte(res), &uploadRes)
if decodeErr != nil {
return nil, decodeErr
}
return &uploadRes, nil
}
// BatchDelete 并行删除给出的文件，返回删除失败的文件，及第一个遇到的错误。此方法将文件分为
// 20个一组，调用Delete并行删除
// TODO 测试
func (client *Client) BatchDelete(ctx context.Context, dst []string) ([]string, error) {
groupNum := len(dst)/20 + 1
finalRes := make([]string, 0, len(dst))
res := make([]string, 0, 20)
var err error
for i := 0; i < groupNum; i++ {
end := 20*i + 20
if i == groupNum-1 {
end = len(dst)
}
res, err = client.Delete(ctx, dst[20*i:end])
finalRes = append(finalRes, res...)
}
return finalRes, err
}
// Delete 并行删除文件，返回删除失败的文件，及第一个遇到的错误，
// 由于API限制，最多删除20个
func (client *Client) Delete(ctx context.Context, dst []string) ([]string, error) {
body := client.makeBatchDeleteRequestsBody(dst)
res, err := client.requestWithStr(ctx, "POST", client.getRequestURL("$batch",
WithDriverResource(false)), body, 200)
if err != nil {
return dst, err
}
var (
decodeErr error
deleteRes BatchResponses
)
decodeErr = json.Unmarshal([]byte(res), &deleteRes)
if decodeErr != nil {
return dst, decodeErr
}
// 取得删除失败的文件
failed := getDeleteFailed(&deleteRes)
if len(failed) != 0 {
return failed, ErrDeleteFile
}
return failed, nil
}
func getDeleteFailed(res *BatchResponses) []string {
var failed = make([]string, 0, len(res.Responses))
for _, v := range res.Responses {
if v.Status != 204 && v.Status != 404 {
failed = append(failed, v.ID)
}
}
return failed
}
// makeBatchDeleteRequestsBody 生成批量删除请求正文
func (client *Client) makeBatchDeleteRequestsBody(files []string) string {
req := BatchRequests{
Requests: make([]BatchRequest, len(files)),
}
for i, v := range files {
v = strings.TrimPrefix(v, "/")
filePath, _ := url.Parse("/" + client.Endpoints.DriverResource + "/root:/")
filePath.Path = path.Join(filePath.Path, v)
req.Requests[i] = BatchRequest{
ID: v,
Method: "DELETE",
URL: filePath.EscapedPath(),
}
}
res, _ := json.Marshal(req)
return string(res)
}
// GetThumbURL 获取给定尺寸的缩略图URL
func (client *Client) GetThumbURL(ctx context.Context, dst string, w, h uint) (string, error) {
dst = strings.TrimPrefix(dst, "/")
requestURL := client.getRequestURL("root:/"+dst+":/thumbnails/0") + "/large"
res, err := client.requestWithStr(ctx, "GET", requestURL, "", 200)
if err != nil {
return "", err
}
var (
decodeErr error
thumbRes ThumbResponse
)
decodeErr = json.Unmarshal([]byte(res), &thumbRes)
if decodeErr != nil {
return "", decodeErr
}
if thumbRes.URL != "" {
return thumbRes.URL, nil
}
if len(thumbRes.Value) == 1 {
if res, ok := thumbRes.Value[0]["large"]; ok {
return res.(map[string]interface{})["url"].(string), nil
}
}
return "", ErrThumbSizeNotFound
}
// MonitorUpload 监控客户端分片上传进度
func (client *Client) MonitorUpload(uploadURL, callbackKey, path string, size uint64, ttl int64) {
// 回调完成通知chan
callbackChan := mq.GlobalMQ.Subscribe(callbackKey, 1)
defer mq.GlobalMQ.Unsubscribe(callbackKey, callbackChan)
timeout := model.GetIntSetting("onedrive_monitor_timeout", 600)
interval := model.GetIntSetting("onedrive_callback_check", 20)
for {
select {
case <-callbackChan:
util.Log().Debug("Client finished OneDrive callback.")
return
case <-time.After(time.Duration(ttl) * time.Second):
// 上传会话到期，仍未完成上传，创建占位符
client.DeleteUploadSession(context.Background(), uploadURL)
_, err := client.SimpleUpload(context.Background(), path, strings.NewReader(""), 0, WithConflictBehavior("replace"))
if err != nil {
util.Log().Debug("Failed to create placeholder file: %s", err)
}
return
case <-time.After(time.Duration(timeout) * time.Second):
util.Log().Debug("Checking OneDrive upload status.")
status, err := client.GetUploadSessionStatus(context.Background(), uploadURL)
if err != nil {
if resErr, ok := err.(*RespError); ok {
if resErr.APIError.Code == notFoundError {
util.Log().Debug("Upload completed, will check upload callback later.")
select {
case <-time.After(time.Duration(interval) * time.Second):
util.Log().Warning("No callback is made, file will be deleted.")
cache.Deletes([]string{callbackKey}, "callback_")
_, err = client.Delete(context.Background(), []string{path})
if err != nil {
util.Log().Warning("Failed to delete file without callback: %s", err)
}
case <-callbackChan:
util.Log().Debug("Client finished callback.")
}
return
}
}
util.Log().Debug("Failed to get upload session status: %s, continue next iteration.", err.Error())
continue
}
// 成功获取分片上传状态，检查文件大小
if len(status.NextExpectedRanges) == 0 {
continue
}
sizeRange := strings.Split(
status.NextExpectedRanges[len(status.NextExpectedRanges)-1],
"-",
)
if len(sizeRange) != 2 {
continue
}
uploadFullSize, _ := strconv.ParseUint(sizeRange[1], 10, 64)
if (sizeRange[0] == "0" && sizeRange[1] == "") || uploadFullSize+1 != size {
util.Log().Debug("Upload has not started, or uploaded file size not match, canceling upload session...")
// 取消上传会话，实测OneDrive取消上传会话后，客户端还是可以上传，
// 所以上传一个空文件占位，阻止客户端上传
client.DeleteUploadSession(context.Background(), uploadURL)
_, err := client.SimpleUpload(context.Background(), path, strings.NewReader(""), 0, WithConflictBehavior("replace"))
if err != nil {
util.Log().Debug("无法创建占位文件，%s", err)
}
return
}
}
}
}
func sysError(err error) *RespError {
return &RespError{APIError: APIError{
Code: "system",
Message: err.Error(),
}}
}
func (client *Client) request(ctx context.Context, method string, url string, body io.Reader, option ...request.Option) (string, error) {
// 获取凭证
err := client.UpdateCredential(ctx, conf.SystemConfig.Mode == "slave")
if err != nil {
return "", sysError(err)
}
option = append(option,
request.WithHeader(http.Header{
"Authorization": {"Bearer " + client.Credential.AccessToken},
"Content-Type": {"application/json"},
}),
request.WithContext(ctx),
request.WithTPSLimit(
fmt.Sprintf("policy_%d", client.Policy.ID),
client.Policy.OptionsSerialized.TPSLimit,
client.Policy.OptionsSerialized.TPSLimitBurst,
),
)
// 发送请求
res := client.Request.Request(
method,
url,
body,
option...,
)
if res.Err != nil {
return "", sysError(res.Err)
}
respBody, err := res.GetResponse()
if err != nil {
return "", sysError(err)
}
// 解析请求响应
var (
errResp RespError
decodeErr error
)
// 如果有错误
if res.Response.StatusCode < 200 || res.Response.StatusCode >= 300 {
decodeErr = json.Unmarshal([]byte(respBody), &errResp)
if decodeErr != nil {
util.Log().Debug("Onedrive returns unknown response: %s", respBody)
return "", sysError(decodeErr)
}
if res.Response.StatusCode == 429 {
util.Log().Warning("OneDrive request is throttled.")
return "", backoff.NewRetryableErrorFromHeader(&errResp, res.Response.Header)
}
return "", &errResp
}
return respBody, nil
}
func (client *Client) requestWithStr(ctx context.Context, method string, url string, body string, expectedCode int) (string, error) {
// 发送请求
bodyReader := io.NopCloser(strings.NewReader(body))
return client.request(ctx, method, url, bodyReader,
request.WithContentLength(int64(len(body))),
)
}

78
pkg/filesystem/driver/onedrive/client.go Normal file
View File

@ -0,0 +1,78 @@
package onedrive
import (
"errors"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
)
var (
// ErrAuthEndpoint 无法解析授权端点地址
ErrAuthEndpoint = errors.New("failed to parse endpoint url")
// ErrInvalidRefreshToken 上传策略无有效的RefreshToken
ErrInvalidRefreshToken = errors.New("no valid refresh token in this policy")
// ErrDeleteFile 无法删除文件
ErrDeleteFile = errors.New("cannot delete file")
// ErrClientCanceled 客户端取消操作
ErrClientCanceled = errors.New("client canceled")
// Desired thumb size not available
ErrThumbSizeNotFound = errors.New("thumb size not found")
)
// Client OneDrive客户端
type Client struct {
Endpoints *Endpoints
Policy *model.Policy
Credential *Credential
ClientID string
ClientSecret string
Redirect string
Request request.Client
ClusterController cluster.Controller
}
// Endpoints OneDrive客户端相关设置
type Endpoints struct {
OAuthURL string // OAuth认证的基URL
OAuthEndpoints *oauthEndpoint
EndpointURL string // 接口请求的基URL
isInChina bool // 是否为世纪互联
DriverResource string // 要使用的驱动器
}
// NewClient 根据存储策略获取新的client
func NewClient(policy *model.Policy) (*Client, error) {
client := &Client{
Endpoints: &Endpoints{
OAuthURL: policy.BaseURL,
EndpointURL: policy.Server,
DriverResource: policy.OptionsSerialized.OdDriver,
},
Credential: &Credential{
RefreshToken: policy.AccessKey,
},
Policy: policy,
ClientID: policy.BucketName,
ClientSecret: policy.SecretKey,
Redirect: policy.OptionsSerialized.OauthRedirect,
Request: request.NewClient(),
ClusterController: cluster.DefaultController,
}
if client.Endpoints.DriverResource == "" {
client.Endpoints.DriverResource = "me/drive"
}
oauthBase := client.getOAuthEndpoint()
if oauthBase == nil {
return nil, ErrAuthEndpoint
}
client.Endpoints.OAuthEndpoints = oauthBase
return client, nil
}

238
pkg/filesystem/driver/onedrive/handler.go Normal file
View File

@ -0,0 +1,238 @@
package onedrive
import (
"context"
"errors"
"fmt"
"net/url"
"path"
"path/filepath"
"strings"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Driver OneDrive 适配器
type Driver struct {
Policy *model.Policy
Client *Client
HTTPClient request.Client
}
// NewDriver 从存储策略初始化新的Driver实例
func NewDriver(policy *model.Policy) (driver.Handler, error) {
client, err := NewClient(policy)
if policy.OptionsSerialized.ChunkSize == 0 {
policy.OptionsSerialized.ChunkSize = 50 << 20 // 50MB
}
return Driver{
Policy: policy,
Client: client,
HTTPClient: request.NewClient(),
}, err
}
// List 列取项目
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
// 列取子项目
objects, _ := handler.Client.ListChildren(ctx, base)
// 获取真实的列取起始根目录
rootPath := base
if realBase, ok := ctx.Value(fsctx.PathCtx).(string); ok {
rootPath = realBase
} else {
ctx = context.WithValue(ctx, fsctx.PathCtx, base)
}
// 整理结果
res := make([]response.Object, 0, len(objects))
for _, object := range objects {
source := path.Join(base, object.Name)
rel, err := filepath.Rel(rootPath, source)
if err != nil {
continue
}
res = append(res, response.Object{
Name: object.Name,
RelativePath: filepath.ToSlash(rel),
Source: source,
Size: object.Size,
IsDir: object.Folder != nil,
LastModify: time.Now(),
})
}
// 递归列取子目录
if recursive {
for _, object := range objects {
if object.Folder != nil {
sub, _ := handler.List(ctx, path.Join(base, object.Name), recursive)
res = append(res, sub...)
}
}
}
return res, nil
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(
ctx,
path,
60,
false,
0,
)
if err != nil {
return nil, err
}
// 获取文件数据流
resp, err := handler.HTTPClient.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithTimeout(time.Duration(0)),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试自主获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
return handler.Client.Upload(ctx, file)
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
return handler.Client.BatchDelete(ctx, files)
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
var (
thumbSize = [2]uint{400, 300}
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("failed to get thumbnail size")
}
res, err := handler.Client.GetThumbURL(ctx, file.SourceName, thumbSize[0], thumbSize[1])
if err != nil {
var apiErr *RespError
if errors.As(err, &apiErr); err == ErrThumbSizeNotFound || (apiErr != nil && apiErr.APIError.Code == notFoundError) {
// OneDrive cannot generate thumbnail for this file
return nil, driver.ErrorThumbNotSupported
}
}
return &response.ContentResponse{
Redirect: true,
URL: res,
}, err
}
// Source 获取外链URL
func (handler Driver) Source(
ctx context.Context,
path string,
ttl int64,
isDownload bool,
speed int,
) (string, error) {
cacheKey := fmt.Sprintf("onedrive_source_%d_%s", handler.Policy.ID, path)
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
cacheKey = fmt.Sprintf("onedrive_source_file_%d_%d", file.UpdatedAt.Unix(), file.ID)
}
// 尝试从缓存中查找
if cachedURL, ok := cache.Get(cacheKey); ok {
return handler.replaceSourceHost(cachedURL.(string))
}
// 缓存不存在，重新获取
res, err := handler.Client.Meta(ctx, "", path)
if err == nil {
// 写入新的缓存
cache.Set(
cacheKey,
res.DownloadURL,
model.GetIntSetting("onedrive_source_timeout", 1800),
)
return handler.replaceSourceHost(res.DownloadURL)
}
return "", err
}
func (handler Driver) replaceSourceHost(origin string) (string, error) {
if handler.Policy.OptionsSerialized.OdProxy != "" {
source, err := url.Parse(origin)
if err != nil {
return "", err
}
cdn, err := url.Parse(handler.Policy.OptionsSerialized.OdProxy)
if err != nil {
return "", err
}
// 替换反代地址
source.Scheme = cdn.Scheme
source.Host = cdn.Host
return source.String(), nil
}
return origin, nil
}
// Token 获取上传会话URL
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
fileInfo := file.Info()
uploadURL, err := handler.Client.CreateUploadSession(ctx, fileInfo.SavePath, WithConflictBehavior("fail"))
if err != nil {
return nil, err
}
// 监控回调及上传
go handler.Client.MonitorUpload(uploadURL, uploadSession.Key, fileInfo.SavePath, fileInfo.Size, ttl)
uploadSession.UploadURL = uploadURL
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadURLs: []string{uploadURL},
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return handler.Client.DeleteUploadSession(ctx, uploadSession.UploadURL)
}

25
pkg/filesystem/driver/onedrive/lock.go Normal file
View File

@ -0,0 +1,25 @@
package onedrive
import "sync"
// CredentialLock 针对存储策略凭证的锁
type CredentialLock interface {
Lock(uint)
Unlock(uint)
}
var GlobalMutex = mutexMap{}
type mutexMap struct {
locks sync.Map
}
func (m *mutexMap) Lock(id uint) {
lock, _ := m.locks.LoadOrStore(id, &sync.Mutex{})
lock.(*sync.Mutex).Lock()
}
func (m *mutexMap) Unlock(id uint) {
lock, _ := m.locks.LoadOrStore(id, &sync.Mutex{})
lock.(*sync.Mutex).Unlock()
}

192
pkg/filesystem/driver/onedrive/oauth.go Normal file
View File

@ -0,0 +1,192 @@
package onedrive
import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"strings"
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/oauth"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
// Error 实现error接口
func (err OAuthError) Error() string {
return err.ErrorDescription
}
// OAuthURL 获取OAuth认证页面URL
func (client *Client) OAuthURL(ctx context.Context, scope []string) string {
query := url.Values{
"client_id": {client.ClientID},
"scope": {strings.Join(scope, " ")},
"response_type": {"code"},
"redirect_uri": {client.Redirect},
}
client.Endpoints.OAuthEndpoints.authorize.RawQuery = query.Encode()
return client.Endpoints.OAuthEndpoints.authorize.String()
}
// getOAuthEndpoint 根据指定的AuthURL获取详细的认证接口地址
func (client *Client) getOAuthEndpoint() *oauthEndpoint {
base, err := url.Parse(client.Endpoints.OAuthURL)
if err != nil {
return nil
}
var (
token *url.URL
authorize *url.URL
)
switch base.Host {
case "login.live.com":
token, _ = url.Parse("https://login.live.com/oauth20_token.srf")
authorize, _ = url.Parse("https://login.live.com/oauth20_authorize.srf")
case "login.chinacloudapi.cn":
client.Endpoints.isInChina = true
token, _ = url.Parse("https://login.chinacloudapi.cn/common/oauth2/v2.0/token")
authorize, _ = url.Parse("https://login.chinacloudapi.cn/common/oauth2/v2.0/authorize")
default:
token, _ = url.Parse("https://login.microsoftonline.com/common/oauth2/v2.0/token")
authorize, _ = url.Parse("https://login.microsoftonline.com/common/oauth2/v2.0/authorize")
}
return &oauthEndpoint{
token: *token,
authorize: *authorize,
}
}
// ObtainToken 通过code或refresh_token兑换token
func (client *Client) ObtainToken(ctx context.Context, opts ...Option) (*Credential, error) {
options := newDefaultOption()
for _, o := range opts {
o.apply(options)
}
body := url.Values{
"client_id": {client.ClientID},
"redirect_uri": {client.Redirect},
"client_secret": {client.ClientSecret},
}
if options.code != "" {
body.Add("grant_type", "authorization_code")
body.Add("code", options.code)
} else {
body.Add("grant_type", "refresh_token")
body.Add("refresh_token", options.refreshToken)
}
strBody := body.Encode()
res := client.Request.Request(
"POST",
client.Endpoints.OAuthEndpoints.token.String(),
ioutil.NopCloser(strings.NewReader(strBody)),
request.WithHeader(http.Header{
"Content-Type": {"application/x-www-form-urlencoded"}},
),
request.WithContentLength(int64(len(strBody))),
)
if res.Err != nil {
return nil, res.Err
}
respBody, err := res.GetResponse()
if err != nil {
return nil, err
}
var (
errResp OAuthError
credential Credential
decodeErr error
)
if res.Response.StatusCode != 200 {
decodeErr = json.Unmarshal([]byte(respBody), &errResp)
} else {
decodeErr = json.Unmarshal([]byte(respBody), &credential)
}
if decodeErr != nil {
return nil, decodeErr
}
if errResp.ErrorType != "" {
return nil, errResp
}
return &credential, nil
}
// UpdateCredential 更新凭证，并检查有效期
func (client *Client) UpdateCredential(ctx context.Context, isSlave bool) error {
if isSlave {
return client.fetchCredentialFromMaster(ctx)
}
oauth.GlobalMutex.Lock(client.Policy.ID)
defer oauth.GlobalMutex.Unlock(client.Policy.ID)
// 如果已存在凭证
if client.Credential != nil && client.Credential.AccessToken != "" {
// 检查已有凭证是否过期
if client.Credential.ExpiresIn > time.Now().Unix() {
// 未过期，不要更新
return nil
}
}
// 尝试从缓存中获取凭证
if cacheCredential, ok := cache.Get("onedrive_" + client.ClientID); ok {
credential := cacheCredential.(Credential)
if credential.ExpiresIn > time.Now().Unix() {
client.Credential = &credential
return nil
}
}
// 获取新的凭证
if client.Credential == nil || client.Credential.RefreshToken == "" {
// 无有效的RefreshToken
util.Log().Error("Failed to refresh credential for policy %q, please login your Microsoft account again.", client.Policy.Name)
return ErrInvalidRefreshToken
}
credential, err := client.ObtainToken(ctx, WithRefreshToken(client.Credential.RefreshToken))
if err != nil {
return err
}
// 更新有效期为绝对时间戳
expires := credential.ExpiresIn - 60
credential.ExpiresIn = time.Now().Add(time.Duration(expires) * time.Second).Unix()
client.Credential = credential
// 更新存储策略的 RefreshToken
client.Policy.UpdateAccessKeyAndClearCache(credential.RefreshToken)
// 更新缓存
cache.Set("onedrive_"+client.ClientID, *credential, int(expires))
return nil
}
func (client *Client) AccessToken() string {
return client.Credential.AccessToken
}
// UpdateCredential 更新凭证，并检查有效期
func (client *Client) fetchCredentialFromMaster(ctx context.Context) error {
res, err := client.ClusterController.GetPolicyOauthToken(client.Policy.MasterID, client.Policy.ID)
if err != nil {
return err
}
client.Credential = &Credential{AccessToken: res}
return nil
}

59
pkg/filesystem/driver/onedrive/options.go Normal file
View File

@ -0,0 +1,59 @@
package onedrive
import "time"
// Option 发送请求的额外设置
type Option interface {
apply(*options)
}
type options struct {
redirect string
code string
refreshToken string
conflictBehavior string
expires time.Time
useDriverResource bool
}
type optionFunc func(*options)
// WithCode 设置接口Code
func WithCode(t string) Option {
return optionFunc(func(o *options) {
o.code = t
})
}
// WithRefreshToken 设置接口RefreshToken
func WithRefreshToken(t string) Option {
return optionFunc(func(o *options) {
o.refreshToken = t
})
}
// WithConflictBehavior 设置文件重名后的处理方式
func WithConflictBehavior(t string) Option {
return optionFunc(func(o *options) {
o.conflictBehavior = t
})
}
// WithConflictBehavior 设置文件重名后的处理方式
func WithDriverResource(t bool) Option {
return optionFunc(func(o *options) {
o.useDriverResource = t
})
}
func (f optionFunc) apply(o *options) {
f(o)
}
func newDefaultOption() *options {
return &options{
conflictBehavior: "fail",
useDriverResource: true,
expires: time.Now().UTC().Add(time.Duration(1) * time.Hour),
}
}

140
pkg/filesystem/driver/onedrive/types.go Normal file
View File

@ -0,0 +1,140 @@
package onedrive
import (
"encoding/gob"
"net/url"
)
// RespError 接口返回错误
type RespError struct {
APIError APIError `json:"error"`
}
// APIError 接口返回的错误内容
type APIError struct {
Code string `json:"code"`
Message string `json:"message"`
}
// UploadSessionResponse 分片上传会话
type UploadSessionResponse struct {
DataContext string `json:"@odata.context"`
ExpirationDateTime string `json:"expirationDateTime"`
NextExpectedRanges []string `json:"nextExpectedRanges"`
UploadURL string `json:"uploadUrl"`
}
// FileInfo 文件元信息
type FileInfo struct {
Name string `json:"name"`
Size uint64 `json:"size"`
Image imageInfo `json:"image"`
ParentReference parentReference `json:"parentReference"`
DownloadURL string `json:"@microsoft.graph.downloadUrl"`
File *file `json:"file"`
Folder *folder `json:"folder"`
}
type file struct {
MimeType string `json:"mimeType"`
}
type folder struct {
ChildCount int `json:"childCount"`
}
type imageInfo struct {
Height int `json:"height"`
Width int `json:"width"`
}
type parentReference struct {
Path string `json:"path"`
Name string `json:"name"`
ID string `json:"id"`
}
// UploadResult 上传结果
type UploadResult struct {
ID string `json:"id"`
Name string `json:"name"`
Size uint64 `json:"size"`
}
// BatchRequests 批量操作请求
type BatchRequests struct {
Requests []BatchRequest `json:"requests"`
}
// BatchRequest 批量操作单个请求
type BatchRequest struct {
ID string `json:"id"`
Method string `json:"method"`
URL string `json:"url"`
Body interface{} `json:"body,omitempty"`
Headers map[string]string `json:"headers,omitempty"`
}
// BatchResponses 批量操作响应
type BatchResponses struct {
Responses []BatchResponse `json:"responses"`
}
// BatchResponse 批量操作单个响应
type BatchResponse struct {
ID string `json:"id"`
Status int `json:"status"`
}
// ThumbResponse 获取缩略图的响应
type ThumbResponse struct {
Value []map[string]interface{} `json:"value"`
URL string `json:"url"`
}
// ListResponse 列取子项目响应
type ListResponse struct {
Value []FileInfo `json:"value"`
Context string `json:"@odata.context"`
}
// oauthEndpoint OAuth接口地址
type oauthEndpoint struct {
token url.URL
authorize url.URL
}
// Credential 获取token时返回的凭证
type Credential struct {
TokenType string `json:"token_type"`
ExpiresIn int64 `json:"expires_in"`
Scope string `json:"scope"`
AccessToken string `json:"access_token"`
RefreshToken string `json:"refresh_token"`
UserID string `json:"user_id"`
}
// OAuthError OAuth相关接口的错误响应
type OAuthError struct {
ErrorType string `json:"error"`
ErrorDescription string `json:"error_description"`
CorrelationID string `json:"correlation_id"`
}
// Site SharePoint 站点信息
type Site struct {
Description string `json:"description"`
ID string `json:"id"`
Name string `json:"name"`
DisplayName string `json:"displayName"`
WebUrl string `json:"webUrl"`
}
func init() {
gob.Register(Credential{})
}
// Error 实现error接口
func (err RespError) Error() string {
return err.APIError.Message
}

117
pkg/filesystem/driver/oss/callback.go Normal file
View File

@ -0,0 +1,117 @@
package oss
import (
"bytes"
"crypto"
"crypto/md5"
"crypto/rsa"
"crypto/x509"
"encoding/base64"
"encoding/pem"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"github.com/cloudreve/Cloudreve/v3/pkg/cache"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
)
// GetPublicKey 从回调请求或缓存中获取OSS的回调签名公钥
func GetPublicKey(r *http.Request) ([]byte, error) {
var pubKey []byte
// 尝试从缓存中获取
pub, exist := cache.Get("oss_public_key")
if exist {
return pub.([]byte), nil
}
// 从请求中获取
pubURL, err := base64.StdEncoding.DecodeString(r.Header.Get("x-oss-pub-key-url"))
if err != nil {
return pubKey, err
}
// 确保这个 public key 是由 OSS 颁发的
if !strings.HasPrefix(string(pubURL), "http://gosspublic.alicdn.com/") &&
!strings.HasPrefix(string(pubURL), "https://gosspublic.alicdn.com/") {
return pubKey, errors.New("public key url invalid")
}
// 获取公钥
client := request.NewClient()
body, err := client.Request("GET", string(pubURL), nil).
CheckHTTPResponse(200).
GetResponse()
if err != nil {
return pubKey, err
}
// 写入缓存
_ = cache.Set("oss_public_key", []byte(body), 86400*7)
return []byte(body), nil
}
func getRequestMD5(r *http.Request) ([]byte, error) {
var byteMD5 []byte
// 获取请求正文
body, err := ioutil.ReadAll(r.Body)
r.Body.Close()
if err != nil {
return byteMD5, err
}
r.Body = ioutil.NopCloser(bytes.NewReader(body))
strURLPathDecode, err := url.PathUnescape(r.URL.Path)
if err != nil {
return byteMD5, err
}
strAuth := fmt.Sprintf("%s\n%s", strURLPathDecode, string(body))
md5Ctx := md5.New()
md5Ctx.Write([]byte(strAuth))
byteMD5 = md5Ctx.Sum(nil)
return byteMD5, nil
}
// VerifyCallbackSignature 验证OSS回调请求
func VerifyCallbackSignature(r *http.Request) error {
bytePublicKey, err := GetPublicKey(r)
if err != nil {
return err
}
byteMD5, err := getRequestMD5(r)
if err != nil {
return err
}
strAuthorizationBase64 := r.Header.Get("authorization")
if strAuthorizationBase64 == "" {
return errors.New("no authorization field in Request header")
}
authorization, _ := base64.StdEncoding.DecodeString(strAuthorizationBase64)
pubBlock, _ := pem.Decode(bytePublicKey)
if pubBlock == nil {
return errors.New("pubBlock not exist")
}
pubInterface, err := x509.ParsePKIXPublicKey(pubBlock.Bytes)
if (pubInterface == nil) || (err != nil) {
return err
}
pub := pubInterface.(*rsa.PublicKey)
errorVerifyPKCS1v15 := rsa.VerifyPKCS1v15(pub, crypto.MD5, byteMD5, authorization)
if errorVerifyPKCS1v15 != nil {
return errorVerifyPKCS1v15
}
return nil
}

501
pkg/filesystem/driver/oss/handler.go Normal file
View File

@ -0,0 +1,501 @@
package oss
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
"path"
"path/filepath"
"strings"
"time"
"github.com/HFO4/aliyun-oss-go-sdk/oss"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
// UploadPolicy 阿里云OSS上传策略
type UploadPolicy struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
// CallbackPolicy 回调策略
type CallbackPolicy struct {
CallbackURL string `json:"callbackUrl"`
CallbackBody string `json:"callbackBody"`
CallbackBodyType string `json:"callbackBodyType"`
}
// Driver 阿里云OSS策略适配器
type Driver struct {
Policy *model.Policy
client *oss.Client
bucket *oss.Bucket
HTTPClient request.Client
}
type key int
const (
chunkRetrySleep = time.Duration(5) * time.Second
// MultiPartUploadThreshold 服务端使用分片上传的阈值
MultiPartUploadThreshold uint64 = 5 * (1 << 30) // 5GB
// VersionID 文件版本标识
VersionID key = iota
)
func NewDriver(policy *model.Policy) (*Driver, error) {
if policy.OptionsSerialized.ChunkSize == 0 {
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
}
driver := &Driver{
Policy: policy,
HTTPClient: request.NewClient(),
}
return driver, driver.InitOSSClient(false)
}
// CORS 创建跨域策略
func (handler *Driver) CORS() error {
return handler.client.SetBucketCORS(handler.Policy.BucketName, []oss.CORSRule{
{
AllowedOrigin: []string{"*"},
AllowedMethod: []string{
"GET",
"POST",
"PUT",
"DELETE",
"HEAD",
},
ExposeHeader: []string{},
AllowedHeader: []string{"*"},
MaxAgeSeconds: 3600,
},
})
}
// InitOSSClient 初始化OSS鉴权客户端
func (handler *Driver) InitOSSClient(forceUsePublicEndpoint bool) error {
if handler.Policy == nil {
return errors.New("empty policy")
}
// 决定是否使用内网 Endpoint
endpoint := handler.Policy.Server
if handler.Policy.OptionsSerialized.ServerSideEndpoint != "" && !forceUsePublicEndpoint {
endpoint = handler.Policy.OptionsSerialized.ServerSideEndpoint
}
// 初始化客户端
client, err := oss.New(endpoint, handler.Policy.AccessKey, handler.Policy.SecretKey)
if err != nil {
return err
}
handler.client = client
// 初始化存储桶
bucket, err := client.Bucket(handler.Policy.BucketName)
if err != nil {
return err
}
handler.bucket = bucket
return nil
}
// List 列出OSS上的文件
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 列取文件
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
}
var (
delimiter string
marker string
objects []oss.ObjectProperties
commons []string
)
if !recursive {
delimiter = "/"
}
for {
subRes, err := handler.bucket.ListObjects(oss.Marker(marker), oss.Prefix(base),
oss.MaxKeys(1000), oss.Delimiter(delimiter))
if err != nil {
return nil, err
}
objects = append(objects, subRes.Objects...)
commons = append(commons, subRes.CommonPrefixes...)
marker = subRes.NextMarker
if marker == "" {
break
}
}
// 处理列取结果
res := make([]response.Object, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(base, object)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(base, object.Key)
if err != nil {
continue
}
if strings.HasSuffix(object.Key, "/") {
res = append(res, response.Object{
Name: path.Base(object.Key),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
} else {
res = append(res, response.Object{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: uint64(object.Size),
IsDir: false,
LastModify: object.LastModified,
})
}
}
return res, nil
}
// Get 获取文件
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 通过VersionID禁止缓存
ctx = context.WithValue(ctx, VersionID, time.Now().UnixNano())
// 尽可能使用私有 Endpoint
ctx = context.WithValue(ctx, fsctx.ForceUsePublicEndpointCtx, false)
// 获取文件源地址
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
// 获取文件数据流
resp, err := handler.HTTPClient.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithTimeout(time.Duration(0)),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试自主获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
fileInfo := file.Info()
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
// 是否允许覆盖
overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite
options := []oss.Option{
oss.Expires(time.Now().Add(time.Duration(credentialTTL) * time.Second)),
oss.ForbidOverWrite(!overwrite),
}
// 小文件直接上传
if fileInfo.Size < MultiPartUploadThreshold {
return handler.bucket.PutObject(fileInfo.SavePath, file, options...)
}
// 超过阈值时使用分片上传
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
if err != nil {
return fmt.Errorf("failed to initiate multipart upload: %w", err)
}
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
_, err := handler.bucket.UploadPart(imur, content, current.Length(), current.Index()+1)
return err
}
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
}
_, err = handler.bucket.CompleteMultipartUpload(imur, oss.CompleteAll("yes"), oss.ForbidOverWrite(!overwrite))
return err
}
// Delete 删除一个或多个文件，
// 返回未删除的文件
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 删除文件
delRes, err := handler.bucket.DeleteObjects(files)
if err != nil {
return files, err
}
// 统计未删除的文件
failed := util.SliceDifference(files, delRes.DeletedObjects)
if len(failed) > 0 {
return failed, errors.New("failed to delete")
}
return []string{}, nil
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
// https://help.aliyun.com/document_detail/183902.html
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "heic", "tiff", "avif"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) || file.Size > (20<<(10*2)) {
return nil, driver.ErrorThumbNotSupported
}
// 初始化客户端
if err := handler.InitOSSClient(true); err != nil {
return nil, err
}
var (
thumbSize = [2]uint{400, 300}
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("failed to get thumbnail size")
}
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
thumbParam := fmt.Sprintf("image/resize,m_lfit,h_%d,w_%d/quality,q_%d", thumbSize[1], thumbSize[0], thumbEncodeQuality)
ctx = context.WithValue(ctx, fsctx.ThumbSizeCtx, thumbParam)
thumbOption := []oss.Option{oss.Process(thumbParam)}
thumbURL, err := handler.signSourceURL(
ctx,
file.SourceName,
int64(model.GetIntSetting("preview_timeout", 60)),
thumbOption,
)
if err != nil {
return nil, err
}
return &response.ContentResponse{
Redirect: true,
URL: thumbURL,
}, nil
}
// Source 获取外链URL
func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 初始化客户端
usePublicEndpoint := true
if forceUsePublicEndpoint, ok := ctx.Value(fsctx.ForceUsePublicEndpointCtx).(bool); ok {
usePublicEndpoint = forceUsePublicEndpoint
}
if err := handler.InitOSSClient(usePublicEndpoint); err != nil {
return "", err
}
// 尝试从上下文获取文件名
fileName := ""
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
fileName = file.Name
}
// 添加各项设置
var signOptions = make([]oss.Option, 0, 2)
if isDownload {
signOptions = append(signOptions, oss.ResponseContentDisposition("attachment; filename=\""+url.PathEscape(fileName)+"\""))
}
if speed > 0 {
// Byte 转换为 bit
speed *= 8
// OSS对速度值有范围限制
if speed < 819200 {
speed = 819200
}
if speed > 838860800 {
speed = 838860800
}
signOptions = append(signOptions, oss.TrafficLimitParam(int64(speed)))
}
return handler.signSourceURL(ctx, path, ttl, signOptions)
}
func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64, options []oss.Option) (string, error) {
signedURL, err := handler.bucket.SignURL(path, oss.HTTPGet, ttl, options...)
if err != nil {
return "", err
}
// 将最终生成的签名URL域名换成用户自定义的加速域名（如果有）
finalURL, err := url.Parse(signedURL)
if err != nil {
return "", err
}
// 公有空间替换掉Key及不支持的头
if !handler.Policy.IsPrivate {
query := finalURL.Query()
query.Del("OSSAccessKeyId")
query.Del("Signature")
query.Del("response-content-disposition")
query.Del("x-oss-traffic-limit")
finalURL.RawQuery = query.Encode()
}
if handler.Policy.BaseURL != "" {
cdnURL, err := url.Parse(handler.Policy.BaseURL)
if err != nil {
return "", err
}
finalURL.Host = cdnURL.Host
finalURL.Scheme = cdnURL.Scheme
}
return finalURL.String(), nil
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 初始化客户端
if err := handler.InitOSSClient(true); err != nil {
return nil, err
}
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/oss/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 回调策略
callbackPolicy := CallbackPolicy{
CallbackURL: apiURL.String(),
CallbackBody: `{"name":${x:fname},"source_name":${object},"size":${size},"pic_info":"${imageInfo.width},${imageInfo.height}"}`,
CallbackBodyType: "application/json",
}
callbackPolicyJSON, err := json.Marshal(callbackPolicy)
if err != nil {
return nil, fmt.Errorf("failed to encode callback policy: %w", err)
}
callbackPolicyEncoded := base64.StdEncoding.EncodeToString(callbackPolicyJSON)
// 初始化分片上传
fileInfo := file.Info()
options := []oss.Option{
oss.Expires(time.Now().Add(time.Duration(ttl) * time.Second)),
oss.ForbidOverWrite(true),
oss.ContentType(fileInfo.DetectMimeType()),
}
imur, err := handler.bucket.InitiateMultipartUpload(fileInfo.SavePath, options...)
if err != nil {
return nil, fmt.Errorf("failed to initialize multipart upload: %w", err)
}
uploadSession.UploadID = imur.UploadID
// 为每个分片签名上传 URL
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
urls := make([]string, chunks.Num())
for chunks.Next() {
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
signedURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPut, ttl,
oss.PartNumber(c.Index()+1),
oss.UploadID(imur.UploadID),
oss.ContentType("application/octet-stream"))
if err != nil {
return err
}
urls[c.Index()] = signedURL
return nil
})
if err != nil {
return nil, err
}
}
// 签名完成分片上传的URL
completeURL, err := handler.bucket.SignURL(fileInfo.SavePath, oss.HTTPPost, ttl,
oss.ContentType("application/octet-stream"),
oss.UploadID(imur.UploadID),
oss.Expires(time.Now().Add(time.Duration(ttl)*time.Second)),
oss.CompleteAll("yes"),
oss.ForbidOverWrite(true),
oss.CallbackParam(callbackPolicyEncoded))
if err != nil {
return nil, err
}
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadID: imur.UploadID,
UploadURLs: urls,
CompleteURL: completeURL,
}, nil
}
// 取消上传凭证
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return handler.bucket.AbortMultipartUpload(oss.InitiateMultipartUploadResult{UploadID: uploadSession.UploadID, Key: uploadSession.SavePath}, nil)
}

354
pkg/filesystem/driver/qiniu/handler.go Normal file
View File

@ -0,0 +1,354 @@
package qiniu
import (
"context"
"encoding/base64"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"path/filepath"
"strings"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/qiniu/go-sdk/v7/auth/qbox"
"github.com/qiniu/go-sdk/v7/storage"
)
// Driver 本地策略适配器
type Driver struct {
Policy *model.Policy
mac *qbox.Mac
cfg *storage.Config
bucket *storage.BucketManager
}
func NewDriver(policy *model.Policy) *Driver {
if policy.OptionsSerialized.ChunkSize == 0 {
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
}
mac := qbox.NewMac(policy.AccessKey, policy.SecretKey)
cfg := &storage.Config{UseHTTPS: true}
return &Driver{
Policy: policy,
mac: mac,
cfg: cfg,
bucket: storage.NewBucketManager(mac, cfg),
}
}
// List 列出给定路径下的文件
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
}
var (
delimiter string
marker string
objects []storage.ListItem
commons []string
)
if !recursive {
delimiter = "/"
}
for {
entries, folders, nextMarker, hashNext, err := handler.bucket.ListFiles(
handler.Policy.BucketName,
base, delimiter, marker, 1000)
if err != nil {
return nil, err
}
objects = append(objects, entries...)
commons = append(commons, folders...)
if !hashNext {
break
}
marker = nextMarker
}
// 处理列取结果
res := make([]response.Object, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(base, object)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(base, object.Key)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(object.Key),
Source: object.Key,
RelativePath: filepath.ToSlash(rel),
Size: uint64(object.Fsize),
IsDir: false,
LastModify: time.Unix(object.PutTime/10000000, 0),
})
}
return res, nil
}
// Get 获取文件
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 给文件名加上随机参数以强制拉取
path = fmt.Sprintf("%s?v=%d", path, time.Now().UnixNano())
// 获取文件源地址
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
// 获取文件数据流
client := request.NewClient()
resp, err := client.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithHeader(
http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}},
),
request.WithTimeout(time.Duration(0)),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试自主获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
// 凭证有效期
credentialTTL := model.GetIntSetting("upload_session_timeout", 3600)
// 生成上传策略
fileInfo := file.Info()
scope := handler.Policy.BucketName
if fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite {
scope = fmt.Sprintf("%s:%s", handler.Policy.BucketName, fileInfo.SavePath)
}
putPolicy := storage.PutPolicy{
// 指定为覆盖策略
Scope: scope,
SaveKey: fileInfo.SavePath,
ForceSaveKey: true,
FsizeLimit: int64(fileInfo.Size),
}
// 是否开启了MIMEType限制
if handler.Policy.OptionsSerialized.MimeType != "" {
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
}
// 生成上传凭证
token, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, int64(credentialTTL), false)
if err != nil {
return err
}
// 创建上传表单
cfg := storage.Config{}
formUploader := storage.NewFormUploader(&cfg)
ret := storage.PutRet{}
putExtra := storage.PutExtra{
Params: map[string]string{},
}
// 开始上传
err = formUploader.Put(ctx, &ret, token.Credential, fileInfo.SavePath, file, int64(fileInfo.Size), &putExtra)
if err != nil {
return err
}
return nil
}
// Delete 删除一个或多个文件，
// 返回未删除的文件
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// TODO 大于一千个文件需要分批发送
deleteOps := make([]string, 0, len(files))
for _, key := range files {
deleteOps = append(deleteOps, storage.URIDelete(handler.Policy.BucketName, key))
}
rets, err := handler.bucket.Batch(deleteOps)
// 处理删除结果
if err != nil {
failed := make([]string, 0, len(rets))
for k, ret := range rets {
if ret.Code != 200 && ret.Code != 612 {
failed = append(failed, files[k])
}
}
return failed, errors.New("删除失败")
}
return []string{}, nil
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
// https://developer.qiniu.com/dora/api/basic-processing-images-imageview2
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "tiff", "avif", "psd"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) || file.Size > (20<<(10*2)) {
return nil, driver.ErrorThumbNotSupported
}
var (
thumbSize = [2]uint{400, 300}
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("failed to get thumbnail size")
}
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
thumb := fmt.Sprintf("%s?imageView2/1/w/%d/h/%d/q/%d", file.SourceName, thumbSize[0], thumbSize[1], thumbEncodeQuality)
return &response.ContentResponse{
Redirect: true,
URL: handler.signSourceURL(
ctx,
thumb,
int64(model.GetIntSetting("preview_timeout", 60)),
),
}, nil
}
// Source 获取外链URL
func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := ""
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
fileName = file.Name
}
// 加入下载相关设置
if isDownload {
path = path + "?attname=" + url.PathEscape(fileName)
}
// 取得原始文件地址
return handler.signSourceURL(ctx, path, ttl), nil
}
func (handler *Driver) signSourceURL(ctx context.Context, path string, ttl int64) string {
var sourceURL string
if handler.Policy.IsPrivate {
deadline := time.Now().Add(time.Second * time.Duration(ttl)).Unix()
sourceURL = storage.MakePrivateURL(handler.mac, handler.Policy.BaseURL, path, deadline)
} else {
sourceURL = storage.MakePublicURL(handler.Policy.BaseURL, path)
}
return sourceURL
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/qiniu/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 创建上传策略
fileInfo := file.Info()
putPolicy := storage.PutPolicy{
Scope: handler.Policy.BucketName,
CallbackURL: apiURL.String(),
CallbackBody: `{"size":$(fsize),"pic_info":"$(imageInfo.width),$(imageInfo.height)"}`,
CallbackBodyType: "application/json",
SaveKey: fileInfo.SavePath,
ForceSaveKey: true,
FsizeLimit: int64(handler.Policy.MaxSize),
}
// 是否开启了MIMEType限制
if handler.Policy.OptionsSerialized.MimeType != "" {
putPolicy.MimeLimit = handler.Policy.OptionsSerialized.MimeType
}
credential, err := handler.getUploadCredential(ctx, putPolicy, fileInfo, ttl, true)
if err != nil {
return nil, fmt.Errorf("failed to init parts: %w", err)
}
credential.SessionID = uploadSession.Key
credential.ChunkSize = handler.Policy.OptionsSerialized.ChunkSize
uploadSession.UploadURL = credential.UploadURLs[0]
uploadSession.Credential = credential.Credential
return credential, nil
}
// getUploadCredential 签名上传策略并创建上传会话
func (handler *Driver) getUploadCredential(ctx context.Context, policy storage.PutPolicy, file *fsctx.UploadTaskInfo, TTL int64, resume bool) (*serializer.UploadCredential, error) {
// 上传凭证
policy.Expires = uint64(TTL)
upToken := policy.UploadToken(handler.mac)
// 初始化分片上传
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
upHost, err := resumeUploader.UpHost(handler.Policy.AccessKey, handler.Policy.BucketName)
if err != nil {
return nil, err
}
ret := &storage.InitPartsRet{}
if resume {
err = resumeUploader.InitParts(ctx, upToken, upHost, handler.Policy.BucketName, file.SavePath, true, ret)
}
return &serializer.UploadCredential{
UploadURLs: []string{upHost + "/buckets/" + handler.Policy.BucketName + "/objects/" + base64.URLEncoding.EncodeToString([]byte(file.SavePath)) + "/uploads/" + ret.UploadID},
Credential: upToken,
}, err
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
resumeUploader := storage.NewResumeUploaderV2(handler.cfg)
return resumeUploader.Client.CallWith(ctx, nil, "DELETE", uploadSession.UploadURL, http.Header{"Authorization": {"UpToken " + uploadSession.Credential}}, nil, 0)
}

195
pkg/filesystem/driver/remote/client.go Normal file
View File

@ -0,0 +1,195 @@
package remote
import (
"context"
"encoding/json"
"fmt"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/gofrs/uuid"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
)
const (
basePath = "/api/v3/slave/"
OverwriteHeader = auth.CrHeaderPrefix + "Overwrite"
chunkRetrySleep = time.Duration(5) * time.Second
)
// Client to operate uploading to remote slave server
type Client interface {
// CreateUploadSession creates remote upload session
CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error
// GetUploadURL signs an url for uploading file
GetUploadURL(ttl int64, sessionID string) (string, string, error)
// Upload uploads file to remote server
Upload(ctx context.Context, file fsctx.FileHeader) error
// DeleteUploadSession deletes remote upload session
DeleteUploadSession(ctx context.Context, sessionID string) error
}
// NewClient creates new Client from given policy
func NewClient(policy *model.Policy) (Client, error) {
authInstance := auth.HMACAuth{[]byte(policy.SecretKey)}
serverURL, err := url.Parse(policy.Server)
if err != nil {
return nil, err
}
base, _ := url.Parse(basePath)
signTTL := model.GetIntSetting("slave_api_timeout", 60)
return &remoteClient{
policy: policy,
authInstance: authInstance,
httpClient: request.NewClient(
request.WithEndpoint(serverURL.ResolveReference(base).String()),
request.WithCredential(authInstance, int64(signTTL)),
request.WithMasterMeta(),
request.WithSlaveMeta(policy.AccessKey),
),
}, nil
}
type remoteClient struct {
policy *model.Policy
authInstance auth.Auth
httpClient request.Client
}
func (c *remoteClient) Upload(ctx context.Context, file fsctx.FileHeader) error {
ttl := model.GetIntSetting("upload_session_timeout", 86400)
fileInfo := file.Info()
session := &serializer.UploadSession{
Key: uuid.Must(uuid.NewV4()).String(),
VirtualPath: fileInfo.VirtualPath,
Name: fileInfo.FileName,
Size: fileInfo.Size,
SavePath: fileInfo.SavePath,
LastModified: fileInfo.LastModified,
Policy: *c.policy,
}
// Create upload session
overwrite := fileInfo.Mode&fsctx.Overwrite == fsctx.Overwrite
if err := c.CreateUploadSession(ctx, session, int64(ttl), overwrite); err != nil {
return fmt.Errorf("failed to create upload session: %w", err)
}
// Initial chunk groups
chunks := chunk.NewChunkGroup(file, c.policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{
Max: model.GetIntSetting("chunk_retries", 5),
Sleep: chunkRetrySleep,
}, model.IsTrueVal(model.GetSettingByName("use_temp_chunk_buffer")))
uploadFunc := func(current *chunk.ChunkGroup, content io.Reader) error {
return c.uploadChunk(ctx, session.Key, current.Index(), content, overwrite, current.Length())
}
// upload chunks
for chunks.Next() {
if err := chunks.Process(uploadFunc); err != nil {
if err := c.DeleteUploadSession(ctx, session.Key); err != nil {
util.Log().Warning("failed to delete upload session: %s", err)
}
return fmt.Errorf("failed to upload chunk #%d: %w", chunks.Index(), err)
}
}
return nil
}
func (c *remoteClient) DeleteUploadSession(ctx context.Context, sessionID string) error {
resp, err := c.httpClient.Request(
"DELETE",
"upload/"+sessionID,
nil,
request.WithContext(ctx),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return err
}
if resp.Code != 0 {
return serializer.NewErrorFromResponse(resp)
}
return nil
}
func (c *remoteClient) CreateUploadSession(ctx context.Context, session *serializer.UploadSession, ttl int64, overwrite bool) error {
reqBodyEncoded, err := json.Marshal(map[string]interface{}{
"session": session,
"ttl": ttl,
"overwrite": overwrite,
})
if err != nil {
return err
}
bodyReader := strings.NewReader(string(reqBodyEncoded))
resp, err := c.httpClient.Request(
"PUT",
"upload",
bodyReader,
request.WithContext(ctx),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return err
}
if resp.Code != 0 {
return serializer.NewErrorFromResponse(resp)
}
return nil
}
func (c *remoteClient) GetUploadURL(ttl int64, sessionID string) (string, string, error) {
base, err := url.Parse(c.policy.Server)
if err != nil {
return "", "", err
}
base.Path = path.Join(base.Path, basePath, "upload", sessionID)
req, err := http.NewRequest("POST", base.String(), nil)
if err != nil {
return "", "", err
}
req = auth.SignRequest(c.authInstance, req, ttl)
return req.URL.String(), req.Header["Authorization"][0], nil
}
func (c *remoteClient) uploadChunk(ctx context.Context, sessionID string, index int, chunk io.Reader, overwrite bool, size int64) error {
resp, err := c.httpClient.Request(
"POST",
fmt.Sprintf("upload/%s?chunk=%d", sessionID, index),
chunk,
request.WithContext(ctx),
request.WithTimeout(time.Duration(0)),
request.WithContentLength(size),
request.WithHeader(map[string][]string{OverwriteHeader: {fmt.Sprintf("%t", overwrite)}}),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return err
}
if resp.Code != 0 {
return serializer.NewErrorFromResponse(resp)
}
return nil
}

311
pkg/filesystem/driver/remote/handler.go Normal file
View File

@ -0,0 +1,311 @@
package remote
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/url"
"path"
"path/filepath"
"strings"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/auth"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
)
// Driver 远程存储策略适配器
type Driver struct {
Client request.Client
Policy *model.Policy
AuthInstance auth.Auth
uploadClient Client
}
// NewDriver initializes a new Driver from policy
// TODO: refactor all method into upload client
func NewDriver(policy *model.Policy) (*Driver, error) {
client, err := NewClient(policy)
if err != nil {
return nil, err
}
return &Driver{
Policy: policy,
Client: request.NewClient(),
AuthInstance: auth.HMACAuth{[]byte(policy.SecretKey)},
uploadClient: client,
}, nil
}
// List 列取文件
func (handler *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
var res []response.Object
reqBody := serializer.ListRequest{
Path: path,
Recursive: recursive,
}
reqBodyEncoded, err := json.Marshal(reqBody)
if err != nil {
return res, err
}
// 发送列表请求
bodyReader := strings.NewReader(string(reqBodyEncoded))
signTTL := model.GetIntSetting("slave_api_timeout", 60)
resp, err := handler.Client.Request(
"POST",
handler.getAPIUrl("list"),
bodyReader,
request.WithCredential(handler.AuthInstance, int64(signTTL)),
request.WithMasterMeta(),
).CheckHTTPResponse(200).DecodeResponse()
if err != nil {
return res, err
}
// 处理列取结果
if resp.Code != 0 {
return res, errors.New(resp.Error)
}
if resStr, ok := resp.Data.(string); ok {
err = json.Unmarshal([]byte(resStr), &res)
if err != nil {
return res, err
}
}
return res, nil
}
// getAPIUrl 获取接口请求地址
func (handler *Driver) getAPIUrl(scope string, routes ...string) string {
serverURL, err := url.Parse(handler.Policy.Server)
if err != nil {
return ""
}
var controller *url.URL
switch scope {
case "delete":
controller, _ = url.Parse("/api/v3/slave/delete")
case "thumb":
controller, _ = url.Parse("/api/v3/slave/thumb")
case "list":
controller, _ = url.Parse("/api/v3/slave/list")
default:
controller = serverURL
}
for _, r := range routes {
controller.Path = path.Join(controller.Path, r)
}
return serverURL.ResolveReference(controller).String()
}
// Get 获取文件内容
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 尝试获取速度限制
speedLimit := 0
if user, ok := ctx.Value(fsctx.UserCtx).(model.User); ok {
speedLimit = user.Group.SpeedLimit
}
// 获取文件源地址
downloadURL, err := handler.Source(ctx, path, 0, true, speedLimit)
if err != nil {
return nil, err
}
// 获取文件数据流
resp, err := handler.Client.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithTimeout(time.Duration(0)),
request.WithMasterMeta(),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
return handler.uploadClient.Upload(ctx, file)
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
// 封装接口请求正文
reqBody := serializer.RemoteDeleteRequest{
Files: files,
}
reqBodyEncoded, err := json.Marshal(reqBody)
if err != nil {
return files, err
}
// 发送删除请求
bodyReader := strings.NewReader(string(reqBodyEncoded))
signTTL := model.GetIntSetting("slave_api_timeout", 60)
resp, err := handler.Client.Request(
"POST",
handler.getAPIUrl("delete"),
bodyReader,
request.WithCredential(handler.AuthInstance, int64(signTTL)),
request.WithMasterMeta(),
request.WithSlaveMeta(handler.Policy.AccessKey),
).CheckHTTPResponse(200).GetResponse()
if err != nil {
return files, err
}
// 处理删除结果
var reqResp serializer.Response
err = json.Unmarshal([]byte(resp), &reqResp)
if err != nil {
return files, err
}
if reqResp.Code != 0 {
var failedResp serializer.RemoteDeleteRequest
if failed, ok := reqResp.Data.(string); ok {
err = json.Unmarshal([]byte(failed), &failedResp)
if err == nil {
return failedResp.Files, errors.New(reqResp.Error)
}
}
return files, errors.New("unknown format of returned response")
}
return []string{}, nil
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
supported := []string{"png", "jpg", "jpeg", "gif"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) {
return nil, driver.ErrorThumbNotSupported
}
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(file.SourceName))
thumbURL := fmt.Sprintf("%s/%s/%s", handler.getAPIUrl("thumb"), sourcePath, filepath.Ext(file.Name))
ttl := model.GetIntSetting("preview_timeout", 60)
signedThumbURL, err := auth.SignURI(handler.AuthInstance, thumbURL, int64(ttl))
if err != nil {
return nil, err
}
return &response.ContentResponse{
Redirect: true,
URL: signedThumbURL.String(),
}, nil
}
// Source 获取外链URL
func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := "file"
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
fileName = file.Name
}
serverURL, err := url.Parse(handler.Policy.Server)
if err != nil {
return "", errors.New("无法解析远程服务端地址")
}
// 是否启用了CDN
if handler.Policy.BaseURL != "" {
cdnURL, err := url.Parse(handler.Policy.BaseURL)
if err != nil {
return "", err
}
serverURL = cdnURL
}
var (
signedURI *url.URL
controller = "/api/v3/slave/download"
)
if !isDownload {
controller = "/api/v3/slave/source"
}
// 签名下载地址
sourcePath := base64.RawURLEncoding.EncodeToString([]byte(path))
signedURI, err = auth.SignURI(
handler.AuthInstance,
fmt.Sprintf("%s/%d/%s/%s", controller, speed, sourcePath, url.PathEscape(fileName)),
ttl,
)
if err != nil {
return "", serializer.NewError(serializer.CodeEncryptError, "Failed to sign URL", err)
}
finalURL := serverURL.ResolveReference(signedURI).String()
return finalURL, nil
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse(path.Join("/api/v3/callback/remote", uploadSession.Key, uploadSession.CallbackSecret))
apiURL := siteURL.ResolveReference(apiBaseURI)
// 在从机端创建上传会话
uploadSession.Callback = apiURL.String()
if err := handler.uploadClient.CreateUploadSession(ctx, uploadSession, ttl, false); err != nil {
return nil, err
}
// 获取上传地址
uploadURL, sign, err := handler.uploadClient.GetUploadURL(ttl, uploadSession.Key)
if err != nil {
return nil, fmt.Errorf("failed to sign upload url: %w", err)
}
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadURLs: []string{uploadURL},
Credential: sign,
}, nil
}
// 取消上传凭证
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return handler.uploadClient.DeleteUploadSession(ctx, uploadSession.Key)
}

440
pkg/filesystem/driver/s3/handler.go Normal file
View File

@ -0,0 +1,440 @@
package s3
import (
"context"
"errors"
"fmt"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"io"
"net/http"
"net/url"
"path"
"path/filepath"
"strings"
"time"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/chunk/backoff"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Driver 适配器模板
type Driver struct {
Policy *model.Policy
sess *session.Session
svc *s3.S3
}
// UploadPolicy S3上传策略
type UploadPolicy struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
// MetaData 文件信息
type MetaData struct {
Size uint64
Etag string
}
func NewDriver(policy *model.Policy) (*Driver, error) {
if policy.OptionsSerialized.ChunkSize == 0 {
policy.OptionsSerialized.ChunkSize = 25 << 20 // 25 MB
}
driver := &Driver{
Policy: policy,
}
return driver, driver.InitS3Client()
}
// InitS3Client 初始化S3会话
func (handler *Driver) InitS3Client() error {
if handler.Policy == nil {
return errors.New("empty policy")
}
if handler.svc == nil {
// 初始化会话
sess, err := session.NewSession(&aws.Config{
Credentials: credentials.NewStaticCredentials(handler.Policy.AccessKey, handler.Policy.SecretKey, ""),
Endpoint: &handler.Policy.Server,
Region: &handler.Policy.OptionsSerialized.Region,
S3ForcePathStyle: &handler.Policy.OptionsSerialized.S3ForcePathStyle,
})
if err != nil {
return err
}
handler.sess = sess
handler.svc = s3.New(sess)
}
return nil
}
// List 列出给定路径下的文件
func (handler *Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
// 初始化列目录参数
base = strings.TrimPrefix(base, "/")
if base != "" {
base += "/"
}
opt := &s3.ListObjectsInput{
Bucket: &handler.Policy.BucketName,
Prefix: &base,
MaxKeys: aws.Int64(1000),
}
// 是否为递归列出
if !recursive {
opt.Delimiter = aws.String("/")
}
var (
objects []*s3.Object
commons []*s3.CommonPrefix
)
for {
res, err := handler.svc.ListObjectsWithContext(ctx, opt)
if err != nil {
return nil, err
}
objects = append(objects, res.Contents...)
commons = append(commons, res.CommonPrefixes...)
// 如果本次未列取完，则继续使用marker获取结果
if *res.IsTruncated {
opt.Marker = res.NextMarker
} else {
break
}
}
// 处理列取结果
res := make([]response.Object, 0, len(objects)+len(commons))
// 处理目录
for _, object := range commons {
rel, err := filepath.Rel(*opt.Prefix, *object.Prefix)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(*object.Prefix),
RelativePath: filepath.ToSlash(rel),
Size: 0,
IsDir: true,
LastModify: time.Now(),
})
}
// 处理文件
for _, object := range objects {
rel, err := filepath.Rel(*opt.Prefix, *object.Key)
if err != nil {
continue
}
res = append(res, response.Object{
Name: path.Base(*object.Key),
Source: *object.Key,
RelativePath: filepath.ToSlash(rel),
Size: uint64(*object.Size),
IsDir: false,
LastModify: time.Now(),
})
}
return res, nil
}
// Get 获取文件
func (handler *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
// 获取文件数据流
client := request.NewClient()
resp, err := client.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithHeader(
http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}},
),
request.WithTimeout(time.Duration(0)),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试自主获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return err
}
uploader := s3manager.NewUploader(handler.sess, func(u *s3manager.Uploader) {
u.PartSize = int64(handler.Policy.OptionsSerialized.ChunkSize)
})
dst := file.Info().SavePath
_, err := uploader.Upload(&s3manager.UploadInput{
Bucket: &handler.Policy.BucketName,
Key: &dst,
Body: io.LimitReader(file, int64(file.Info().Size)),
})
if err != nil {
return err
}
return nil
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
failed := make([]string, 0, len(files))
deleted := make([]string, 0, len(files))
keys := make([]*s3.ObjectIdentifier, 0, len(files))
for _, file := range files {
filePath := file
keys = append(keys, &s3.ObjectIdentifier{Key: &filePath})
}
// 发送异步删除请求
res, err := handler.svc.DeleteObjects(
&s3.DeleteObjectsInput{
Bucket: &handler.Policy.BucketName,
Delete: &s3.Delete{
Objects: keys,
},
})
if err != nil {
return files, err
}
// 统计未删除的文件
for _, deleteRes := range res.Deleted {
deleted = append(deleted, *deleteRes.Key)
}
failed = util.SliceDifference(files, deleted)
return failed, nil
}
// Thumb 获取文件缩略图
func (handler *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, driver.ErrorThumbNotSupported
}
// Source 获取外链URL
func (handler *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := ""
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
fileName = file.Name
}
// 初始化客户端
if err := handler.InitS3Client(); err != nil {
return "", err
}
contentDescription := aws.String("attachment; filename=\"" + url.PathEscape(fileName) + "\"")
if !isDownload {
contentDescription = nil
}
req, _ := handler.svc.GetObjectRequest(
&s3.GetObjectInput{
Bucket: &handler.Policy.BucketName,
Key: &path,
ResponseContentDisposition: contentDescription,
})
signedURL, err := req.Presign(time.Duration(ttl) * time.Second)
if err != nil {
return "", err
}
// 将最终生成的签名URL域名换成用户自定义的加速域名（如果有）
finalURL, err := url.Parse(signedURL)
if err != nil {
return "", err
}
// 公有空间替换掉Key及不支持的头
if !handler.Policy.IsPrivate {
finalURL.RawQuery = ""
}
if handler.Policy.BaseURL != "" {
cdnURL, err := url.Parse(handler.Policy.BaseURL)
if err != nil {
return "", err
}
finalURL.Host = cdnURL.Host
finalURL.Scheme = cdnURL.Scheme
}
return finalURL.String(), nil
}
// Token 获取上传策略和认证Token
func (handler *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 检查文件是否存在
fileInfo := file.Info()
if _, err := handler.Meta(ctx, fileInfo.SavePath); err == nil {
return nil, fmt.Errorf("file already exist")
}
// 创建分片上传
expires := time.Now().Add(time.Duration(ttl) * time.Second)
res, err := handler.svc.CreateMultipartUpload(&s3.CreateMultipartUploadInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
Expires: &expires,
ContentType: aws.String(fileInfo.DetectMimeType()),
})
if err != nil {
return nil, fmt.Errorf("failed to create multipart upload: %w", err)
}
uploadSession.UploadID = *res.UploadId
// 为每个分片签名上传 URL
chunks := chunk.NewChunkGroup(file, handler.Policy.OptionsSerialized.ChunkSize, &backoff.ConstantBackoff{}, false)
urls := make([]string, chunks.Num())
for chunks.Next() {
err := chunks.Process(func(c *chunk.ChunkGroup, chunk io.Reader) error {
signedReq, _ := handler.svc.UploadPartRequest(&s3.UploadPartInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
PartNumber: aws.Int64(int64(c.Index() + 1)),
UploadId: res.UploadId,
})
signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
if err != nil {
return err
}
urls[c.Index()] = signedURL
return nil
})
if err != nil {
return nil, err
}
}
// 签名完成分片上传的请求URL
signedReq, _ := handler.svc.CompleteMultipartUploadRequest(&s3.CompleteMultipartUploadInput{
Bucket: &handler.Policy.BucketName,
Key: &fileInfo.SavePath,
UploadId: res.UploadId,
})
signedURL, err := signedReq.Presign(time.Duration(ttl) * time.Second)
if err != nil {
return nil, err
}
// 生成上传凭证
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
ChunkSize: handler.Policy.OptionsSerialized.ChunkSize,
UploadID: *res.UploadId,
UploadURLs: urls,
CompleteURL: signedURL,
}, nil
}
// Meta 获取文件信息
func (handler *Driver) Meta(ctx context.Context, path string) (*MetaData, error) {
res, err := handler.svc.HeadObject(
&s3.HeadObjectInput{
Bucket: &handler.Policy.BucketName,
Key: &path,
})
if err != nil {
return nil, err
}
return &MetaData{
Size: uint64(*res.ContentLength),
Etag: *res.ETag,
}, nil
}
// CORS 创建跨域策略
func (handler *Driver) CORS() error {
rule := s3.CORSRule{
AllowedMethods: aws.StringSlice([]string{
"GET",
"POST",
"PUT",
"DELETE",
"HEAD",
}),
AllowedOrigins: aws.StringSlice([]string{"*"}),
AllowedHeaders: aws.StringSlice([]string{"*"}),
ExposeHeaders: aws.StringSlice([]string{"ETag"}),
MaxAgeSeconds: aws.Int64(3600),
}
_, err := handler.svc.PutBucketCors(&s3.PutBucketCorsInput{
Bucket: &handler.Policy.BucketName,
CORSConfiguration: &s3.CORSConfiguration{
CORSRules: []*s3.CORSRule{&rule},
},
})
return err
}
// 取消上传凭证
func (handler *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
_, err := handler.svc.AbortMultipartUpload(&s3.AbortMultipartUploadInput{
UploadId: &uploadSession.UploadID,
Bucket: &handler.Policy.BucketName,
Key: &uploadSession.SavePath,
})
return err
}

7
pkg/filesystem/driver/shadow/masterinslave/errors.go Normal file
View File

@ -0,0 +1,7 @@
package masterinslave
import "errors"
var (
ErrNotImplemented = errors.New("this method of shadowed policy is not implemented")
)

60
pkg/filesystem/driver/shadow/masterinslave/handler.go Normal file
View File

@ -0,0 +1,60 @@
package masterinslave
import (
"context"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Driver 影子存储策略，用于在从机端上传文件
type Driver struct {
master cluster.Node
handler driver.Handler
policy *model.Policy
}
// NewDriver 返回新的处理器
func NewDriver(master cluster.Node, handler driver.Handler, policy *model.Policy) driver.Handler {
return &Driver{
master: master,
handler: handler,
policy: policy,
}
}
func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
return d.handler.Put(ctx, file)
}
func (d *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
return d.handler.Delete(ctx, files)
}
func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, ErrNotImplemented
}
func (d *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, ErrNotImplemented
}
func (d *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
return "", ErrNotImplemented
}
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
return nil, ErrNotImplemented
}
func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
return nil, ErrNotImplemented
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}

9
pkg/filesystem/driver/shadow/slaveinmaster/errors.go Normal file
View File

@ -0,0 +1,9 @@
package slaveinmaster
import "errors"
var (
ErrNotImplemented = errors.New("this method of shadowed policy is not implemented")
ErrSlaveSrcPathNotExist = errors.New("cannot determine source file path in slave node")
ErrWaitResultTimeout = errors.New("timeout waiting for slave transfer result")
)

124
pkg/filesystem/driver/shadow/slaveinmaster/handler.go Normal file
View File

@ -0,0 +1,124 @@
package slaveinmaster
import (
"bytes"
"context"
"encoding/json"
"errors"
"net/url"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/mq"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
)
// Driver 影子存储策略，将上传任务指派给从机节点处理，并等待从机通知上传结果
type Driver struct {
node cluster.Node
handler driver.Handler
policy *model.Policy
client request.Client
}
// NewDriver 返回新的从机指派处理器
func NewDriver(node cluster.Node, handler driver.Handler, policy *model.Policy) driver.Handler {
var endpoint *url.URL
if serverURL, err := url.Parse(node.DBModel().Server); err == nil {
var controller *url.URL
controller, _ = url.Parse("/api/v3/slave/")
endpoint = serverURL.ResolveReference(controller)
}
signTTL := model.GetIntSetting("slave_api_timeout", 60)
return &Driver{
node: node,
handler: handler,
policy: policy,
client: request.NewClient(
request.WithMasterMeta(),
request.WithTimeout(time.Duration(signTTL)*time.Second),
request.WithCredential(node.SlaveAuthInstance(), int64(signTTL)),
request.WithEndpoint(endpoint.String()),
),
}
}
// Put 将ctx中指定的从机物理文件由从机上传到目标存储策略
func (d *Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
fileInfo := file.Info()
req := serializer.SlaveTransferReq{
Src: fileInfo.Src,
Dst: fileInfo.SavePath,
Policy: d.policy,
}
body, err := json.Marshal(req)
if err != nil {
return err
}
// 订阅转存结果
resChan := mq.GlobalMQ.Subscribe(req.Hash(model.GetSettingByName("siteID")), 0)
defer mq.GlobalMQ.Unsubscribe(req.Hash(model.GetSettingByName("siteID")), resChan)
res, err := d.client.Request("PUT", "task/transfer", bytes.NewReader(body)).
CheckHTTPResponse(200).
DecodeResponse()
if err != nil {
return err
}
if res.Code != 0 {
return serializer.NewErrorFromResponse(res)
}
// 等待转存结果或者超时
waitTimeout := model.GetIntSetting("slave_transfer_timeout", 172800)
select {
case <-time.After(time.Duration(waitTimeout) * time.Second):
return ErrWaitResultTimeout
case msg := <-resChan:
if msg.Event != serializer.SlaveTransferSuccess {
return errors.New(msg.Content.(serializer.SlaveTransferResult).Error)
}
}
return nil
}
func (d *Driver) Delete(ctx context.Context, files []string) ([]string, error) {
return d.handler.Delete(ctx, files)
}
func (d *Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
return nil, ErrNotImplemented
}
func (d *Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
return nil, ErrNotImplemented
}
func (d *Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
return "", ErrNotImplemented
}
func (d *Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
return nil, ErrNotImplemented
}
func (d *Driver) List(ctx context.Context, path string, recursive bool) ([]response.Object, error) {
return nil, ErrNotImplemented
}
// 取消上传凭证
func (d *Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}

358
pkg/filesystem/driver/upyun/handler.go Normal file
View File

@ -0,0 +1,358 @@
package upyun
import (
"context"
"crypto/hmac"
"crypto/md5"
"crypto/sha1"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"strconv"
"strings"
"sync"
"time"
model "github.com/cloudreve/Cloudreve/v3/models"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/driver"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/fsctx"
"github.com/cloudreve/Cloudreve/v3/pkg/filesystem/response"
"github.com/cloudreve/Cloudreve/v3/pkg/request"
"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
"github.com/cloudreve/Cloudreve/v3/pkg/util"
"github.com/upyun/go-sdk/upyun"
)
// UploadPolicy 又拍云上传策略
type UploadPolicy struct {
Bucket string `json:"bucket"`
SaveKey string `json:"save-key"`
Expiration int64 `json:"expiration"`
CallbackURL string `json:"notify-url"`
ContentLength uint64 `json:"content-length"`
ContentLengthRange string `json:"content-length-range,omitempty"`
AllowFileType string `json:"allow-file-type,omitempty"`
}
// Driver 又拍云策略适配器
type Driver struct {
Policy *model.Policy
}
func (handler Driver) List(ctx context.Context, base string, recursive bool) ([]response.Object, error) {
base = strings.TrimPrefix(base, "/")
// 用于接受SDK返回对象的chan
objChan := make(chan *upyun.FileInfo)
objects := []*upyun.FileInfo{}
// 列取配置
listConf := &upyun.GetObjectsConfig{
Path: "/" + base,
ObjectsChan: objChan,
MaxListTries: 1,
}
// 递归列取时不限制递归次数
if recursive {
listConf.MaxListLevel = -1
}
// 启动一个goroutine收集列取的对象信
wg := &sync.WaitGroup{}
wg.Add(1)
go func(input chan *upyun.FileInfo, output *[]*upyun.FileInfo, wg *sync.WaitGroup) {
defer wg.Done()
for {
file, ok := <-input
if !ok {
return
}
*output = append(*output, file)
}
}(objChan, &objects, wg)
up := upyun.NewUpYun(&upyun.UpYunConfig{
Bucket: handler.Policy.BucketName,
Operator: handler.Policy.AccessKey,
Password: handler.Policy.SecretKey,
})
err := up.List(listConf)
if err != nil {
return nil, err
}
wg.Wait()
// 汇总处理列取结果
res := make([]response.Object, 0, len(objects))
for _, object := range objects {
res = append(res, response.Object{
Name: path.Base(object.Name),
RelativePath: object.Name,
Source: path.Join(base, object.Name),
Size: uint64(object.Size),
IsDir: object.IsDir,
LastModify: object.Time,
})
}
return res, nil
}
// Get 获取文件
func (handler Driver) Get(ctx context.Context, path string) (response.RSCloser, error) {
// 获取文件源地址
downloadURL, err := handler.Source(ctx, path, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
// 获取文件数据流
client := request.NewClient()
resp, err := client.Request(
"GET",
downloadURL,
nil,
request.WithContext(ctx),
request.WithHeader(
http.Header{"Cache-Control": {"no-cache", "no-store", "must-revalidate"}},
),
request.WithTimeout(time.Duration(0)),
).CheckHTTPResponse(200).GetRSCloser()
if err != nil {
return nil, err
}
resp.SetFirstFakeChunk()
// 尝试自主获取文件大小
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
resp.SetContentLength(int64(file.Size))
}
return resp, nil
}
// Put 将文件流保存到指定目录
func (handler Driver) Put(ctx context.Context, file fsctx.FileHeader) error {
defer file.Close()
up := upyun.NewUpYun(&upyun.UpYunConfig{
Bucket: handler.Policy.BucketName,
Operator: handler.Policy.AccessKey,
Password: handler.Policy.SecretKey,
})
err := up.Put(&upyun.PutObjectConfig{
Path: file.Info().SavePath,
Reader: file,
})
return err
}
// Delete 删除一个或多个文件，
// 返回未删除的文件，及遇到的最后一个错误
func (handler Driver) Delete(ctx context.Context, files []string) ([]string, error) {
up := upyun.NewUpYun(&upyun.UpYunConfig{
Bucket: handler.Policy.BucketName,
Operator: handler.Policy.AccessKey,
Password: handler.Policy.SecretKey,
})
var (
failed = make([]string, 0, len(files))
lastErr error
currentIndex = 0
indexLock sync.Mutex
failedLock sync.Mutex
wg sync.WaitGroup
routineNum = 4
)
wg.Add(routineNum)
// upyun不支持批量操作，这里开四个协程并行操作
for i := 0; i < routineNum; i++ {
go func() {
for {
// 取得待删除文件
indexLock.Lock()
if currentIndex >= len(files) {
// 所有文件处理完成
wg.Done()
indexLock.Unlock()
return
}
path := files[currentIndex]
currentIndex++
indexLock.Unlock()
// 发送异步删除请求
err := up.Delete(&upyun.DeleteObjectConfig{
Path: path,
Async: true,
})
// 处理错误
if err != nil {
failedLock.Lock()
lastErr = err
failed = append(failed, path)
failedLock.Unlock()
}
}
}()
}
wg.Wait()
return failed, lastErr
}
// Thumb 获取文件缩略图
func (handler Driver) Thumb(ctx context.Context, file *model.File) (*response.ContentResponse, error) {
// quick check by extension name
// https://help.upyun.com/knowledge-base/image/
supported := []string{"png", "jpg", "jpeg", "gif", "bmp", "webp", "svg"}
if len(handler.Policy.OptionsSerialized.ThumbExts) > 0 {
supported = handler.Policy.OptionsSerialized.ThumbExts
}
if !util.IsInExtensionList(supported, file.Name) {
return nil, driver.ErrorThumbNotSupported
}
var (
thumbSize = [2]uint{400, 300}
ok = false
)
if thumbSize, ok = ctx.Value(fsctx.ThumbSizeCtx).([2]uint); !ok {
return nil, errors.New("failed to get thumbnail size")
}
thumbEncodeQuality := model.GetIntSetting("thumb_encode_quality", 85)
thumbParam := fmt.Sprintf("!/fwfh/%dx%d/quality/%d", thumbSize[0], thumbSize[1], thumbEncodeQuality)
thumbURL, err := handler.Source(ctx, file.SourceName+thumbParam, int64(model.GetIntSetting("preview_timeout", 60)), false, 0)
if err != nil {
return nil, err
}
return &response.ContentResponse{
Redirect: true,
URL: thumbURL,
}, nil
}
// Source 获取外链URL
func (handler Driver) Source(ctx context.Context, path string, ttl int64, isDownload bool, speed int) (string, error) {
// 尝试从上下文获取文件名
fileName := ""
if file, ok := ctx.Value(fsctx.FileModelCtx).(model.File); ok {
fileName = file.Name
}
sourceURL, err := url.Parse(handler.Policy.BaseURL)
if err != nil {
return "", err
}
fileKey, err := url.Parse(url.PathEscape(path))
if err != nil {
return "", err
}
sourceURL = sourceURL.ResolveReference(fileKey)
// 如果是下载文件URL
if isDownload {
query := sourceURL.Query()
query.Add("_upd", fileName)
sourceURL.RawQuery = query.Encode()
}
return handler.signURL(ctx, sourceURL, ttl)
}
func (handler Driver) signURL(ctx context.Context, path *url.URL, TTL int64) (string, error) {
if !handler.Policy.IsPrivate {
// 未开启Token防盗链时，直接返回
return path.String(), nil
}
etime := time.Now().Add(time.Duration(TTL) * time.Second).Unix()
signStr := fmt.Sprintf(
"%s&%d&%s",
handler.Policy.OptionsSerialized.Token,
etime,
path.Path,
)
signMd5 := fmt.Sprintf("%x", md5.Sum([]byte(signStr)))
finalSign := signMd5[12:20] + strconv.FormatInt(etime, 10)
// 将签名添加到URL中
query := path.Query()
query.Add("_upt", finalSign)
path.RawQuery = query.Encode()
return path.String(), nil
}
// Token 获取上传策略和认证Token
func (handler Driver) Token(ctx context.Context, ttl int64, uploadSession *serializer.UploadSession, file fsctx.FileHeader) (*serializer.UploadCredential, error) {
// 生成回调地址
siteURL := model.GetSiteURL()
apiBaseURI, _ := url.Parse("/api/v3/callback/upyun/" + uploadSession.Key)
apiURL := siteURL.ResolveReference(apiBaseURI)
// 上传策略
fileInfo := file.Info()
putPolicy := UploadPolicy{
Bucket: handler.Policy.BucketName,
// TODO escape
SaveKey: fileInfo.SavePath,
Expiration: time.Now().Add(time.Duration(ttl) * time.Second).Unix(),
CallbackURL: apiURL.String(),
ContentLength: fileInfo.Size,
ContentLengthRange: fmt.Sprintf("0,%d", fileInfo.Size),
AllowFileType: strings.Join(handler.Policy.OptionsSerialized.FileType, ","),
}
// 生成上传凭证
policyJSON, err := json.Marshal(putPolicy)
if err != nil {
return nil, err
}
policyEncoded := base64.StdEncoding.EncodeToString(policyJSON)
// 生成签名
elements := []string{"POST", "/" + handler.Policy.BucketName, policyEncoded}
signStr := handler.Sign(ctx, elements)
return &serializer.UploadCredential{
SessionID: uploadSession.Key,
Policy: policyEncoded,
Credential: signStr,
UploadURLs: []string{"https://v0.api.upyun.com/" + handler.Policy.BucketName},
}, nil
}
// 取消上传凭证
func (handler Driver) CancelToken(ctx context.Context, uploadSession *serializer.UploadSession) error {
return nil
}
// Sign 计算又拍云的签名头
func (handler Driver) Sign(ctx context.Context, elements []string) string {
password := fmt.Sprintf("%x", md5.Sum([]byte(handler.Policy.SecretKey)))
mac := hmac.New(sha1.New, []byte(password))
value := strings.Join(elements, "&")
mac.Write([]byte(value))
signStr := base64.StdEncoding.EncodeToString((mac.Sum(nil)))
return fmt.Sprintf("UPYUN %s:%s", handler.Policy.AccessKey, signStr)
}