From 8ef8023c20bfeee97ec82155b52eae0d80b1410e Mon Sep 17 00:00:00 2001 From: NewbieOrange Date: Thu, 19 Oct 2023 19:17:09 +0800 Subject: [PATCH 001/659] fix(aliyundrive_open): upload progress for normal upload (#5398) --- drivers/aliyundrive_open/upload.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index e4a0cf7ec0d..73c37dcfbde 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -5,7 +5,6 @@ import ( "context" "encoding/base64" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" "io" "math" "net/http" @@ -16,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" @@ -258,6 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m return nil, err } offset += partSize + up(i * 100 / count) } } else { log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId) From aaffaee2b54fc067d240ea0c20ea3c2f39615d6e Mon Sep 17 00:00:00 2001 From: gmugu <94156510@qq.com> Date: Thu, 19 Oct 2023 19:17:53 +0800 Subject: [PATCH 002/659] perf(webdav): support request with cookies (#5391) --- drivers/webdav/util.go | 8 ++++++++ pkg/gowebdav/client.go | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 92557c4f553..84eebb2ec3c 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -2,6 +2,7 @@ package webdav import ( "net/http" + "net/http/cookiejar" "github.com/alist-org/alist/v3/drivers/webdav/odrvcookie" "github.com/alist-org/alist/v3/internal/model" @@ -26,6 +27,13 @@ func (d *WebDav) setClient() error { } else { return err } + } else { + cookieJar, err := cookiejar.New(nil) + if err == nil { + c.SetJar(cookieJar) + } else { + return err + } } d.client = c return nil diff --git a/pkg/gowebdav/client.go b/pkg/gowebdav/client.go index 6e12289c1ac..2fca0b7f43d 100644 --- a/pkg/gowebdav/client.go +++ b/pkg/gowebdav/client.go @@ -83,6 +83,11 @@ func (c *Client) SetTransport(transport http.RoundTripper) { c.c.Transport = transport } +// SetJar exposes the ability to set a cookie jar to the client. +func (c *Client) SetJar(jar http.CookieJar) { + c.c.Jar = jar +} + // Connect connects to our dav server func (c *Client) Connect() error { rs, err := c.options("/") @@ -351,6 +356,11 @@ func (c *Client) Link(path string) (string, http.Header, error) { return "", nil, newPathErrorErr("Link", path, err) } + if c.c.Jar != nil { + for _, cookie := range c.c.Jar.Cookies(r.URL) { + r.AddCookie(cookie) + } + } for k, vals := range c.headers { for _, v := range vals { r.Header.Add(k, v) From 4fc0a77565702f9bf498485d42336502f2ee9776 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 20 Oct 2023 21:06:25 +0800 Subject: [PATCH 003/659] fix(baidu_netdisk): upload file > 4GB (close #5392) --- drivers/baidu_netdisk/driver.go | 20 ++++++++++++-------- drivers/baidu_netdisk/util.go | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 654c8b01bc7..d5f71814d0f 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -27,10 +27,9 @@ type BaiduNetdisk struct { Addition uploadThread int + vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) } -const DefaultSliceSize int64 = 4 * utils.MB - func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -53,7 +52,11 @@ func (d *BaiduNetdisk) Init(ctx context.Context) error { "method": "uinfo", }, nil) log.Debugf("[baidu] get uinfo: %s", string(res)) - return err + if err != nil { + return err + } + d.vipType = utils.Json.Get(res, "vip_type").ToInt() + return nil } func (d *BaiduNetdisk) Drop(ctx context.Context) error { @@ -177,17 +180,18 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } streamSize := stream.GetSize() - count := int(math.Max(math.Ceil(float64(streamSize)/float64(DefaultSliceSize)), 1)) - lastBlockSize := streamSize % DefaultSliceSize + sliceSize := d.getSliceSize() + count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1)) + lastBlockSize := streamSize % sliceSize if streamSize > 0 && lastBlockSize == 0 { - lastBlockSize = DefaultSliceSize + lastBlockSize = sliceSize } //cal md5 for first 256k data const SliceSize int64 = 256 * 1024 // cal md5 blockList := make([]string, 0, count) - byteSize := DefaultSliceSize + byteSize := sliceSize fileMd5H := md5.New() sliceMd5H := md5.New() sliceMd5H2 := md5.New() @@ -257,7 +261,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F break } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*DefaultSliceSize, DefaultSliceSize + i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize if partseq+1 == count { byteSize = lastBlockSize } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index d972eb83fa7..6c51156c22f 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -242,6 +242,23 @@ func updateObjMd5(obj model.Obj, userAgent, u string) { } } +const ( + DefaultSliceSize int64 = 4 * utils.MB + VipSliceSize = 16 * utils.MB + SVipSliceSize = 32 * utils.MB +) + +func (d *BaiduNetdisk) getSliceSize() int64 { + switch d.vipType { + case 1: + return VipSliceSize + case 2: + return SVipSliceSize + default: + return DefaultSliceSize + } +} + // func encodeURIComponent(str string) string { // r := url.QueryEscape(str) // r = strings.ReplaceAll(r, "+", "%20") From c0f9c8ebafdf8dd2afe5c0b9fba24456819c3155 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 26 Oct 2023 19:21:09 +0800 Subject: [PATCH 004/659] feat: add ignore direct link params (close #5434) --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 1 + internal/op/hook.go | 4 ++++ server/handles/down.go | 8 ++++++-- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index ca17b6b148f..6afd774265f 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -139,6 +139,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL}, {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, + {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, // aria2 settings diff --git a/internal/conf/const.go b/internal/conf/const.go index 02a00060a68..eb70602ad3f 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -41,6 +41,7 @@ const ( OcrApi = "ocr_api" FilenameCharMapping = "filename_char_mapping" ForwardDirectLinkParams = "forward_direct_link_params" + IgnoreDirectLinkParams = "ignore_direct_link_params" WebauthnLoginEnabled = "webauthn_login_enabled" // index diff --git a/internal/op/hook.go b/internal/op/hook.go index e37e52df269..23b8e59af2c 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -78,6 +78,10 @@ var settingItemHooks = map[string]SettingItemHook{ log.Debugf("filename char mapping: %+v", conf.FilenameCharMap) return nil }, + conf.IgnoreDirectLinkParams: func(item *model.SettingItem) error { + conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") + return nil + }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { diff --git a/server/handles/down.go b/server/handles/down.go index e4aec494243..d3d41e85a2b 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -52,7 +52,9 @@ func Down(c *gin.Context) { c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") if setting.GetBool(conf.ForwardDirectLinkParams) { query := c.Request.URL.Query() - query.Del("sign") + for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { + query.Del(v) + } link.URL, err = utils.InjectQuery(link.URL, query) if err != nil { common.ErrorResp(c, err, 500) @@ -95,7 +97,9 @@ func Proxy(c *gin.Context) { } if link.URL != "" && setting.GetBool(conf.ForwardDirectLinkParams) { query := c.Request.URL.Query() - query.Del("sign") + for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { + query.Del(v) + } link.URL, err = utils.InjectQuery(link.URL, query) if err != nil { common.ErrorResp(c, err, 500) From cc86d6f3d1ff2120669c9dda719b7faabb922f52 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 29 Oct 2023 14:45:55 +0800 Subject: [PATCH 005/659] fix(deps): update module golang.org/x/net to v0.17.0 [security] (#5370) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f185aef9b1e..f2201f8aca9 100644 --- a/go.mod +++ b/go.mod @@ -50,8 +50,9 @@ require ( golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/image v0.11.0 - golang.org/x/net v0.16.0 + golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.12.0 + golang.org/x/time v0.3.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 gorm.io/driver/sqlite v1.4.4 @@ -189,7 +190,6 @@ require ( golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect google.golang.org/api v0.134.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect diff --git a/go.sum b/go.sum index 0fb9a131f25..19150ac9b6d 100644 --- a/go.sum +++ b/go.sum @@ -546,6 +546,8 @@ golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= From 4dff49470adce36416d8c56594e84868c04d023b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:10:36 +0800 Subject: [PATCH 006/659] fix(deps): update golang.org/x/exp digest to 7918f67 (#5366) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f2201f8aca9..f2986ddabf6 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 golang.org/x/crypto v0.14.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/image v0.11.0 golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.12.0 diff --git a/go.sum b/go.sum index 19150ac9b6d..b278dbb2a41 100644 --- a/go.sum +++ b/go.sum @@ -520,6 +520,8 @@ golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= From a6325967d0de18e6b6c744f06cb1ebaa08ec687e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 15:11:20 +0800 Subject: [PATCH 007/659] fix(deps): update module github.com/charmbracelet/lipgloss to v0.9.1 (#5234) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f2986ddabf6..5a6ed9b2390 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.16.1 github.com/charmbracelet/bubbletea v0.24.2 - github.com/charmbracelet/lipgloss v0.7.1 + github.com/charmbracelet/lipgloss v0.9.1 github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.3.1 github.com/disintegration/imaging v1.6.2 @@ -138,7 +138,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.0 // indirect @@ -151,7 +151,7 @@ require ( github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.15.1 // indirect + github.com/muesli/termenv v0.15.2 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.9.0 // indirect diff --git a/go.sum b/go.sum index b278dbb2a41..940eed1d6ed 100644 --- a/go.sum +++ b/go.sum @@ -117,6 +117,8 @@ github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06 github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= +github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= +github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= @@ -320,6 +322,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= @@ -349,6 +353,8 @@ github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= From 65c5ec0c34d5f027a65933fe89af53791747bdd4 Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Sat, 4 Nov 2023 13:35:09 +0800 Subject: [PATCH 008/659] feat(cloudreve): folder size count and switch (#5457 close #5395) --- drivers/cloudreve/driver.go | 8 ++++++++ drivers/cloudreve/meta.go | 11 ++++++----- drivers/cloudreve/types.go | 4 ++++ drivers/cloudreve/util.go | 3 +++ 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 2a22380e314..49c2d5f00f2 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -53,6 +53,14 @@ func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs if err != nil { return nil, err } + if src.Type == "dir" && d.EnableThumbAndFolderSize { + var dprop DirectoryProp + err = d.request(http.MethodGet, "/object/property/"+src.Id+"?is_folder=true", nil, &dprop) + if err != nil { + return nil, err + } + src.Size = dprop.Size + } return objectToObj(src, thumb), nil }) } diff --git a/drivers/cloudreve/meta.go b/drivers/cloudreve/meta.go index d01d54791d3..92c0b9fb1d7 100644 --- a/drivers/cloudreve/meta.go +++ b/drivers/cloudreve/meta.go @@ -9,11 +9,12 @@ type Addition struct { // Usually one of two driver.RootPath // define other - Address string `json:"address" required:"true"` - Username string `json:"username"` - Password string `json:"password"` - Cookie string `json:"cookie"` - CustomUA string `json:"custom_ua"` + Address string `json:"address" required:"true"` + Username string `json:"username"` + Password string `json:"password"` + Cookie string `json:"cookie"` + CustomUA string `json:"custom_ua"` + EnableThumbAndFolderSize bool `json:"enable_thumb_and_folder_size"` } var config = driver.Config{ diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index e25673829a8..241d993ebb8 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -44,6 +44,10 @@ type Object struct { SourceEnabled bool `json:"source_enabled"` } +type DirectoryProp struct { + Size int `json:"size"` +} + func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index ed8794667c1..284e3289dee 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -151,6 +151,9 @@ func convertSrc(obj model.Obj) map[string]interface{} { } func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { + if !d.Addition.EnableThumbAndFolderSize { + return model.Thumbnail{}, nil + } ua := d.CustomUA if ua == "" { ua = base.UserAgent From 68f440abdb471f9580237611d0b1804466fe248c Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sun, 5 Nov 2023 22:41:14 +0800 Subject: [PATCH 009/659] fix(weiyun): unmarshal overflow (#5459) --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 5a6ed9b2390..70b8383d7ab 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/djherbis/times v1.5.0 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.4 - github.com/foxxorcat/weiyun-sdk-go v0.1.2 + github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.9.1 diff --git a/go.sum b/go.sum index 940eed1d6ed..6ea29399734 100644 --- a/go.sum +++ b/go.sum @@ -147,12 +147,10 @@ github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU= github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= -github.com/foxxorcat/mopan-sdk-go v0.1.3 h1:6ww0ulyLDh6neXZBqUM2PDbxQ6lfdkQbr0FCh9BTY0Y= -github.com/foxxorcat/mopan-sdk-go v0.1.3/go.mod h1:iWHA2JFhzmKR28ySp1ON0g6DjLaYtvb5jhTqPVTDW9A= github.com/foxxorcat/mopan-sdk-go v0.1.4 h1:6utvPiBv8KDRDVKB7A4FERdrVxcHKZd2fBFCNuKcXzU= github.com/foxxorcat/mopan-sdk-go v0.1.4/go.mod h1:iWHA2JFhzmKR28ySp1ON0g6DjLaYtvb5jhTqPVTDW9A= -github.com/foxxorcat/weiyun-sdk-go v0.1.2 h1:waRWIBmjL9GCcndJ8HvOYrrVB4hhoPYzRrn3I/Cnzqw= -github.com/foxxorcat/weiyun-sdk-go v0.1.2/go.mod h1:AKsLFuWhWlClpGrg1zxTdMejugZEZtmhIuElAk3W83s= +github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= +github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= From 3bbdd4fa894840b73b882b1e6f744ead2a7b2ab8 Mon Sep 17 00:00:00 2001 From: sheltonzhu <498220739@qq.com> Date: Mon, 6 Nov 2023 16:53:57 +0800 Subject: [PATCH 010/659] fix(115): fix driver package import and variable (#5482) names --- drivers/115/util.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/drivers/115/util.go b/drivers/115/util.go index 35d1fbda759..8e638d79a6a 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -18,26 +18,25 @@ import ( "sync" "time" - "github.com/SheltonZhu/115driver/pkg/driver" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/conf" "github.com/pkg/errors" ) -var UserAgent = driver.UA115Desktop +var UserAgent = driver115.UA115Desktop func (d *Pan115) login() error { var err error - opts := []driver.Option{ - driver.UA(UserAgent), - func(c *driver.Pan115Client) { + opts := []driver115.Option{ + driver115.UA(UserAgent), + func(c *driver115.Pan115Client) { c.Client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) }, } - d.client = driver.New(opts...) - cr := &driver.Credential{} + d.client = driver115.New(opts...) + cr := &driver115.Credential{} if d.Addition.QRCodeToken != "" { - s := &driver.QRCodeSession{ + s := &driver115.QRCodeSession{ UID: d.Addition.QRCodeToken, } if cr, err = d.client.QRCodeLogin(s); err != nil { @@ -59,7 +58,7 @@ func (d *Pan115) login() error { func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { res := make([]FileObj, 0) if d.PageSize <= 0 { - d.PageSize = driver.FileListLimit + d.PageSize = driver115.FileListLimit } files, err := d.client.ListWithLimit(fileId, d.PageSize) if err != nil { @@ -249,7 +248,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i go func(threadId int) { defer func() { if r := recover(); r != nil { - errCh <- fmt.Errorf("Recovered in %v", r) + errCh <- fmt.Errorf("recovered in %v", r) } }() for chunk := range chunksCh { From 769281bd405ff05c4514292c11cb87843868763a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 6 Nov 2023 16:56:55 +0800 Subject: [PATCH 011/659] feat: refactor offline download (#5408 close #4108) * wip: refactor offline download (#5331) * base tool * working: aria2 * refactor: change type of percentage to float64 * wip: adapt aria2 * wip: use items in offline_download * wip: use tool manager * wip: adapt qBittorrent * chore: fix typo * Squashed commit of the following: commit 4fc0a77565702f9bf498485d42336502f2ee9776 Author: Andy Hsu Date: Fri Oct 20 21:06:25 2023 +0800 fix(baidu_netdisk): upload file > 4GB (close #5392) commit aaffaee2b54fc067d240ea0c20ea3c2f39615d6e Author: gmugu <94156510@qq.com> Date: Thu Oct 19 19:17:53 2023 +0800 perf(webdav): support request with cookies (#5391) commit 8ef8023c20bfeee97ec82155b52eae0d80b1410e Author: NewbieOrange Date: Thu Oct 19 19:17:09 2023 +0800 fix(aliyundrive_open): upload progress for normal upload (#5398) commit cdfbe6dcf2b361e4c93c2703c2f8c9bddeac0ee6 Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Wed Oct 18 16:27:07 2023 +0800 fix: hash gcid empty file (#5394) commit 94d028743abf8e0d736f80c0ec4fb294a1cc064c Author: Andy Hsu Date: Sat Oct 14 13:17:51 2023 +0800 ci: remove `pr-welcome` label when close issue [skip ci] commit 7f7335435c2f32a3eef76fac4c4f783d9d8624fd Author: itsHenry <2671230065@qq.com> Date: Sat Oct 14 13:12:46 2023 +0800 feat(cloudreve): support thumbnail (#5373 close #5348) * feat(cloudreve): support thumbnail * chore: remove unnecessary code commit b9e192b29cffddf14a0dfb2d3885def57a56ce16 Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Thu Oct 12 20:57:12 2023 +0800 fix(115): limit request rate (#5367 close #5275) * fix(115):limit request rate * chore(115): fix unit of `limit_rate` --------- Co-authored-by: Andy Hsu commit 69a98eaef612b58596e5c26c341b6d7cedecdf19 Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Wed Oct 11 22:01:55 2023 +0800 fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.9+incompatible (#5141) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit 1ebc96a4e5220c979fd581bb3b5640e9436f6665 Author: Andy Hsu Date: Tue Oct 10 18:32:00 2023 +0800 fix(wopan): fatal error concurrent map writes (close #5352) commit 66e2324cac75cb3ef05af45dbdd10b124d534aff Author: Andy Hsu Date: Tue Oct 10 18:23:11 2023 +0800 chore(deps): upgrade dependencies commit 7600dc28df137c439e538b4257731c33a63db9b5 Author: Andy Hsu Date: Tue Oct 10 18:13:58 2023 +0800 fix(aliyundrive_open): change default api to raw server (close #5358) commit 8ef89ad0a496d5acc398794c0afa4f77c67ad371 Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Tue Oct 10 18:08:27 2023 +0800 fix(baidu_netdisk): hash and `error 2` (#5356) * fix(baidu):hash and error:2 * fix:invalid memory address commit 35d672217dde69e65b41b1fcd9786c1cfebcdc45 Author: jeffmingup <1960588251@qq.com> Date: Sun Oct 8 19:29:45 2023 +0800 fix(onedrive_app): incorrect api on `_accessToken` (#5346) commit 1a283bb2720eff6d1b0c1dd6f1667a6449905a9b Author: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Fri Oct 6 16:04:39 2023 +0800 feat(google_drive): add `hash_info`, `ctime`, `thumbnail` (#5334) commit a008f54f4d5eda5738abfd54bf1abf1e18c08430 Author: nkh0472 <67589323+nkh0472@users.noreply.github.com> Date: Thu Oct 5 13:10:51 2023 +0800 docs: minor language improvements (#5329) [skip ci] * fix: adapt update progress type * Squashed commit of the following: commit 65c5ec0c34d5f027a65933fe89af53791747bdd4 Author: itsHenry <2671230065@qq.com> Date: Sat Nov 4 13:35:09 2023 +0800 feat(cloudreve): folder size count and switch (#5457 close #5395) commit a6325967d0de18e6b6c744f06cb1ebaa08ec687e Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Mon Oct 30 15:11:20 2023 +0800 fix(deps): update module github.com/charmbracelet/lipgloss to v0.9.1 (#5234) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit 4dff49470adce36416d8c56594e84868c04d023b Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Mon Oct 30 15:10:36 2023 +0800 fix(deps): update golang.org/x/exp digest to 7918f67 (#5366) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit cc86d6f3d1ff2120669c9dda719b7faabb922f52 Author: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Date: Sun Oct 29 14:45:55 2023 +0800 fix(deps): update module golang.org/x/net to v0.17.0 [security] (#5370) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> commit c0f9c8ebafdf8dd2afe5c0b9fba24456819c3155 Author: Andy Hsu Date: Thu Oct 26 19:21:09 2023 +0800 feat: add ignore direct link params (close #5434) --- cmd/root.go | 2 + cmd/server.go | 4 +- drivers/123/upload.go | 2 +- drivers/189/util.go | 2 +- drivers/189pc/utils.go | 6 +- drivers/aliyundrive/driver.go | 5 +- drivers/aliyundrive_open/upload.go | 2 +- drivers/baidu_netdisk/driver.go | 2 +- drivers/baidu_photo/driver.go | 2 +- drivers/dropbox/driver.go | 2 +- drivers/mega/driver.go | 7 +- drivers/mopan/driver.go | 2 +- drivers/onedrive/util.go | 2 +- drivers/onedrive_app/util.go | 2 +- drivers/quark_uc/driver.go | 2 +- drivers/s3/driver.go | 5 +- drivers/teambition/util.go | 2 +- drivers/terabox/driver.go | 2 +- drivers/trainbit/driver.go | 3 +- drivers/wopan/driver.go | 2 +- internal/aria2/monitor.go | 5 +- internal/bootstrap/aria2.go | 16 - internal/bootstrap/data/setting.go | 10 +- internal/bootstrap/offline_download.go | 17 + internal/bootstrap/qbittorrent.go | 15 - internal/driver/driver.go | 4 +- internal/model/setting.go | 2 +- internal/model/user.go | 9 +- internal/offline_download/all.go | 6 + internal/offline_download/aria2/aria2.go | 133 ++++++++ internal/offline_download/aria2/notify.go | 70 ++++ internal/offline_download/qbit/qbit.go | 80 +++++ internal/offline_download/tool/add.go | 84 +++++ internal/offline_download/tool/all_test.go | 17 + internal/offline_download/tool/base.go | 59 ++++ internal/offline_download/tool/monitor.go | 159 +++++++++ internal/offline_download/tool/tools.go | 42 +++ internal/offline_download/tool/util.go | 28 ++ internal/op/fs.go | 2 +- internal/qbittorrent/monitor.go | 5 +- pkg/qbittorrent/client.go | 366 +++++++++++++++++++++ pkg/task/task.go | 6 +- pkg/utils/io.go | 7 +- server/handles/aria2.go | 80 ----- server/handles/offline_download.go | 111 +++++++ server/handles/qbittorrent.go | 79 ----- server/handles/task.go | 20 +- server/router.go | 6 +- 48 files changed, 1238 insertions(+), 258 deletions(-) delete mode 100644 internal/bootstrap/aria2.go create mode 100644 internal/bootstrap/offline_download.go delete mode 100644 internal/bootstrap/qbittorrent.go create mode 100644 internal/offline_download/all.go create mode 100644 internal/offline_download/aria2/aria2.go create mode 100644 internal/offline_download/aria2/notify.go create mode 100644 internal/offline_download/qbit/qbit.go create mode 100644 internal/offline_download/tool/add.go create mode 100644 internal/offline_download/tool/all_test.go create mode 100644 internal/offline_download/tool/base.go create mode 100644 internal/offline_download/tool/monitor.go create mode 100644 internal/offline_download/tool/tools.go create mode 100644 internal/offline_download/tool/util.go create mode 100644 pkg/qbittorrent/client.go delete mode 100644 server/handles/aria2.go create mode 100644 server/handles/offline_download.go delete mode 100644 server/handles/qbittorrent.go diff --git a/cmd/root.go b/cmd/root.go index 297eb7f8940..6bd82b7a4a3 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,6 +5,8 @@ import ( "os" "github.com/alist-org/alist/v3/cmd/flags" + _ "github.com/alist-org/alist/v3/drivers" + _ "github.com/alist-org/alist/v3/internal/offline_download" "github.com/spf13/cobra" ) diff --git a/cmd/server.go b/cmd/server.go index 94a60c7208f..0678e3e1188 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -13,7 +13,6 @@ import ( "time" "github.com/alist-org/alist/v3/cmd/flags" - _ "github.com/alist-org/alist/v3/drivers" "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/pkg/utils" @@ -35,8 +34,7 @@ the address is defined in config file`, utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart) time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second) } - bootstrap.InitAria2() - bootstrap.InitQbittorrent() + bootstrap.InitOfflineDownloadTools() bootstrap.LoadStorages() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) diff --git a/drivers/123/upload.go b/drivers/123/upload.go index ae28d6aa519..6f6221f1148 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -107,7 +107,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi if err != nil { return err } - up(j * 100 / chunkCount) + up(float64(j) * 100 / float64(chunkCount)) } } // complete s3 upload diff --git a/drivers/189/util.go b/drivers/189/util.go index 680ce252133..0b4c0633d7b 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -380,7 +380,7 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F if err != nil { return err } - up(int(i * 100 / count)) + up(float64(i) * 100 / float64(count)) } fileMd5 := hex.EncodeToString(md5Sum.Sum(nil)) sliceMd5 := fileMd5 diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 1868aeb25ff..5e403a830e4 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -513,7 +513,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if err != nil { return err } - up(int(threadG.Success()) * 100 / count) + up(float64(threadG.Success()) * 100 / float64(count)) return nil }) } @@ -676,7 +676,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode return err } - up(int(threadG.Success()) * 100 / len(uploadUrls)) + up(float64(threadG.Success()) * 100 / float64(len(uploadUrls))) uploadProgress.UploadParts[i] = "" return nil }) @@ -812,7 +812,7 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model if _, err := tempFile.Seek(status.GetSize(), io.SeekStart); err != nil { return nil, err } - up(int(status.GetSize()/file.GetSize()) * 100) + up(float64(status.GetSize()) / float64(file.GetSize()) * 100) } return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId) diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index f11452629a4..83c3f522452 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -7,7 +7,6 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "math" "math/big" @@ -15,6 +14,8 @@ import ( "os" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" @@ -304,7 +305,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil } res.Body.Close() if count > 0 { - up(i * 100 / count) + up(float64(i) * 100 / float64(count)) } } var resp2 base.Json diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 73c37dcfbde..3b224e7d225 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -258,7 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m return nil, err } offset += partSize - up(i * 100 / count) + up(float64(i*100) / float64(count)) } } else { log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index d5f71814d0f..20810a768de 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -278,7 +278,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if err != nil { return err } - up(int(threadG.Success()) * 100 / len(precreateResp.BlockList)) + up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) precreateResp.BlockList[i] = -1 return nil }) diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index 9105260d94d..c29bc110095 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -329,7 +329,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - up(int(threadG.Success()) * 100 / len(precreateResp.BlockList)) + up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) precreateResp.BlockList[i] = -1 return nil }) diff --git a/drivers/dropbox/driver.go b/drivers/dropbox/driver.go index 7559d645858..95148b94e96 100644 --- a/drivers/dropbox/driver.go +++ b/drivers/dropbox/driver.go @@ -203,7 +203,7 @@ func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt _ = res.Body.Close() if count > 0 { - up((i + 1) * 100 / count) + up(float64(i+1) * 100 / float64(count)) } offset += byteSize diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index c1ae9f7f6c9..9fa1d0eeafa 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -4,11 +4,12 @@ import ( "context" "errors" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/rclone/rclone/lib/readers" "io" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/rclone/rclone/lib/readers" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -169,7 +170,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return err } - up(id * 100 / u.Chunks()) + up(float64(id) * 100 / float64(u.Chunks())) } _, err = u.Finish() diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index bd2de2b30af..78ec0423cc3 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -308,7 +308,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if resp.StatusCode != http.StatusOK { return fmt.Errorf("upload err,code=%d", resp.StatusCode) } - up(100 * int(threadG.Success()) / len(parts)) + up(100 * float64(threadG.Success()) / float64(len(parts))) initUpdload.PartInfos[i] = "" return nil }) diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 0539e098682..a0c6fa8fcbf 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -203,7 +203,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil return errors.New(string(data)) } res.Body.Close() - up(int(finish * 100 / stream.GetSize())) + up(float64(finish) * 100 / float64(stream.GetSize())) } return nil } diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 525a451dd7e..28b34837806 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -194,7 +194,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. return errors.New(string(data)) } res.Body.Close() - up(int(finish * 100 / stream.GetSize())) + up(float64(finish) * 100 / float64(stream.GetSize())) } return nil } diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 7c254022a92..291189ce088 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -209,7 +209,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File } md5s = append(md5s, m) partNumber++ - up(int(100 * (total - left) / total)) + up(100 * float64(total-left) / float64(total)) } err = d.upCommit(pre, md5s) if err != nil { diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index dd643f5d76e..c8099ee43dd 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -4,13 +4,14 @@ import ( "bytes" "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/url" stdpath "path" "strings" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/aws/aws-sdk-go/aws/session" @@ -104,7 +105,7 @@ func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) e }, Reader: io.NopCloser(bytes.NewReader([]byte{})), Mimetype: "application/octet-stream", - }, func(int) {}) + }, func(float64) {}) } func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { diff --git a/drivers/teambition/util.go b/drivers/teambition/util.go index 04f222de95f..c39ffb18286 100644 --- a/drivers/teambition/util.go +++ b/drivers/teambition/util.go @@ -189,7 +189,7 @@ func (d *Teambition) chunkUpload(ctx context.Context, file model.FileStreamer, t if err != nil { return nil, err } - up(i * 100 / newChunk.Chunks) + up(float64(i) * 100 / float64(newChunk.Chunks)) } _, err = base.RestyClient.R().SetHeader("Authorization", token).Post( fmt.Sprintf("https://%s.teambition.net/upload/chunk/%s", diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index b5287f5a7f8..c9662fce03a 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -213,7 +213,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt } log.Debugln(res.String()) if len(precreateResp.BlockList) > 0 { - up(i * 100 / len(precreateResp.BlockList)) + up(float64(i) * 100 / float64(len(precreateResp.BlockList))) } } _, err = d.create(rawPath, stream.GetSize(), 0, precreateResp.Uploadid, block_list_str) diff --git a/drivers/trainbit/driver.go b/drivers/trainbit/driver.go index 63bd0627f63..795b2fb8a2e 100644 --- a/drivers/trainbit/driver.go +++ b/drivers/trainbit/driver.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "math" "net/http" "net/url" "strings" @@ -128,7 +127,7 @@ func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, stream model.FileS stream, func(byteNum int) { total += int64(byteNum) - up(int(math.Round(float64(total) / float64(stream.GetSize()) * 100))) + up(float64(total) / float64(stream.GetSize()) * 100) }, } req, err := http.NewRequest(http.MethodPost, endpoint.String(), progressReader) diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index a3f222e8ef3..e5e26c94a08 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -159,7 +159,7 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre ContentType: stream.GetMimetype(), }, dstDir.GetID(), d.FamilyID, wopan.Upload2COption{ OnProgress: func(current, total int64) { - up(int(100 * current / total)) + up(100 * float64(current) / float64(total)) }, }) return err diff --git a/internal/aria2/monitor.go b/internal/aria2/monitor.go index 77265b372b1..aaef3fd7c70 100644 --- a/internal/aria2/monitor.go +++ b/internal/aria2/monitor.go @@ -2,7 +2,6 @@ package aria2 import ( "fmt" - "github.com/alist-org/alist/v3/internal/stream" "os" "path" "path/filepath" @@ -11,6 +10,8 @@ import ( "sync/atomic" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/task" @@ -100,7 +101,7 @@ func (m *Monitor) Update() (bool, error) { downloaded = 0 } progress := float64(downloaded) / float64(total) * 100 - m.tsk.SetProgress(int(progress)) + m.tsk.SetProgress(progress) switch info.Status { case "complete": err := m.Complete() diff --git a/internal/bootstrap/aria2.go b/internal/bootstrap/aria2.go deleted file mode 100644 index 60017ddaa6f..00000000000 --- a/internal/bootstrap/aria2.go +++ /dev/null @@ -1,16 +0,0 @@ -package bootstrap - -import ( - "github.com/alist-org/alist/v3/internal/aria2" - "github.com/alist-org/alist/v3/pkg/utils" -) - -func InitAria2() { - go func() { - _, err := aria2.InitClient(2) - if err != nil { - //utils.Log.Errorf("failed to init aria2 client: %+v", err) - utils.Log.Infof("Aria2 not ready.") - } - }() -} diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 6afd774265f..21a432ddebd 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -4,6 +4,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" @@ -142,10 +143,6 @@ func InitialSettings() []model.SettingItem { {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, - // aria2 settings - {Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - {Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX}, @@ -168,11 +165,8 @@ func InitialSettings() []model.SettingItem { {Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC}, - - // qbittorrent settings - {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, } + initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { initialSettingItems = append(initialSettingItems, []model.SettingItem{ {Key: "test_deprecated", Value: "test_value", Type: conf.TypeString, Flag: model.DEPRECATED}, diff --git a/internal/bootstrap/offline_download.go b/internal/bootstrap/offline_download.go new file mode 100644 index 00000000000..26e04071b10 --- /dev/null +++ b/internal/bootstrap/offline_download.go @@ -0,0 +1,17 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func InitOfflineDownloadTools() { + for k, v := range tool.Tools { + res, err := v.Init() + if err != nil { + utils.Log.Warnf("init tool %s failed: %s", k, err) + } else { + utils.Log.Infof("init tool %s success: %s", k, res) + } + } +} diff --git a/internal/bootstrap/qbittorrent.go b/internal/bootstrap/qbittorrent.go deleted file mode 100644 index 315977ebe3f..00000000000 --- a/internal/bootstrap/qbittorrent.go +++ /dev/null @@ -1,15 +0,0 @@ -package bootstrap - -import ( - "github.com/alist-org/alist/v3/internal/qbittorrent" - "github.com/alist-org/alist/v3/pkg/utils" -) - -func InitQbittorrent() { - go func() { - err := qbittorrent.InitClient() - if err != nil { - utils.Log.Infof("qbittorrent not ready.") - } - }() -} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index e0a7c93d908..781e85325ee 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -109,7 +109,7 @@ type PutResult interface { Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error) } -type UpdateProgress func(percentage int) +type UpdateProgress func(percentage float64) type Progress struct { Total int64 @@ -120,7 +120,7 @@ type Progress struct { func (p *Progress) Write(b []byte) (n int, err error) { n = len(b) p.Done += int64(n) - p.up(int(float64(p.Done) / float64(p.Total) * 100)) + p.up(float64(p.Done) / float64(p.Total) * 100) return } diff --git a/internal/model/setting.go b/internal/model/setting.go index f4202ee022c..3b2c30f1361 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -6,7 +6,7 @@ const ( STYLE PREVIEW GLOBAL - ARIA2 + OFFLINE_DOWNLOAD INDEX SSO ) diff --git a/internal/model/user.go b/internal/model/user.go index d7b2863cebe..46fe9bd301d 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -32,7 +32,7 @@ type User struct { // Determine permissions by bit // 0: can see hidden files // 1: can access without password - // 2: can add aria2 tasks + // 2: can add offline download tasks // 3: can mkdir and upload // 4: can rename // 5: can move @@ -40,7 +40,6 @@ type User struct { // 7: can remove // 8: webdav read // 9: webdav write - // 10: can add qbittorrent tasks Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -83,7 +82,7 @@ func (u *User) CanAccessWithoutPassword() bool { return u.IsAdmin() || (u.Permission>>1)&1 == 1 } -func (u *User) CanAddAria2Tasks() bool { +func (u *User) CanAddOfflineDownloadTasks() bool { return u.IsAdmin() || (u.Permission>>2)&1 == 1 } @@ -115,10 +114,6 @@ func (u *User) CanWebdavManage() bool { return u.IsAdmin() || (u.Permission>>9)&1 == 1 } -func (u *User) CanAddQbittorrentTasks() bool { - return u.IsAdmin() || (u.Permission>>10)&1 == 1 -} - func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go new file mode 100644 index 00000000000..0c7853cb13f --- /dev/null +++ b/internal/offline_download/all.go @@ -0,0 +1,6 @@ +package offline_download + +import ( + _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" + _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" +) diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go new file mode 100644 index 00000000000..f2b9628c240 --- /dev/null +++ b/internal/offline_download/aria2/aria2.go @@ -0,0 +1,133 @@ +package aria2 + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/aria2/rpc" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +var notify = NewNotify() + +type Aria2 struct { + client rpc.Client +} + +func (a *Aria2) Items() []model.SettingItem { + // aria2 settings + return []model.SettingItem{ + {Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (a *Aria2) Init() (string, error) { + a.client = nil + uri := setting.GetStr(conf.Aria2Uri) + secret := setting.GetStr(conf.Aria2Secret) + c, err := rpc.New(context.Background(), uri, secret, 4*time.Second, notify) + if err != nil { + return "", errors.Wrap(err, "failed to init aria2 client") + } + version, err := c.GetVersion() + if err != nil { + return "", errors.Wrapf(err, "failed get aria2 version") + } + a.client = c + log.Infof("using aria2 version: %s", version.Version) + return fmt.Sprintf("aria2 version: %s", version.Version), nil +} + +func (a *Aria2) IsReady() bool { + return a.client != nil +} + +func (a *Aria2) AddURL(args *tool.AddUrlArgs) (string, error) { + options := map[string]interface{}{ + "dir": args.TempDir, + } + gid, err := a.client.AddURI([]string{args.Url}, options) + if err != nil { + return "", err + } + return gid, nil +} + +func (a *Aria2) Remove(tid string) error { + _, err := a.client.Remove(tid) + return err +} + +func (a *Aria2) Status(tid string) (*tool.Status, error) { + info, err := a.client.TellStatus(tid) + if err != nil { + return nil, err + } + total, err := strconv.ParseUint(info.TotalLength, 10, 64) + if err != nil { + total = 0 + } + downloaded, err := strconv.ParseUint(info.CompletedLength, 10, 64) + if err != nil { + downloaded = 0 + } + s := &tool.Status{ + Completed: info.Status == "complete", + Err: err, + } + s.Progress = float64(downloaded) / float64(total) * 100 + if len(info.FollowedBy) != 0 { + s.NewTID = info.FollowedBy[0] + notify.Signals.Delete(tid) + //notify.Signals.Store(gid, m.c) + } + switch info.Status { + case "complete": + s.Completed = true + case "error": + s.Err = errors.Errorf("failed to download %s, error: %s", tid, info.ErrorMessage) + case "active": + s.Status = "aria2: " + info.Status + if info.Seeder == "true" { + s.Completed = true + } + case "waiting", "paused": + s.Status = "aria2: " + info.Status + case "removed": + s.Err = errors.Errorf("failed to download %s, removed", tid) + default: + return nil, errors.Errorf("[aria2] unknown status %s", info.Status) + } + return s, nil +} + +func (a *Aria2) GetFiles(tid string) []tool.File { + //files, err := a.client.GetFiles(tid) + //if err != nil { + // return nil + //} + //return utils.MustSliceConvert(files, func(f rpc.FileInfo) tool.File { + // return tool.File{ + // //ReadCloser: nil, + // Name: path.Base(f.Path), + // Size: f.Length, + // Path: "", + // Modified: time.Time{}, + // } + //}) + return nil +} + +var _ tool.Tool = (*Aria2)(nil) + +func init() { + tool.Tools.Add("aria2", &Aria2{}) +} diff --git a/internal/offline_download/aria2/notify.go b/internal/offline_download/aria2/notify.go new file mode 100644 index 00000000000..056fe5147b4 --- /dev/null +++ b/internal/offline_download/aria2/notify.go @@ -0,0 +1,70 @@ +package aria2 + +import ( + "github.com/alist-org/alist/v3/pkg/aria2/rpc" + "github.com/alist-org/alist/v3/pkg/generic_sync" +) + +const ( + Downloading = iota + Paused + Stopped + Completed + Errored +) + +type Notify struct { + Signals generic_sync.MapOf[string, chan int] +} + +func NewNotify() *Notify { + return &Notify{Signals: generic_sync.MapOf[string, chan int]{}} +} + +func (n *Notify) OnDownloadStart(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Downloading + } + } +} + +func (n *Notify) OnDownloadPause(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Paused + } + } +} + +func (n *Notify) OnDownloadStop(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Stopped + } + } +} + +func (n *Notify) OnDownloadComplete(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Completed + } + } +} + +func (n *Notify) OnDownloadError(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Errored + } + } +} + +func (n *Notify) OnBtDownloadComplete(events []rpc.Event) { + for _, e := range events { + if signal, ok := n.Signals.Load(e.Gid); ok { + signal <- Completed + } + } +} diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go new file mode 100644 index 00000000000..594088f0eb2 --- /dev/null +++ b/internal/offline_download/qbit/qbit.go @@ -0,0 +1,80 @@ +package qbit + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/qbittorrent" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/pkg/errors" +) + +type QBittorrent struct { + client qbittorrent.Client +} + +func (a *QBittorrent) Items() []model.SettingItem { + // qBittorrent settings + return []model.SettingItem{ + {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (a *QBittorrent) Init() (string, error) { + a.client = nil + url := setting.GetStr(conf.QbittorrentUrl) + qbClient, err := qbittorrent.New(url) + if err != nil { + return "", err + } + a.client = qbClient + return "ok", nil +} + +func (a *QBittorrent) IsReady() bool { + return a.client != nil +} + +func (a *QBittorrent) AddURL(args *tool.AddUrlArgs) (string, error) { + err := a.client.AddFromLink(args.Url, args.TempDir, args.UID) + if err != nil { + return "", err + } + return args.UID, nil +} + +func (a *QBittorrent) Remove(tid string) error { + err := a.client.Delete(tid, true) + return err +} + +func (a *QBittorrent) Status(tid string) (*tool.Status, error) { + info, err := a.client.GetInfo(tid) + if err != nil { + return nil, err + } + s := &tool.Status{} + s.Progress = float64(info.Completed) / float64(info.Size) * 100 + switch info.State { + case qbittorrent.UPLOADING, qbittorrent.PAUSEDUP, qbittorrent.QUEUEDUP, qbittorrent.STALLEDUP, qbittorrent.FORCEDUP, qbittorrent.CHECKINGUP: + s.Completed = true + case qbittorrent.ALLOCATING, qbittorrent.DOWNLOADING, qbittorrent.METADL, qbittorrent.PAUSEDDL, qbittorrent.QUEUEDDL, qbittorrent.STALLEDDL, qbittorrent.CHECKINGDL, qbittorrent.FORCEDDL, qbittorrent.CHECKINGRESUMEDATA, qbittorrent.MOVING: + s.Status = "[qBittorrent] downloading" + case qbittorrent.ERROR, qbittorrent.MISSINGFILES, qbittorrent.UNKNOWN: + s.Err = errors.Errorf("[qBittorrent] failed to download %s, error: %s", tid, info.State) + default: + s.Err = errors.Errorf("[qBittorrent] unknown error occurred downloading %s", tid) + } + return s, nil +} + +func (a *QBittorrent) GetFiles(tid string) []tool.File { + return nil +} + +var _ tool.Tool = (*QBittorrent)(nil) + +func init() { + tool.Tools.Add("qBittorrent", &QBittorrent{}) +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go new file mode 100644 index 00000000000..ceaf92d3bc3 --- /dev/null +++ b/internal/offline_download/tool/add.go @@ -0,0 +1,84 @@ +package tool + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/task" + "github.com/google/uuid" + "github.com/pkg/errors" +) + +type AddURLArgs struct { + URL string + DstDirPath string + Tool string +} + +func AddURL(ctx context.Context, args *AddURLArgs) error { + // get tool + tool, err := Tools.Get(args.Tool) + if err != nil { + return errors.Wrapf(err, "failed get tool") + } + // check tool is ready + if !tool.IsReady() { + // try to init tool + if _, err := tool.Init(); err != nil { + return errors.Wrapf(err, "failed init tool %s", args.Tool) + } + } + // check storage + storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + // check is it could upload + if storage.Config().NoUpload { + return errors.WithStack(errs.UploadNotSupported) + } + // check path is valid + obj, err := op.Get(ctx, storage, dstDirActualPath) + if err != nil { + if !errs.IsObjectNotFound(err) { + return errors.WithMessage(err, "failed get object") + } + } else { + if !obj.IsDir() { + // can't add to a file + return errors.WithStack(errs.NotFolder) + } + } + + uid := uuid.NewString() + tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) + signal := make(chan int) + gid, err := tool.AddURL(&AddUrlArgs{ + Url: args.URL, + UID: uid, + TempDir: tempDir, + Signal: signal, + }) + if err != nil { + return errors.Wrapf(err, "[%s] failed to add uri %s", args.Tool, args.URL) + } + DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ + ID: gid, + Name: fmt.Sprintf("download %s to [%s](%s)", args.URL, storage.GetStorage().MountPath, dstDirActualPath), + Func: func(tsk *task.Task[string]) error { + m := &Monitor{ + tool: tool, + tsk: tsk, + tempDir: tempDir, + dstDirPath: args.DstDirPath, + signal: signal, + } + return m.Loop() + }, + })) + return nil +} diff --git a/internal/offline_download/tool/all_test.go b/internal/offline_download/tool/all_test.go new file mode 100644 index 00000000000..27da5e32a89 --- /dev/null +++ b/internal/offline_download/tool/all_test.go @@ -0,0 +1,17 @@ +package tool_test + +import ( + "testing" + + "github.com/alist-org/alist/v3/internal/offline_download/tool" +) + +func TestGetFiles(t *testing.T) { + files, err := tool.GetFiles("..") + if err != nil { + t.Fatal(err) + } + for _, file := range files { + t.Log(file.Name, file.Size, file.Path, file.Modified) + } +} diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go new file mode 100644 index 00000000000..4689635b54e --- /dev/null +++ b/internal/offline_download/tool/base.go @@ -0,0 +1,59 @@ +package tool + +import ( + "io" + "os" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type AddUrlArgs struct { + Url string + UID string + TempDir string + Signal chan int +} + +type Status struct { + Progress float64 + NewTID string + Completed bool + Status string + Err error +} + +type Tool interface { + // Items return the setting items the tool need + Items() []model.SettingItem + Init() (string, error) + IsReady() bool + // AddURL add an uri to download, return the task id + AddURL(args *AddUrlArgs) (string, error) + // Remove the download if task been canceled + Remove(tid string) error + // Status return the status of the download task, if an error occurred, return the error in Status.Err + Status(tid string) (*Status, error) + // GetFiles return the files of the download task, if nil, means walk the temp dir to get the files + GetFiles(tid string) []File +} + +type File struct { + // ReadCloser for http client + io.ReadCloser + Name string + Size int64 + Path string + Modified time.Time +} + +func (f *File) GetReadCloser() (io.ReadCloser, error) { + if f.ReadCloser != nil { + return f.ReadCloser, nil + } + file, err := os.Open(f.Path) + if err != nil { + return nil, err + } + return file, nil +} diff --git a/internal/offline_download/tool/monitor.go b/internal/offline_download/tool/monitor.go new file mode 100644 index 00000000000..984bda17cb7 --- /dev/null +++ b/internal/offline_download/tool/monitor.go @@ -0,0 +1,159 @@ +package tool + +import ( + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/task" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type Monitor struct { + tool Tool + tsk *task.Task[string] + tempDir string + retried int + dstDirPath string + finish chan struct{} + signal chan int +} + +func (m *Monitor) Loop() error { + m.finish = make(chan struct{}) + var ( + err error + ok bool + ) +outer: + for { + select { + case <-m.tsk.Ctx.Done(): + err := m.tool.Remove(m.tsk.ID) + return err + case <-m.signal: + ok, err = m.Update() + if ok { + break outer + } + case <-time.After(time.Second * 2): + ok, err = m.Update() + if ok { + break outer + } + } + } + if err != nil { + return err + } + m.tsk.SetStatus("aria2 download completed, transferring") + <-m.finish + m.tsk.SetStatus("completed") + return nil +} + +// Update download status, return true if download completed +func (m *Monitor) Update() (bool, error) { + info, err := m.tool.Status(m.tsk.ID) + if err != nil { + m.retried++ + log.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) + return false, nil + } + if m.retried > 5 { + return true, errors.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) + } + m.retried = 0 + m.tsk.SetProgress(info.Progress) + m.tsk.SetStatus("tool: " + info.Status) + if info.NewTID != "" { + log.Debugf("followen by: %+v", info.NewTID) + DownTaskManager.RawTasks().Delete(m.tsk.ID) + m.tsk.ID = info.NewTID + DownTaskManager.RawTasks().Store(m.tsk.ID, m.tsk) + return false, nil + } + // if download completed + if info.Completed { + err := m.Complete() + return true, errors.WithMessage(err, "failed to transfer file") + } + // if download failed + if info.Err != nil { + return true, errors.Errorf("failed to download %s, error: %s", m.tsk.ID, info.Err.Error()) + } + return false, nil +} + +var TransferTaskManager = task.NewTaskManager(3, func(k *uint64) { + atomic.AddUint64(k, 1) +}) + +func (m *Monitor) Complete() error { + // check dstDir again + storage, dstDirActualPath, err := op.GetStorageAndActualPath(m.dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + var files []File + if f := m.tool.GetFiles(m.tsk.ID); f != nil { + files = f + } else { + files, err = GetFiles(m.tempDir) + if err != nil { + return errors.Wrapf(err, "failed to get files") + } + } + // upload files + var wg sync.WaitGroup + wg.Add(len(files)) + go func() { + wg.Wait() + err := os.RemoveAll(m.tempDir) + m.finish <- struct{}{} + if err != nil { + log.Errorf("failed to remove aria2 temp dir: %+v", err.Error()) + } + }() + for i, _ := range files { + file := files[i] + TransferTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ + Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, storage.GetStorage().MountPath, dstDirActualPath), + Func: func(tsk *task.Task[uint64]) error { + defer wg.Done() + mimetype := utils.GetMimeType(file.Path) + rc, err := file.GetReadCloser() + if err != nil { + return errors.Wrapf(err, "failed to open file %s", file.Path) + } + s := &stream.FileStream{ + Ctx: nil, + Obj: &model.Object{ + Name: filepath.Base(file.Path), + Size: file.Size, + Modified: file.Modified, + IsFolder: false, + }, + Reader: rc, + Mimetype: mimetype, + Closers: utils.NewClosers(rc), + } + relDir, err := filepath.Rel(m.tempDir, filepath.Dir(file.Path)) + if err != nil { + log.Errorf("find relation directory error: %v", err) + } + newDistDir := filepath.Join(dstDirActualPath, relDir) + return op.Put(tsk.Ctx, storage, newDistDir, s, tsk.SetProgress) + }, + })) + } + return nil +} diff --git a/internal/offline_download/tool/tools.go b/internal/offline_download/tool/tools.go new file mode 100644 index 00000000000..b7eacbd2b9b --- /dev/null +++ b/internal/offline_download/tool/tools.go @@ -0,0 +1,42 @@ +package tool + +import ( + "fmt" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/task" +) + +var ( + Tools = make(ToolsManager) + DownTaskManager = task.NewTaskManager[string](3) +) + +type ToolsManager map[string]Tool + +func (t ToolsManager) Get(name string) (Tool, error) { + if tool, ok := t[name]; ok { + return tool, nil + } + return nil, fmt.Errorf("tool %s not found", name) +} + +func (t ToolsManager) Add(name string, tool Tool) { + t[name] = tool +} + +func (t ToolsManager) Names() []string { + names := make([]string, 0, len(t)) + for name := range t { + names = append(names, name) + } + return names +} + +func (t ToolsManager) Items() []model.SettingItem { + var items []model.SettingItem + for _, tool := range t { + items = append(items, tool.Items()...) + } + return items +} diff --git a/internal/offline_download/tool/util.go b/internal/offline_download/tool/util.go new file mode 100644 index 00000000000..4258eff61e0 --- /dev/null +++ b/internal/offline_download/tool/util.go @@ -0,0 +1,28 @@ +package tool + +import ( + "os" + "path/filepath" +) + +func GetFiles(dir string) ([]File, error) { + var files []File + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + files = append(files, File{ + Name: info.Name(), + Size: info.Size(), + Path: path, + Modified: info.ModTime(), + }) + } + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} diff --git a/internal/op/fs.go b/internal/op/fs.go index 8ee6993e091..9fe7d5e6a3f 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -534,7 +534,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } // if up is nil, set a default to prevent panic if up == nil { - up = func(p int) {} + up = func(p float64) {} } switch s := storage.(type) { diff --git a/internal/qbittorrent/monitor.go b/internal/qbittorrent/monitor.go index 12bb4ad21c5..bfb1bcf42e1 100644 --- a/internal/qbittorrent/monitor.go +++ b/internal/qbittorrent/monitor.go @@ -2,13 +2,14 @@ package qbittorrent import ( "fmt" - "github.com/alist-org/alist/v3/internal/stream" "os" "path/filepath" "sync" "sync/atomic" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/task" @@ -85,7 +86,7 @@ func (m *Monitor) update() (bool, error) { } progress := float64(info.Completed) / float64(info.Size) * 100 - m.tsk.SetProgress(int(progress)) + m.tsk.SetProgress(progress) switch info.State { case UPLOADING, PAUSEDUP, QUEUEDUP, STALLEDUP, FORCEDUP, CHECKINGUP: err = m.complete() diff --git a/pkg/qbittorrent/client.go b/pkg/qbittorrent/client.go new file mode 100644 index 00000000000..ec3f7e7b00c --- /dev/null +++ b/pkg/qbittorrent/client.go @@ -0,0 +1,366 @@ +package qbittorrent + +import ( + "bytes" + "errors" + "io" + "mime/multipart" + "net/http" + "net/http/cookiejar" + "net/url" + + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Client interface { + AddFromLink(link string, savePath string, id string) error + GetInfo(id string) (TorrentInfo, error) + GetFiles(id string) ([]FileInfo, error) + Delete(id string, deleteFiles bool) error +} + +type client struct { + url *url.URL + client http.Client + Client +} + +func New(webuiUrl string) (Client, error) { + u, err := url.Parse(webuiUrl) + if err != nil { + return nil, err + } + + jar, err := cookiejar.New(nil) + if err != nil { + return nil, err + } + var c = &client{ + url: u, + client: http.Client{Jar: jar}, + } + + err = c.checkAuthorization() + if err != nil { + return nil, err + } + return c, nil +} + +func (c *client) checkAuthorization() error { + // check authorization + if c.authorized() { + return nil + } + + // check authorization after logging in + err := c.login() + if err != nil { + return err + } + if c.authorized() { + return nil + } + return errors.New("unauthorized qbittorrent url") +} + +func (c *client) authorized() bool { + resp, err := c.post("/api/v2/app/version", nil) + if err != nil { + return false + } + return resp.StatusCode == 200 // the status code will be 403 if not authorized +} + +func (c *client) login() error { + // prepare HTTP request + v := url.Values{} + v.Set("username", c.url.User.Username()) + passwd, _ := c.url.User.Password() + v.Set("password", passwd) + resp, err := c.post("/api/v2/auth/login", v) + if err != nil { + return err + } + + // check result + body := make([]byte, 2) + _, err = resp.Body.Read(body) + if err != nil { + return err + } + if string(body) != "Ok" { + return errors.New("failed to login into qBittorrent webui with url: " + c.url.String()) + } + return nil +} + +func (c *client) post(path string, data url.Values) (*http.Response, error) { + u := c.url.JoinPath(path) + u.User = nil // remove userinfo for requests + + req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(data.Encode()))) + if err != nil { + return nil, err + } + if data != nil { + req.Header.Add("Content-Type", "application/x-www-form-urlencoded") + } + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + if resp.Cookies() != nil { + c.client.Jar.SetCookies(u, resp.Cookies()) + } + return resp, nil +} + +func (c *client) AddFromLink(link string, savePath string, id string) error { + err := c.checkAuthorization() + if err != nil { + return err + } + + buf := new(bytes.Buffer) + writer := multipart.NewWriter(buf) + + addField := func(name string, value string) { + if err != nil { + return + } + err = writer.WriteField(name, value) + } + addField("urls", link) + addField("savepath", savePath) + addField("tags", "alist-"+id) + addField("autoTMM", "false") + if err != nil { + return err + } + + err = writer.Close() + if err != nil { + return err + } + + u := c.url.JoinPath("/api/v2/torrents/add") + u.User = nil // remove userinfo for requests + req, err := http.NewRequest("POST", u.String(), buf) + if err != nil { + return err + } + req.Header.Add("Content-Type", writer.FormDataContentType()) + + resp, err := c.client.Do(req) + if err != nil { + return err + } + + // check result + body := make([]byte, 2) + _, err = resp.Body.Read(body) + if err != nil { + return err + } + if resp.StatusCode != 200 || string(body) != "Ok" { + return errors.New("failed to add qBittorrent task: " + link) + } + return nil +} + +type TorrentStatus string + +const ( + ERROR TorrentStatus = "error" + MISSINGFILES TorrentStatus = "missingFiles" + UPLOADING TorrentStatus = "uploading" + PAUSEDUP TorrentStatus = "pausedUP" + QUEUEDUP TorrentStatus = "queuedUP" + STALLEDUP TorrentStatus = "stalledUP" + CHECKINGUP TorrentStatus = "checkingUP" + FORCEDUP TorrentStatus = "forcedUP" + ALLOCATING TorrentStatus = "allocating" + DOWNLOADING TorrentStatus = "downloading" + METADL TorrentStatus = "metaDL" + PAUSEDDL TorrentStatus = "pausedDL" + QUEUEDDL TorrentStatus = "queuedDL" + STALLEDDL TorrentStatus = "stalledDL" + CHECKINGDL TorrentStatus = "checkingDL" + FORCEDDL TorrentStatus = "forcedDL" + CHECKINGRESUMEDATA TorrentStatus = "checkingResumeData" + MOVING TorrentStatus = "moving" + UNKNOWN TorrentStatus = "unknown" +) + +// https://github.com/DGuang21/PTGo/blob/main/app/client/client_distributer.go +type TorrentInfo struct { + AddedOn int `json:"added_on"` // 将 torrent 添加到客户端的时间（Unix Epoch） + AmountLeft int64 `json:"amount_left"` // 剩余大小（字节） + AutoTmm bool `json:"auto_tmm"` // 此 torrent 是否由 Automatic Torrent Management 管理 + Availability float64 `json:"availability"` // 当前百分比 + Category string `json:"category"` // + Completed int64 `json:"completed"` // 完成的传输数据量（字节） + CompletionOn int `json:"completion_on"` // Torrent 完成的时间（Unix Epoch） + ContentPath string `json:"content_path"` // torrent 内容的绝对路径（多文件 torrent 的根路径，单文件 torrent 的绝对文件路径） + DlLimit int `json:"dl_limit"` // Torrent 下载速度限制（字节/秒） + Dlspeed int `json:"dlspeed"` // Torrent 下载速度（字节/秒） + Downloaded int64 `json:"downloaded"` // 已经下载大小 + DownloadedSession int64 `json:"downloaded_session"` // 此会话下载的数据量 + Eta int `json:"eta"` // + FLPiecePrio bool `json:"f_l_piece_prio"` // 如果第一个最后一块被优先考虑，则为true + ForceStart bool `json:"force_start"` // 如果为此 torrent 启用了强制启动，则为true + Hash string `json:"hash"` // + LastActivity int `json:"last_activity"` // 上次活跃的时间（Unix Epoch） + MagnetURI string `json:"magnet_uri"` // 与此 torrent 对应的 Magnet URI + MaxRatio float64 `json:"max_ratio"` // 种子/上传停止种子前的最大共享比率 + MaxSeedingTime int `json:"max_seeding_time"` // 停止种子种子前的最长种子时间（秒） + Name string `json:"name"` // + NumComplete int `json:"num_complete"` // + NumIncomplete int `json:"num_incomplete"` // + NumLeechs int `json:"num_leechs"` // 连接到的 leechers 的数量 + NumSeeds int `json:"num_seeds"` // 连接到的种子数 + Priority int `json:"priority"` // 速度优先。如果队列被禁用或 torrent 处于种子模式，则返回 -1 + Progress float64 `json:"progress"` // 进度 + Ratio float64 `json:"ratio"` // Torrent 共享比率 + RatioLimit int `json:"ratio_limit"` // + SavePath string `json:"save_path"` + SeedingTime int `json:"seeding_time"` // Torrent 完成用时（秒） + SeedingTimeLimit int `json:"seeding_time_limit"` // max_seeding_time + SeenComplete int `json:"seen_complete"` // 上次 torrent 完成的时间 + SeqDl bool `json:"seq_dl"` // 如果启用顺序下载，则为true + Size int64 `json:"size"` // + State TorrentStatus `json:"state"` // 参见https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1)#get-torrent-list + SuperSeeding bool `json:"super_seeding"` // 如果启用超级播种，则为true + Tags string `json:"tags"` // Torrent 的逗号连接标签列表 + TimeActive int `json:"time_active"` // 总活动时间（秒） + TotalSize int64 `json:"total_size"` // 此 torrent 中所有文件的总大小（字节）（包括未选择的文件） + Tracker string `json:"tracker"` // 第一个具有工作状态的tracker。如果没有tracker在工作，则返回空字符串。 + TrackersCount int `json:"trackers_count"` // + UpLimit int `json:"up_limit"` // 上传限制 + Uploaded int64 `json:"uploaded"` // 累计上传 + UploadedSession int64 `json:"uploaded_session"` // 当前session累计上传 + Upspeed int `json:"upspeed"` // 上传速度（字节/秒） +} + +type InfoNotFoundError struct { + Id string + Err error +} + +func (i InfoNotFoundError) Error() string { + return "there should be exactly one task with tag \"alist-" + i.Id + "\"" +} + +func NewInfoNotFoundError(id string) InfoNotFoundError { + return InfoNotFoundError{Id: id} +} + +func (c *client) GetInfo(id string) (TorrentInfo, error) { + var infos []TorrentInfo + + err := c.checkAuthorization() + if err != nil { + return TorrentInfo{}, err + } + + v := url.Values{} + v.Set("tag", "alist-"+id) + response, err := c.post("/api/v2/torrents/info", v) + if err != nil { + return TorrentInfo{}, err + } + + body, err := io.ReadAll(response.Body) + if err != nil { + return TorrentInfo{}, err + } + err = utils.Json.Unmarshal(body, &infos) + if err != nil { + return TorrentInfo{}, err + } + if len(infos) != 1 { + return TorrentInfo{}, NewInfoNotFoundError(id) + } + return infos[0], nil +} + +type FileInfo struct { + Index int `json:"index"` + Name string `json:"name"` + Size int64 `json:"size"` + Progress float32 `json:"progress"` + Priority int `json:"priority"` + IsSeed bool `json:"is_seed"` + PieceRange []int `json:"piece_range"` + Availability float32 `json:"availability"` +} + +func (c *client) GetFiles(id string) ([]FileInfo, error) { + var infos []FileInfo + + err := c.checkAuthorization() + if err != nil { + return []FileInfo{}, err + } + + tInfo, err := c.GetInfo(id) + if err != nil { + return []FileInfo{}, err + } + + v := url.Values{} + v.Set("hash", tInfo.Hash) + response, err := c.post("/api/v2/torrents/files", v) + if err != nil { + return []FileInfo{}, err + } + + body, err := io.ReadAll(response.Body) + if err != nil { + return []FileInfo{}, err + } + err = utils.Json.Unmarshal(body, &infos) + if err != nil { + return []FileInfo{}, err + } + return infos, nil +} + +func (c *client) Delete(id string, deleteFiles bool) error { + err := c.checkAuthorization() + if err != nil { + return err + } + + info, err := c.GetInfo(id) + if err != nil { + return err + } + v := url.Values{} + v.Set("hashes", info.Hash) + if deleteFiles { + v.Set("deleteFiles", "true") + } else { + v.Set("deleteFiles", "false") + } + response, err := c.post("/api/v2/torrents/delete", v) + if err != nil { + return err + } + if response.StatusCode != 200 { + return errors.New("failed to delete qbittorrent task") + } + + v = url.Values{} + v.Set("tags", "alist-"+id) + response, err = c.post("/api/v2/torrents/deleteTags", v) + if err != nil { + return err + } + if response.StatusCode != 200 { + return errors.New("failed to delete qbittorrent tag") + } + return nil +} diff --git a/pkg/task/task.go b/pkg/task/task.go index f47eb7472ce..5b634f10cdb 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -26,7 +26,7 @@ type Task[K comparable] struct { Name string state string // pending, running, finished, canceling, canceled, errored status string - progress int + progress float64 Error error @@ -41,11 +41,11 @@ func (t *Task[K]) SetStatus(status string) { t.status = status } -func (t *Task[K]) SetProgress(percentage int) { +func (t *Task[K]) SetProgress(percentage float64) { t.progress = percentage } -func (t Task[K]) GetProgress() int { +func (t Task[K]) GetProgress() float64 { return t.progress } diff --git a/pkg/utils/io.go b/pkg/utils/io.go index d106531bd3d..6852e28a83d 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -5,10 +5,11 @@ import ( "context" "errors" "fmt" - "golang.org/x/exp/constraints" "io" "time" + "golang.org/x/exp/constraints" + log "github.com/sirupsen/logrus" ) @@ -21,7 +22,7 @@ func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) } // CopyWithCtx slightly modified function signature: // - context has been added in order to propagate cancellation // - I do not return the number of bytes written, has it is not useful in my use case -func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage int)) error { +func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage float64)) error { // Copy will call the Reader and Writer interface multiple time, in order // to copy by chunk (avoiding loading the whole file in memory). // I insert the ability to cancel before read time as it is the earliest @@ -40,7 +41,7 @@ func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, p n, err := in.Read(p) if s > 0 && (err == nil || err == io.EOF) { finish += int64(n) - progress(int(finish / s)) + progress(float64(finish) / float64(s)) } return n, err } diff --git a/server/handles/aria2.go b/server/handles/aria2.go deleted file mode 100644 index 325367a796a..00000000000 --- a/server/handles/aria2.go +++ /dev/null @@ -1,80 +0,0 @@ -package handles - -import ( - "github.com/alist-org/alist/v3/internal/aria2" - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/server/common" - "github.com/gin-gonic/gin" -) - -type SetAria2Req struct { - Uri string `json:"uri" form:"uri"` - Secret string `json:"secret" form:"secret"` -} - -func SetAria2(c *gin.Context) { - var req SetAria2Req - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - items := []model.SettingItem{ - {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - version, err := aria2.InitClient(2) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - common.SuccessResp(c, version) -} - -type AddAria2Req struct { - Urls []string `json:"urls"` - Path string `json:"path"` -} - -func AddAria2(c *gin.Context) { - user := c.MustGet("user").(*model.User) - if !user.CanAddAria2Tasks() { - common.ErrorStrResp(c, "permission denied", 403) - return - } - if !aria2.IsAria2Ready() { - // try to init client - _, err := aria2.InitClient(2) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if !aria2.IsAria2Ready() { - common.ErrorStrResp(c, "aria2 still not ready after init", 500) - return - } - } - var req AddAria2Req - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - reqPath, err := user.JoinPath(req.Path) - if err != nil { - common.ErrorResp(c, err, 403) - return - } - for _, url := range req.Urls { - err := aria2.AddURI(c, url, reqPath) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - } - common.SuccessResp(c) -} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go new file mode 100644 index 00000000000..cf9c1775bae --- /dev/null +++ b/server/handles/offline_download.go @@ -0,0 +1,111 @@ +package handles + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +type SetAria2Req struct { + Uri string `json:"uri" form:"uri"` + Secret string `json:"secret" form:"secret"` +} + +func SetAria2(c *gin.Context) { + var req SetAria2Req + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + items := []model.SettingItem{ + {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("aria2") + version, err := _tool.Init() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, version) +} + +type SetQbittorrentReq struct { + Url string `json:"url" form:"url"` + Seedtime string `json:"seedtime" form:"seedtime"` +} + +func SetQbittorrent(c *gin.Context) { + var req SetQbittorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + items := []model.SettingItem{ + {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("qBittorrent") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +func OfflineDownloadTools(c *gin.Context) { + tools := tool.Tools.Names() + common.SuccessResp(c, tools) +} + +type AddOfflineDownloadReq struct { + Urls []string `json:"urls"` + Path string `json:"path"` + Tool string `json:"tool"` +} + +func AddOfflineDownload(c *gin.Context) { + user := c.MustGet("user").(*model.User) + if !user.CanAddOfflineDownloadTasks() { + common.ErrorStrResp(c, "permission denied", 403) + return + } + + var req AddOfflineDownloadReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + for _, url := range req.Urls { + err := tool.AddURL(c, &tool.AddURLArgs{ + URL: url, + DstDirPath: reqPath, + Tool: req.Tool, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + } + common.SuccessResp(c) +} diff --git a/server/handles/qbittorrent.go b/server/handles/qbittorrent.go deleted file mode 100644 index b22804546ef..00000000000 --- a/server/handles/qbittorrent.go +++ /dev/null @@ -1,79 +0,0 @@ -package handles - -import ( - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/qbittorrent" - "github.com/alist-org/alist/v3/server/common" - "github.com/gin-gonic/gin" -) - -type SetQbittorrentReq struct { - Url string `json:"url" form:"url"` - Seedtime string `json:"seedtime" form:"seedtime"` -} - -func SetQbittorrent(c *gin.Context) { - var req SetQbittorrentReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - items := []model.SettingItem{ - {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, - } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - if err := qbittorrent.InitClient(); err != nil { - common.ErrorResp(c, err, 500) - return - } - common.SuccessResp(c, "ok") -} - -type AddQbittorrentReq struct { - Urls []string `json:"urls"` - Path string `json:"path"` -} - -func AddQbittorrent(c *gin.Context) { - user := c.MustGet("user").(*model.User) - if !user.CanAddQbittorrentTasks() { - common.ErrorStrResp(c, "permission denied", 403) - return - } - if !qbittorrent.IsQbittorrentReady() { - // try to init client - err := qbittorrent.InitClient() - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if !qbittorrent.IsQbittorrentReady() { - common.ErrorStrResp(c, "qbittorrent still not ready after init", 500) - return - } - } - var req AddQbittorrentReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - reqPath, err := user.JoinPath(req.Path) - if err != nil { - common.ErrorResp(c, err, 403) - return - } - for _, url := range req.Urls { - err := qbittorrent.AddURL(c, url, reqPath) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - } - common.SuccessResp(c) -} diff --git a/server/handles/task.go b/server/handles/task.go index d76bb586e27..15e8067248a 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -3,8 +3,8 @@ package handles import ( "strconv" - "github.com/alist-org/alist/v3/internal/aria2" "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/qbittorrent" "github.com/alist-org/alist/v3/pkg/task" "github.com/alist-org/alist/v3/server/common" @@ -12,12 +12,12 @@ import ( ) type TaskInfo struct { - ID string `json:"id"` - Name string `json:"name"` - State string `json:"state"` - Status string `json:"status"` - Progress int `json:"progress"` - Error string `json:"error"` + ID string `json:"id"` + Name string `json:"name"` + State string `json:"state"` + Status string `json:"status"` + Progress float64 `json:"progress"` + Error string `json:"error"` } type K2Str[K comparable] func(k K) string @@ -116,10 +116,12 @@ func taskRoute[K comparable](g *gin.RouterGroup, manager *task.Manager[K], k2Str } func SetupTaskRoute(g *gin.RouterGroup) { - taskRoute(g.Group("/aria2_down"), aria2.DownTaskManager, strK2Str, str2StrK) - taskRoute(g.Group("/aria2_transfer"), aria2.TransferTaskManager, uint64K2Str, str2Uint64K) taskRoute(g.Group("/upload"), fs.UploadTaskManager, uint64K2Str, str2Uint64K) taskRoute(g.Group("/copy"), fs.CopyTaskManager, uint64K2Str, str2Uint64K) taskRoute(g.Group("/qbit_down"), qbittorrent.DownTaskManager, strK2Str, str2StrK) taskRoute(g.Group("/qbit_transfer"), qbittorrent.TransferTaskManager, uint64K2Str, str2Uint64K) + //taskRoute(g.Group("/aria2_down"), aria2.DownTaskManager, strK2Str, str2StrK) + //taskRoute(g.Group("/aria2_transfer"), aria2.TransferTaskManager, uint64K2Str, str2Uint64K) + taskRoute(g.Group("/offline_download"), tool.DownTaskManager, strK2Str, str2StrK) + taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager, uint64K2Str, str2Uint64K) } diff --git a/server/router.go b/server/router.go index 92ede88bfde..7f179231ecd 100644 --- a/server/router.go +++ b/server/router.go @@ -70,6 +70,7 @@ func Init(e *gin.Engine) { // no need auth public := api.Group("/public") public.Any("/settings", handles.PublicSettings) + public.Any("/offline_download_tools", handles.OfflineDownloadTools) _fs(auth.Group("/fs")) admin(auth.Group("/admin", middlewares.AuthAdmin)) @@ -155,8 +156,9 @@ func _fs(g *gin.RouterGroup) { g.PUT("/put", middlewares.FsUp, handles.FsStream) g.PUT("/form", middlewares.FsUp, handles.FsForm) g.POST("/link", middlewares.AuthAdmin, handles.Link) - g.POST("/add_aria2", handles.AddAria2) - g.POST("/add_qbit", handles.AddQbittorrent) + //g.POST("/add_aria2", handles.AddOfflineDownload) + //g.POST("/add_qbit", handles.AddQbittorrent) + g.POST("/add_offline_download", handles.AddOfflineDownload) } func Cors(r *gin.Engine) { From da1c7a4c230f731e0b2ce6a1b316cf779fb57700 Mon Sep 17 00:00:00 2001 From: sheltonzhu <498220739@qq.com> Date: Mon, 6 Nov 2023 16:58:57 +0800 Subject: [PATCH 012/659] feat: add `115_share` driver (#5481 close #5384) This update introduces the ability to mount 115 share links. Currently, only listing and downloading are supported. Note that login and share link are required for this feature to work. Close #5384 --- drivers/115_share/driver.go | 112 ++++++++++++++++++++++++++++++++++++ drivers/115_share/meta.go | 33 +++++++++++ drivers/115_share/utils.go | 111 +++++++++++++++++++++++++++++++++++ drivers/all.go | 1 + go.mod | 2 +- go.sum | 98 +------------------------------ 6 files changed, 261 insertions(+), 96 deletions(-) create mode 100644 drivers/115_share/driver.go create mode 100644 drivers/115_share/meta.go create mode 100644 drivers/115_share/utils.go diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go new file mode 100644 index 00000000000..886a369c1b8 --- /dev/null +++ b/drivers/115_share/driver.go @@ -0,0 +1,112 @@ +package _115_share + +import ( + "context" + + driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "golang.org/x/time/rate" +) + +type Pan115Share struct { + model.Storage + Addition + client *driver115.Pan115Client + limiter *rate.Limiter +} + +func (d *Pan115Share) Config() driver.Config { + return config +} + +func (d *Pan115Share) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Pan115Share) Init(ctx context.Context) error { + if d.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) + } + + return d.login() +} + +func (d *Pan115Share) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + +func (d *Pan115Share) Drop(ctx context.Context) error { + return nil +} + +func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + + files := make([]driver115.ShareFile, 0) + fileResp, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + if err != nil { + return nil, err + } + files = append(files, fileResp.Data.List...) + total := fileResp.Data.Count + count := len(fileResp.Data.List) + for total > count { + fileResp, err := d.client.GetShareSnap( + d.ShareCode, d.ReceiveCode, dir.GetID(), + driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count), + ) + if err != nil { + return nil, err + } + files = append(files, fileResp.Data.List...) + count += len(fileResp.Data.List) + } + + return utils.SliceConvert(files, transFunc) +} + +func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + downloadInfo, err := d.client.DownloadByShareCode(d.ShareCode, d.ReceiveCode, file.GetID()) + if err != nil { + return nil, err + } + + return &model.Link{URL: downloadInfo.URL.URL}, nil +} + +func (d *Pan115Share) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return errs.NotSupport +} + +func (d *Pan115Share) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *Pan115Share) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return errs.NotSupport +} + +func (d *Pan115Share) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *Pan115Share) Remove(ctx context.Context, obj model.Obj) error { + return errs.NotSupport +} + +func (d *Pan115Share) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + return errs.NotSupport +} + +var _ driver.Driver = (*Pan115Share)(nil) diff --git a/drivers/115_share/meta.go b/drivers/115_share/meta.go new file mode 100644 index 00000000000..b7f060e3d9f --- /dev/null +++ b/drivers/115_share/meta.go @@ -0,0 +1,33 @@ +package _115_share + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` + QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` + PageSize int64 `json:"page_size" type:"number" default:"20" help:"list api per page size of 115 driver"` + LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` + ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"` + ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"` + driver.RootID +} + +var config = driver.Config{ + Name: "115 Share", + DefaultRoot: "", + // OnlyProxy: true, + // OnlyLocal: true, + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, + NoUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Pan115Share{} + }) +} diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go new file mode 100644 index 00000000000..42567c0eede --- /dev/null +++ b/drivers/115_share/utils.go @@ -0,0 +1,111 @@ +package _115_share + +import ( + "fmt" + "strconv" + "time" + + driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" +) + +var _ model.Obj = (*FileObj)(nil) + +type FileObj struct { + Size int64 + Sha1 string + Utm time.Time + FileName string + isDir bool + FileID string +} + +func (f *FileObj) CreateTime() time.Time { + return f.Utm +} + +func (f *FileObj) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.SHA1, f.Sha1) +} + +func (f *FileObj) GetSize() int64 { + return f.Size +} + +func (f *FileObj) GetName() string { + return f.FileName +} + +func (f *FileObj) ModTime() time.Time { + return f.Utm +} + +func (f *FileObj) IsDir() bool { + return f.isDir +} + +func (f *FileObj) GetID() string { + return f.FileID +} + +func (f *FileObj) GetPath() string { + return "" +} + +func transFunc(sf driver115.ShareFile) (model.Obj, error) { + timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) + if err != nil { + return nil, err + } + var ( + utm = time.Unix(timeInt, 0) + isDir = (sf.IsFile == 0) + fileID = string(sf.FileID) + ) + if isDir { + fileID = string(sf.CategoryID) + } + return &FileObj{ + Size: int64(sf.Size), + Sha1: sf.Sha1, + Utm: utm, + FileName: string(sf.FileName), + isDir: isDir, + FileID: fileID, + }, nil +} + +var UserAgent = driver115.UA115Browser + +func (d *Pan115Share) login() error { + var err error + opts := []driver115.Option{ + driver115.UA(UserAgent), + } + d.client = driver115.New(opts...) + if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { + return errors.Wrap(err, "failed to get share snap") + } + cr := &driver115.Credential{} + if d.QRCodeToken != "" { + s := &driver115.QRCodeSession{ + UID: d.QRCodeToken, + } + if cr, err = d.client.QRCodeLogin(s); err != nil { + return errors.Wrap(err, "failed to login by qrcode") + } + d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s", cr.UID, cr.CID, cr.SEID) + d.QRCodeToken = "" + } else if d.Cookie != "" { + if err = cr.FromCookie(d.Cookie); err != nil { + return errors.Wrap(err, "failed to login by cookies") + } + d.client.ImportCredential(cr) + } else { + return errors.New("missing cookie or qrcode account") + } + + return d.client.LoginCheck() +} diff --git a/drivers/all.go b/drivers/all.go index 921a467620d..4f7fa839be0 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -2,6 +2,7 @@ package drivers import ( _ "github.com/alist-org/alist/v3/drivers/115" + _ "github.com/alist-org/alist/v3/drivers/115_share" _ "github.com/alist-org/alist/v3/drivers/123" _ "github.com/alist-org/alist/v3/drivers/123_link" _ "github.com/alist-org/alist/v3/drivers/123_share" diff --git a/go.mod b/go.mod index 70b8383d7ab..c8b4356db53 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/alist-org/alist/v3 go 1.20 require ( - github.com/SheltonZhu/115driver v1.0.16 + github.com/SheltonZhu/115driver v1.0.21 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/Xhofe/wopan-sdk-go v0.1.2 diff --git a/go.sum b/go.sum index 6ea29399734..502c3bfd908 100644 --- a/go.sum +++ b/go.sum @@ -8,31 +8,21 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= -github.com/SheltonZhu/115driver v1.0.15 h1:RRvgXvXEzvrPwkRno0CUIg7ucEphbsfwct2mQxfNOdQ= -github.com/SheltonZhu/115driver v1.0.15/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= -github.com/SheltonZhu/115driver v1.0.16 h1:XOhqRtKF9huTCobM5rWVd9DtJyBKLJSnBwWPF3+GM+k= -github.com/SheltonZhu/115driver v1.0.16/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= +github.com/SheltonZhu/115driver v1.0.21 h1:Pz6r14VwIiuSyHj+OmJe57FHhbmWB/6IfnXAFL2iXbU= +github.com/SheltonZhu/115driver v1.0.21/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaVLk4gI/Co0paF92FmxSR6U1zY= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY= -github.com/Xhofe/wopan-sdk-go v0.1.1 h1:dSrTxNYclqNuo9libjtC+R6C4RCen/inh/dUXd12vpM= -github.com/Xhofe/wopan-sdk-go v0.1.1/go.mod h1:xWcUS7PoFLDD9gy2BK2VQfilEsZngLMz2Vkx3oF2zJY= github.com/Xhofe/wopan-sdk-go v0.1.2 h1:6Gh4YTT7b7YHN0OoJ33j7Jm9ru/ckuvcDxPnRmH07jc= github.com/Xhofe/wopan-sdk-go v0.1.2/go.mod h1:ktLYb4t7rnPFq1AshLaPXq5kZER+DkEagT6/i/in0uo= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= -github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1eXdZyMw3F7uGA9qPn2J4+R8= -github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= -github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/andreburgaud/crypt2go v1.1.0 h1:eitZxTPY1krUsxinsng3Qvt/Ud7q/aQmmYRh8p4hyPw= -github.com/andreburgaud/crypt2go v1.1.0/go.mod h1:4qhZPzarj1dCIRmCkpdgCklwp+hBq9yEt0zPe9Ayuhc= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/Hcta4l4VrIXrI= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= @@ -50,16 +40,10 @@ github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjL github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/blevesearch/bleve/v2 v2.3.9 h1:pUMvK0mxAexqasZcVj8lazmWnEW5XiV0tASIqANiNTQ= -github.com/blevesearch/bleve/v2 v2.3.9/go.mod h1:1PibElcjlQMQHF9uS9mRv58ODQgj4pCWHA1Wfd+qagU= github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= -github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw= -github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= -github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA= -github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= @@ -68,8 +52,6 @@ github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZG github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.1.5 h1:1g713kpCQZ8u4a3stRGBfrwVOuGRnmxOVU5MQkUPrHU= -github.com/blevesearch/scorch_segment_api/v2 v2.1.5/go.mod h1:f2nOkKS1HcjgIWZgDAErgBdxmr2eyt0Kn7IY+FU1Xe4= github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= @@ -80,24 +62,14 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= -github.com/blevesearch/zapx/v11 v11.3.9 h1:y3ijS4h4MJdmQ07MHASxat4owAixreK2xdo76w9ncrw= -github.com/blevesearch/zapx/v11 v11.3.9/go.mod h1:jcAYnQwlr+LqD2vLjDWjWiZDXDXGFqPbpPDRTd3XmS4= github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= -github.com/blevesearch/zapx/v12 v12.3.9 h1:MXGLlZ03oxXH3DMJTZaBaRj2xb6t4wQVZeZK/wu1M6w= -github.com/blevesearch/zapx/v12 v12.3.9/go.mod h1:QXCMwmOkdLnMDgTN1P4CcuX5F851iUOtOwXbw0HMBYs= github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= -github.com/blevesearch/zapx/v13 v13.3.9 h1:+VAz9V0VmllHXlZV4DCvfYj0nqaZHgF3MeEHwOyRBwQ= -github.com/blevesearch/zapx/v13 v13.3.9/go.mod h1:s+WjNp4WSDtrBVBpa37DUOd7S/Gr/jTZ7ST/MbCVj/0= github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= -github.com/blevesearch/zapx/v14 v14.3.9 h1:wuqxATgsTCNHM9xsOFOeFp8H2heZ/gMX/tsl9lRK8U4= -github.com/blevesearch/zapx/v14 v14.3.9/go.mod h1:MWZ4v8AzFBRurhDzkLvokFW8ljcq9Evm27mkWe8OGbM= github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= -github.com/blevesearch/zapx/v15 v15.3.12 h1:w/kU9aHyfMDEdwHGZzCiakC3HZ9z5gYlXaALDC4Dct8= -github.com/blevesearch/zapx/v15 v15.3.12/go.mod h1:tx53gDJS/7Oa3Je820cmVurqCuJ4dqdAy1kiDMV/IUo= github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -115,8 +87,6 @@ github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5 github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= -github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= -github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= @@ -184,10 +154,7 @@ github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXS github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= -github.com/go-resty/resty/v2 v2.8.0 h1:J29d0JFWwSWrDCysnOK/YjsPMLQTx0TvgJEHVGvf2L8= -github.com/go-resty/resty/v2 v2.8.0/go.mod h1:UCui0cMHekLrSntoMyofdSTaPpinlRHFtPpizuyDW2w= github.com/go-resty/resty/v2 v2.9.1 h1:PIgGx4VrHvag0juCJ4dDv3MiFRlDmP0vicBucwf+gLM= github.com/go-resty/resty/v2 v2.9.1/go.mod h1:4/GYJVjh9nhkhGR6AUNW3XhpDYNUr+Uvy9gV/VGZIy4= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= @@ -207,7 +174,6 @@ github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVI github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -224,8 +190,6 @@ github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= @@ -242,14 +206,10 @@ github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRK github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs= -github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= -github.com/ipfs/go-ipfs-api v0.6.1 h1:nK5oeFOdMh1ogT+GCOcyBFOOcFGNuudSb1rg9YDyAKE= -github.com/ipfs/go-ipfs-api v0.6.1/go.mod h1:8pl+ZMF2LX42szbqGbpOBEiI1/rYaImvTvJtG0g+rL4= github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2Q= github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -270,11 +230,9 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= @@ -318,8 +276,6 @@ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= @@ -349,8 +305,6 @@ github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELU github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= -github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= @@ -361,41 +315,30 @@ github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sgg github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= -github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= -github.com/orzogc/fake115uploader v0.3.3-0.20221009101310-08b764073b77 h1:dg/EaaJLPIg4xn2kaZil7Ax3wfoxcFXaBwyOTlcz5AI= -github.com/orzogc/fake115uploader v0.3.3-0.20221009101310-08b764073b77/go.mod h1:FD9a09Vw07CSMTdT0Y7ttStOa1WZsnPBslliMw2DkeM= github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu4h5aYIOzUtLjN08L4Qt4WGaJONMgcaD0ayBJQ= github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 h1:5TvW1dv00Y13njmQ1AWkxSWtPkwE7ZEF6yDuv9q+Als= -github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= @@ -473,8 +416,6 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/u2takey/ffmpeg-go v0.4.1 h1:l5ClIwL3N2LaH1zF3xivb3kP2HW95eyG5xhHE1JdZ9Y= -github.com/u2takey/ffmpeg-go v0.4.1/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys= @@ -485,7 +426,7 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= -github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= +github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -510,25 +451,12 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= -golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= -golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -544,18 +472,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= -golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= -golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -587,9 +506,6 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -598,11 +514,6 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= @@ -614,9 +525,6 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= From 4355dae491ba251ce99826ad8833a47ed8cbdad5 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 6 Nov 2023 18:20:25 +0800 Subject: [PATCH 013/659] fix: incorrect content-type of apk files (close #5385) --- pkg/utils/file.go | 7 +++++++ server/common/proxy.go | 1 + 2 files changed, 8 insertions(+) diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 31803a95b2d..7ae07158998 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -163,8 +163,15 @@ func GetObjType(filename string, isDir bool) int { return GetFileType(filename) } +var extraMimeTypes = map[string]string{ + ".apk": "application/vnd.android.package-archive", +} + func GetMimeType(name string) string { ext := path.Ext(name) + if m, ok := extraMimeTypes[ext]; ok { + return m + } m := mime.TypeByExtension(ext) if m != "" { return m diff --git a/server/common/proxy.go b/server/common/proxy.go index 370e46eb21f..8156dc885ef 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -76,4 +76,5 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. func attachFileName(w http.ResponseWriter, file model.Obj) { fileName := file.GetName() w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, fileName, url.PathEscape(fileName))) + w.Header().Set("Content-Type", utils.GetMimeType(fileName)) } From 91f51f17d063809f49e71a8851281785da898242 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 10 Nov 2023 15:38:23 +0800 Subject: [PATCH 014/659] feat(webdav): add `tls_insecure_skip_verify` field (close #5490) --- drivers/webdav/meta.go | 1 + drivers/webdav/util.go | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/drivers/webdav/meta.go b/drivers/webdav/meta.go index d66499bc3f9..2294d482a6e 100644 --- a/drivers/webdav/meta.go +++ b/drivers/webdav/meta.go @@ -11,6 +11,7 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootPath + TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false"` } var config = driver.Config{ diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 84eebb2ec3c..23dc909ff88 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -1,6 +1,7 @@ package webdav import ( + "crypto/tls" "net/http" "net/http/cookiejar" @@ -17,6 +18,10 @@ func (d *WebDav) isSharepoint() bool { func (d *WebDav) setClient() error { c := gowebdav.NewClient(d.Address, d.Username, d.Password) + c.SetTransport(&http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: d.TlsInsecureSkipVerify}, + }) if d.isSharepoint() { cookie, err := odrvcookie.GetCookie(d.Username, d.Password, d.Address) if err == nil { From 55a14bc2714458df1508dafe7a0b4a2e474bbd25 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sun, 12 Nov 2023 15:13:55 +0800 Subject: [PATCH 015/659] fix(mopan): 302 Redirect (#5505 close #5502) * fix(mopan):302 Redirect * fix(mopan): do not forget to close the body --------- Co-authored-by: Andy Hsu --- drivers/mopan/driver.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index 78ec0423cc3..f3bb4e74928 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -119,10 +119,13 @@ func (d *MoPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } data.DownloadUrl = strings.Replace(strings.ReplaceAll(data.DownloadUrl, "&", "&"), "http://", "https://", 1) - res, err := base.NoRedirectClient.R().SetContext(ctx).Head(data.DownloadUrl) + res, err := base.NoRedirectClient.R().SetDoNotParseResponse(true).SetContext(ctx).Get(data.DownloadUrl) if err != nil { return nil, err } + defer func() { + _ = res.RawBody().Close() + }() if res.StatusCode() == 302 { data.DownloadUrl = res.Header().Get("location") } From a7421d8fc29b2c8b9e730efebb5d8a0b1f73daa0 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 12 Nov 2023 15:14:27 +0800 Subject: [PATCH 016/659] fix(deps): update module github.com/aws/aws-sdk-go to v1.46.7 (#5068) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c8b4356db53..c6a6bc21592 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Xhofe/wopan-sdk-go v0.1.2 github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.44.327 + github.com/aws/aws-sdk-go v1.46.7 github.com/blevesearch/bleve/v2 v2.3.10 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.16.1 diff --git a/go.sum b/go.sum index 502c3bfd908..5abe5aefd6b 100644 --- a/go.sum +++ b/go.sum @@ -30,6 +30,8 @@ github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY= github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.46.7 h1:IjvAWeiJZlbETOemOwvheN5L17CvKvKW0T1xOC6d3Sc= +github.com/aws/aws-sdk-go v1.46.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= From 3d51845f57d5670893c336c80c561a64f9496f3c Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 13 Nov 2023 15:22:42 +0800 Subject: [PATCH 017/659] feat: invalidate old token after changing the password (close #5515) --- internal/model/user.go | 3 +++ server/common/auth.go | 7 +++++-- server/handles/auth.go | 2 +- server/handles/ssologin.go | 4 ++-- server/handles/webauthn.go | 2 +- server/middlewares/auth.go | 12 ++++++++++++ 6 files changed, 24 insertions(+), 6 deletions(-) diff --git a/internal/model/user.go b/internal/model/user.go index 46fe9bd301d..2d61a971c3d 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "time" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/pkg/utils" @@ -24,6 +25,7 @@ type User struct { ID uint `json:"id" gorm:"primaryKey"` // unique key Username string `json:"username" gorm:"unique" binding:"required"` // username PwdHash string `json:"-"` // password hash + PwdTS int64 `json:"-"` // password timestamp Salt string `json:"-"` // unique salt Password string `json:"password"` // password BasePath string `json:"base_path"` // base path @@ -71,6 +73,7 @@ func (u *User) ValidatePwdStaticHash(pwdStaticHash string) error { func (u *User) SetPassword(pwd string) *User { u.Salt = random.String(16) u.PwdHash = TwoHashPwd(pwd, u.Salt) + u.PwdTS = time.Now().Unix() return u } diff --git a/server/common/auth.go b/server/common/auth.go index 017390bdd7a..b6a79b752aa 100644 --- a/server/common/auth.go +++ b/server/common/auth.go @@ -4,6 +4,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" "github.com/golang-jwt/jwt/v4" "github.com/pkg/errors" ) @@ -12,12 +13,14 @@ var SecretKey []byte type UserClaims struct { Username string `json:"username"` + PwdTS int64 `json:"pwd_ts"` jwt.RegisteredClaims } -func GenerateToken(username string) (tokenString string, err error) { +func GenerateToken(user *model.User) (tokenString string, err error) { claim := UserClaims{ - Username: username, + Username: user.Username, + PwdTS: user.PwdTS, RegisteredClaims: jwt.RegisteredClaims{ ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Duration(conf.Conf.TokenExpiresIn) * time.Hour)), IssuedAt: jwt.NewNumericDate(time.Now()), diff --git a/server/handles/auth.go b/server/handles/auth.go index 37ae736c2ef..209bdd3a2b8 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -78,7 +78,7 @@ func loginHash(c *gin.Context, req *LoginReq) { } } // generate token - token, err := common.GenerateToken(user.Username) + token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 89d8ecaf480..52486b97839 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -260,7 +260,7 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorResp(c, err, 400) } } - token, err := common.GenerateToken(user.Username) + token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) } @@ -426,7 +426,7 @@ func SSOLoginCallback(c *gin.Context) { return } } - token, err := common.GenerateToken(user.Username) + token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) } diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index 952cf480b46..28a89522cfc 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -94,7 +94,7 @@ func FinishAuthnLogin(c *gin.Context) { return } - token, err := common.GenerateToken(user.Username) + token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 14bba5678a1..14f186be8bf 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -57,6 +57,12 @@ func Auth(c *gin.Context) { c.Abort() return } + // validate password timestamp + if userClaims.PwdTS != user.PwdTS { + common.ErrorStrResp(c, "Password has been changed, login please", 401) + c.Abort() + return + } if user.Disabled { common.ErrorStrResp(c, "Current user is disabled, replace please", 401) c.Abort() @@ -105,6 +111,12 @@ func Authn(c *gin.Context) { c.Abort() return } + // validate password timestamp + if userClaims.PwdTS != user.PwdTS { + common.ErrorStrResp(c, "Password has been changed, login please", 401) + c.Abort() + return + } if user.Disabled { common.ErrorStrResp(c, "Current user is disabled, replace please", 401) c.Abort() From f904596cbcfe111c838b81da9f2847a1c4fc4bc8 Mon Sep 17 00:00:00 2001 From: guangwu Date: Thu, 16 Nov 2023 19:16:15 +0800 Subject: [PATCH 018/659] chore: remove refs to deprecated io/ioutil (#5519) Signed-off-by: guoguangwu --- drivers/google_drive/util.go | 5 ++--- internal/net/request_test.go | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 2c1f13eb8a5..0d3801127a4 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -5,7 +5,6 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "io/ioutil" "net/http" "os" "regexp" @@ -44,7 +43,7 @@ func (d *GoogleDrive) refreshToken() error { gdsaFileThis := d.RefreshToken if gdsaFile.IsDir() { if len(d.ServiceAccountFileList) <= 0 { - gdsaReadDir, gdsaDirErr := ioutil.ReadDir(d.RefreshToken) + gdsaReadDir, gdsaDirErr := os.ReadDir(d.RefreshToken) if gdsaDirErr != nil { log.Error("read dir fail") return gdsaDirErr @@ -76,7 +75,7 @@ func (d *GoogleDrive) refreshToken() error { } } - gdsaFileThisContent, err := ioutil.ReadFile(gdsaFileThis) + gdsaFileThisContent, err := os.ReadFile(gdsaFileThis) if err != nil { return err } diff --git a/internal/net/request_test.go b/internal/net/request_test.go index e41971fbf61..032b7376585 100644 --- a/internal/net/request_test.go +++ b/internal/net/request_test.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "sync" "testing" @@ -169,7 +168,7 @@ func newDownloadRangeClient(data []byte) (*downloadCaptureClient, *int, *[]strin header := &http.Header{} header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, fin-1, len(data))) return &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader(bodyBytes)), + Body: io.NopCloser(bytes.NewReader(bodyBytes)), Header: *header, ContentLength: int64(len(bodyBytes)), }, nil From 6fc6751463f907b29c2ff5e7ed5adae343616493 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 18 Nov 2023 19:56:22 +0800 Subject: [PATCH 019/659] feat: support using external dist files (close #5531) --- internal/conf/config.go | 1 + server/static/static.go | 35 +++++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/internal/conf/config.go b/internal/conf/config.go index 67c2dc0fa56..de26e1fe0c4 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -49,6 +49,7 @@ type Config struct { Scheme Scheme `json:"scheme"` TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` + DistDir string `json:"dist_dir"` Log LogConfig `json:"log"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` diff --git a/server/static/static.go b/server/static/static.go index 624637936cd..8e2054af7f9 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -3,25 +3,48 @@ package static import ( "errors" "fmt" + "github.com/alist-org/alist/v3/public" + "io" "io/fs" "net/http" + "os" "strings" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/alist-org/alist/v3/public" "github.com/gin-gonic/gin" ) -func InitIndex() { - index, err := public.Public.ReadFile("dist/index.html") +var static fs.FS = public.Public + +func initStatic() { + if conf.Conf.DistDir == "" { + dist, err := fs.Sub(static, "dist") + if err != nil { + utils.Log.Fatalf("failed to read dist dir") + } + static = dist + return + } + static = os.DirFS(conf.Conf.DistDir) +} + +func initIndex() { + indexFile, err := static.Open("index.html") if err != nil { if errors.Is(err, fs.ErrNotExist) { utils.Log.Fatalf("index.html not exist, you may forget to put dist of frontend to public/dist") } utils.Log.Fatalf("failed to read index.html: %v", err) } + defer func() { + _ = indexFile.Close() + }() + index, err := io.ReadAll(indexFile) + if err != nil { + utils.Log.Fatalf("failed to read dist/index.html") + } conf.RawIndexHtml = string(index) siteConfig := getSiteConfig() replaceMap := map[string]string{ @@ -60,7 +83,8 @@ func UpdateIndex() { } func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { - InitIndex() + initStatic() + initIndex() folders := []string{"assets", "images", "streamer", "static"} r.Use(func(c *gin.Context) { for i := range folders { @@ -70,8 +94,7 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { } }) for i, folder := range folders { - folder = "dist/" + folder - sub, err := fs.Sub(public.Public, folder) + sub, err := fs.Sub(static, folder) if err != nil { utils.Log.Fatalf("can't find folder: %s", folder) } From 867accafd1dd6e30f7fc6339c3a47ba9f3821ec4 Mon Sep 17 00:00:00 2001 From: MuGu <94156510@qq.com> Date: Sat, 18 Nov 2023 22:36:41 +0800 Subject: [PATCH 020/659] fix(local): video file thumbnails not displaying on iOS Safari (#5420) * perf(webdav): support for cookies on webdav drive * fix(local): video file thumbnails not displaying on iOS Safari --- server/common/proxy.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/common/proxy.go b/server/common/proxy.go index 8156dc885ef..a4f04abfe2c 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -17,6 +17,10 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. if link.MFile != nil { defer link.MFile.Close() attachFileName(w, file) + contentType := link.Header.Get("Content-Type") + if contentType != "" { + w.Header().Add("Content-Type", contentType) + } http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) return nil } else if link.RangeReadCloser != nil { From 8d5283604c12d4a8405a8d5720b0e6f2ac5e2631 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 19 Nov 2023 15:21:25 +0800 Subject: [PATCH 021/659] ci: add short sha to artifact --- .github/workflows/build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a8f41be434..70fe145c018 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,9 @@ jobs: - name: Checkout uses: actions/checkout@v3 + - uses: benjlevesque/short-sha@v2.2 + id: short-sha + - name: Install dependencies run: | sudo snap install zig --classic --beta @@ -41,5 +44,5 @@ jobs: - name: Upload artifact uses: actions/upload-artifact@v3 with: - name: alist + name: alist_${{ env.SHA }} path: dist \ No newline at end of file From de9647a5faae8882214eff53599e9b0468df69fa Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 19 Nov 2023 20:05:09 +0800 Subject: [PATCH 022/659] chore: remove useless code --- internal/aria2/add.go | 61 ----- internal/aria2/aria2.go | 42 --- internal/aria2/aria2_test.go | 87 ------ internal/aria2/monitor.go | 192 ------------- internal/aria2/notify.go | 70 ----- internal/offline_download/qbit/qbit.go | 2 +- internal/qbittorrent/add.go | 60 ---- internal/qbittorrent/client.go | 366 ------------------------- internal/qbittorrent/client_test.go | 154 ----------- internal/qbittorrent/monitor.go | 181 ------------ internal/qbittorrent/qbittorrent.go | 23 -- server/handles/task.go | 5 - 12 files changed, 1 insertion(+), 1242 deletions(-) delete mode 100644 internal/aria2/add.go delete mode 100644 internal/aria2/aria2.go delete mode 100644 internal/aria2/aria2_test.go delete mode 100644 internal/aria2/monitor.go delete mode 100644 internal/aria2/notify.go delete mode 100644 internal/qbittorrent/add.go delete mode 100644 internal/qbittorrent/client.go delete mode 100644 internal/qbittorrent/client_test.go delete mode 100644 internal/qbittorrent/monitor.go delete mode 100644 internal/qbittorrent/qbittorrent.go diff --git a/internal/aria2/add.go b/internal/aria2/add.go deleted file mode 100644 index 4eb83f3dd35..00000000000 --- a/internal/aria2/add.go +++ /dev/null @@ -1,61 +0,0 @@ -package aria2 - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/google/uuid" - "github.com/pkg/errors" -) - -func AddURI(ctx context.Context, uri string, dstDirPath string) error { - // check storage - storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - // check is it could upload - if storage.Config().NoUpload { - return errors.WithStack(errs.UploadNotSupported) - } - // check path is valid - obj, err := op.Get(ctx, storage, dstDirActualPath) - if err != nil { - if !errs.IsObjectNotFound(err) { - return errors.WithMessage(err, "failed get object") - } - } else { - if !obj.IsDir() { - // can't add to a file - return errors.WithStack(errs.NotFolder) - } - } - // call aria2 rpc - tempDir := filepath.Join(conf.Conf.TempDir, "aria2", uuid.NewString()) - options := map[string]interface{}{ - "dir": tempDir, - } - gid, err := client.AddURI([]string{uri}, options) - if err != nil { - return errors.Wrapf(err, "failed to add uri %s", uri) - } - DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ - ID: gid, - Name: fmt.Sprintf("download %s to [%s](%s)", uri, storage.GetStorage().MountPath, dstDirActualPath), - Func: func(tsk *task.Task[string]) error { - m := &Monitor{ - tsk: tsk, - tempDir: tempDir, - retried: 0, - dstDirPath: dstDirPath, - } - return m.Loop() - }, - })) - return nil -} diff --git a/internal/aria2/aria2.go b/internal/aria2/aria2.go deleted file mode 100644 index 7250afabc16..00000000000 --- a/internal/aria2/aria2.go +++ /dev/null @@ -1,42 +0,0 @@ -package aria2 - -import ( - "context" - "time" - - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/pkg/aria2/rpc" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -var DownTaskManager = task.NewTaskManager[string](3) -var notify = NewNotify() -var client rpc.Client - -func InitClient(timeout int) (string, error) { - client = nil - uri := setting.GetStr(conf.Aria2Uri) - secret := setting.GetStr(conf.Aria2Secret) - return InitAria2Client(uri, secret, timeout) -} - -func InitAria2Client(uri string, secret string, timeout int) (string, error) { - c, err := rpc.New(context.Background(), uri, secret, time.Duration(timeout)*time.Second, notify) - if err != nil { - return "", errors.Wrap(err, "failed to init aria2 client") - } - version, err := c.GetVersion() - if err != nil { - return "", errors.Wrapf(err, "failed get aria2 version") - } - client = c - log.Infof("using aria2 version: %s", version.Version) - return version.Version, nil -} - -func IsAria2Ready() bool { - return client != nil -} diff --git a/internal/aria2/aria2_test.go b/internal/aria2/aria2_test.go deleted file mode 100644 index 1e1b296b24a..00000000000 --- a/internal/aria2/aria2_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package aria2 - -import ( - "context" - "path/filepath" - "testing" - "time" - - _ "github.com/alist-org/alist/v3/drivers" - conf2 "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/db" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" - "gorm.io/driver/sqlite" - "gorm.io/gorm" -) - -func init() { - conf2.Conf = conf2.DefaultConfig() - absPath, err := filepath.Abs("../../data/temp") - if err != nil { - panic(err) - } - conf2.Conf.TempDir = absPath - dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) - if err != nil { - panic("failed to connect database") - } - db.Init(dB) -} - -func TestConnect(t *testing.T) { - _, err := InitAria2Client("http://localhost:16800/jsonrpc", "secret", 3) - if err != nil { - t.Errorf("failed to init aria2: %+v", err) - } -} - -func TestDown(t *testing.T) { - TestConnect(t) - _, err := op.CreateStorage(context.Background(), model.Storage{ - ID: 0, - MountPath: "/", - Order: 0, - Driver: "Local", - Status: "", - Addition: `{"root_folder":"../../data"}`, - Remark: "", - }) - if err != nil { - t.Fatalf("failed to create storage: %+v", err) - } - err = AddURI(context.Background(), "https://nodejs.org/dist/index.json", "/test") - if err != nil { - t.Errorf("failed to add uri: %+v", err) - } - tasks := DownTaskManager.GetAll() - if len(tasks) != 1 { - t.Errorf("failed to get tasks: %+v", tasks) - } - for { - tsk := tasks[0] - t.Logf("task: %+v", tsk) - if tsk.GetState() == task.SUCCEEDED { - break - } - if tsk.GetState() == task.ERRORED { - t.Fatalf("failed to download: %+v", tsk) - } - time.Sleep(time.Second) - } - for { - if len(TransferTaskManager.GetAll()) == 0 { - continue - } - tsk := TransferTaskManager.GetAll()[0] - t.Logf("task: %+v", tsk) - if tsk.GetState() == task.SUCCEEDED { - break - } - if tsk.GetState() == task.ERRORED { - t.Fatalf("failed to download: %+v", tsk) - } - time.Sleep(time.Second) - } -} diff --git a/internal/aria2/monitor.go b/internal/aria2/monitor.go deleted file mode 100644 index aaef3fd7c70..00000000000 --- a/internal/aria2/monitor.go +++ /dev/null @@ -1,192 +0,0 @@ -package aria2 - -import ( - "fmt" - "os" - "path" - "path/filepath" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/alist-org/alist/v3/internal/stream" - - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -type Monitor struct { - tsk *task.Task[string] - tempDir string - retried int - c chan int - dstDirPath string - finish chan struct{} -} - -func (m *Monitor) Loop() error { - defer func() { - notify.Signals.Delete(m.tsk.ID) - // clear temp dir, should do while complete - //_ = os.RemoveAll(m.tempDir) - }() - m.c = make(chan int) - m.finish = make(chan struct{}) - notify.Signals.Store(m.tsk.ID, m.c) - var ( - err error - ok bool - ) -outer: - for { - select { - case <-m.tsk.Ctx.Done(): - _, err := client.Remove(m.tsk.ID) - return err - case <-m.c: - ok, err = m.Update() - if ok { - break outer - } - case <-time.After(time.Second * 2): - ok, err = m.Update() - if ok { - break outer - } - } - } - if err != nil { - return err - } - m.tsk.SetStatus("aria2 download completed, transferring") - <-m.finish - m.tsk.SetStatus("completed") - return nil -} - -func (m *Monitor) Update() (bool, error) { - info, err := client.TellStatus(m.tsk.ID) - if err != nil { - m.retried++ - log.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) - return false, nil - } - if m.retried > 5 { - return true, errors.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) - } - m.retried = 0 - if len(info.FollowedBy) != 0 { - log.Debugf("followen by: %+v", info.FollowedBy) - gid := info.FollowedBy[0] - notify.Signals.Delete(m.tsk.ID) - oldId := m.tsk.ID - m.tsk.ID = gid - DownTaskManager.RawTasks().Delete(oldId) - DownTaskManager.RawTasks().Store(m.tsk.ID, m.tsk) - notify.Signals.Store(gid, m.c) - return false, nil - } - // update download status - total, err := strconv.ParseUint(info.TotalLength, 10, 64) - if err != nil { - total = 0 - } - downloaded, err := strconv.ParseUint(info.CompletedLength, 10, 64) - if err != nil { - downloaded = 0 - } - progress := float64(downloaded) / float64(total) * 100 - m.tsk.SetProgress(progress) - switch info.Status { - case "complete": - err := m.Complete() - return true, errors.WithMessage(err, "failed to transfer file") - case "error": - return true, errors.Errorf("failed to download %s, error: %s", m.tsk.ID, info.ErrorMessage) - case "active": - m.tsk.SetStatus("aria2: " + info.Status) - if info.Seeder == "true" { - err := m.Complete() - return true, errors.WithMessage(err, "failed to transfer file") - } - return false, nil - case "waiting", "paused": - m.tsk.SetStatus("aria2: " + info.Status) - return false, nil - case "removed": - return true, errors.Errorf("failed to download %s, removed", m.tsk.ID) - default: - return true, errors.Errorf("failed to download %s, unknown status %s", m.tsk.ID, info.Status) - } -} - -var TransferTaskManager = task.NewTaskManager(3, func(k *uint64) { - atomic.AddUint64(k, 1) -}) - -func (m *Monitor) Complete() error { - // check dstDir again - storage, dstDirActualPath, err := op.GetStorageAndActualPath(m.dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - // get files - files, err := client.GetFiles(m.tsk.ID) - log.Debugf("files len: %d", len(files)) - if err != nil { - return errors.Wrapf(err, "failed to get files of %s", m.tsk.ID) - } - // upload files - var wg sync.WaitGroup - wg.Add(len(files)) - go func() { - wg.Wait() - err := os.RemoveAll(m.tempDir) - m.finish <- struct{}{} - if err != nil { - log.Errorf("failed to remove aria2 temp dir: %+v", err.Error()) - } - }() - for i, _ := range files { - file := files[i] - TransferTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, storage.GetStorage().MountPath, dstDirActualPath), - Func: func(tsk *task.Task[uint64]) error { - defer wg.Done() - size, _ := strconv.ParseInt(file.Length, 10, 64) - mimetype := utils.GetMimeType(file.Path) - f, err := os.Open(file.Path) - if err != nil { - return errors.Wrapf(err, "failed to open file %s", file.Path) - } - s := stream.FileStream{ - Obj: &model.Object{ - Name: path.Base(file.Path), - Size: size, - Modified: time.Now(), - IsFolder: false, - }, - Reader: f, - Closers: utils.NewClosers(f), - Mimetype: mimetype, - } - ss, err := stream.NewSeekableStream(s, nil) - if err != nil { - return err - } - relDir, err := filepath.Rel(m.tempDir, filepath.Dir(file.Path)) - if err != nil { - log.Errorf("find relation directory error: %v", err) - } - newDistDir := filepath.Join(dstDirActualPath, relDir) - return op.Put(tsk.Ctx, storage, newDistDir, ss, tsk.SetProgress) - }, - })) - } - return nil -} diff --git a/internal/aria2/notify.go b/internal/aria2/notify.go deleted file mode 100644 index 056fe5147b4..00000000000 --- a/internal/aria2/notify.go +++ /dev/null @@ -1,70 +0,0 @@ -package aria2 - -import ( - "github.com/alist-org/alist/v3/pkg/aria2/rpc" - "github.com/alist-org/alist/v3/pkg/generic_sync" -) - -const ( - Downloading = iota - Paused - Stopped - Completed - Errored -) - -type Notify struct { - Signals generic_sync.MapOf[string, chan int] -} - -func NewNotify() *Notify { - return &Notify{Signals: generic_sync.MapOf[string, chan int]{}} -} - -func (n *Notify) OnDownloadStart(events []rpc.Event) { - for _, e := range events { - if signal, ok := n.Signals.Load(e.Gid); ok { - signal <- Downloading - } - } -} - -func (n *Notify) OnDownloadPause(events []rpc.Event) { - for _, e := range events { - if signal, ok := n.Signals.Load(e.Gid); ok { - signal <- Paused - } - } -} - -func (n *Notify) OnDownloadStop(events []rpc.Event) { - for _, e := range events { - if signal, ok := n.Signals.Load(e.Gid); ok { - signal <- Stopped - } - } -} - -func (n *Notify) OnDownloadComplete(events []rpc.Event) { - for _, e := range events { - if signal, ok := n.Signals.Load(e.Gid); ok { - signal <- Completed - } - } -} - -func (n *Notify) OnDownloadError(events []rpc.Event) { - for _, e := range events { - if signal, ok := n.Signals.Load(e.Gid); ok { - signal <- Errored - } - } -} - -func (n *Notify) OnBtDownloadComplete(events []rpc.Event) { - for _, e := range events { - if signal, ok := n.Signals.Load(e.Gid); ok { - signal <- Completed - } - } -} diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go index 594088f0eb2..388ce22edbd 100644 --- a/internal/offline_download/qbit/qbit.go +++ b/internal/offline_download/qbit/qbit.go @@ -4,8 +4,8 @@ import ( "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" - "github.com/alist-org/alist/v3/internal/qbittorrent" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/qbittorrent" "github.com/pkg/errors" ) diff --git a/internal/qbittorrent/add.go b/internal/qbittorrent/add.go deleted file mode 100644 index f552a9ec227..00000000000 --- a/internal/qbittorrent/add.go +++ /dev/null @@ -1,60 +0,0 @@ -package qbittorrent - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/google/uuid" - "github.com/pkg/errors" -) - -func AddURL(ctx context.Context, url string, dstDirPath string) error { - // check storage - storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - // check is it could upload - if storage.Config().NoUpload { - return errors.WithStack(errs.UploadNotSupported) - } - // check path is valid - obj, err := op.Get(ctx, storage, dstDirActualPath) - if err != nil { - if !errs.IsObjectNotFound(err) { - return errors.WithMessage(err, "failed get object") - } - } else { - if !obj.IsDir() { - // can't add to a file - return errors.WithStack(errs.NotFolder) - } - } - // call qbittorrent - id := uuid.NewString() - tempDir := filepath.Join(conf.Conf.TempDir, "qbittorrent", id) - err = qbclient.AddFromLink(url, tempDir, id) - if err != nil { - return errors.Wrapf(err, "failed to add url %s", url) - } - DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ - ID: id, - Name: fmt.Sprintf("download %s to [%s](%s)", url, storage.GetStorage().MountPath, dstDirActualPath), - Func: func(tsk *task.Task[string]) error { - m := &Monitor{ - tsk: tsk, - tempDir: tempDir, - dstDirPath: dstDirPath, - seedtime: setting.GetInt(conf.QbittorrentSeedtime, 0), - } - return m.Loop() - }, - })) - return nil -} diff --git a/internal/qbittorrent/client.go b/internal/qbittorrent/client.go deleted file mode 100644 index ec3f7e7b00c..00000000000 --- a/internal/qbittorrent/client.go +++ /dev/null @@ -1,366 +0,0 @@ -package qbittorrent - -import ( - "bytes" - "errors" - "io" - "mime/multipart" - "net/http" - "net/http/cookiejar" - "net/url" - - "github.com/alist-org/alist/v3/pkg/utils" -) - -type Client interface { - AddFromLink(link string, savePath string, id string) error - GetInfo(id string) (TorrentInfo, error) - GetFiles(id string) ([]FileInfo, error) - Delete(id string, deleteFiles bool) error -} - -type client struct { - url *url.URL - client http.Client - Client -} - -func New(webuiUrl string) (Client, error) { - u, err := url.Parse(webuiUrl) - if err != nil { - return nil, err - } - - jar, err := cookiejar.New(nil) - if err != nil { - return nil, err - } - var c = &client{ - url: u, - client: http.Client{Jar: jar}, - } - - err = c.checkAuthorization() - if err != nil { - return nil, err - } - return c, nil -} - -func (c *client) checkAuthorization() error { - // check authorization - if c.authorized() { - return nil - } - - // check authorization after logging in - err := c.login() - if err != nil { - return err - } - if c.authorized() { - return nil - } - return errors.New("unauthorized qbittorrent url") -} - -func (c *client) authorized() bool { - resp, err := c.post("/api/v2/app/version", nil) - if err != nil { - return false - } - return resp.StatusCode == 200 // the status code will be 403 if not authorized -} - -func (c *client) login() error { - // prepare HTTP request - v := url.Values{} - v.Set("username", c.url.User.Username()) - passwd, _ := c.url.User.Password() - v.Set("password", passwd) - resp, err := c.post("/api/v2/auth/login", v) - if err != nil { - return err - } - - // check result - body := make([]byte, 2) - _, err = resp.Body.Read(body) - if err != nil { - return err - } - if string(body) != "Ok" { - return errors.New("failed to login into qBittorrent webui with url: " + c.url.String()) - } - return nil -} - -func (c *client) post(path string, data url.Values) (*http.Response, error) { - u := c.url.JoinPath(path) - u.User = nil // remove userinfo for requests - - req, err := http.NewRequest("POST", u.String(), bytes.NewReader([]byte(data.Encode()))) - if err != nil { - return nil, err - } - if data != nil { - req.Header.Add("Content-Type", "application/x-www-form-urlencoded") - } - - resp, err := c.client.Do(req) - if err != nil { - return nil, err - } - if resp.Cookies() != nil { - c.client.Jar.SetCookies(u, resp.Cookies()) - } - return resp, nil -} - -func (c *client) AddFromLink(link string, savePath string, id string) error { - err := c.checkAuthorization() - if err != nil { - return err - } - - buf := new(bytes.Buffer) - writer := multipart.NewWriter(buf) - - addField := func(name string, value string) { - if err != nil { - return - } - err = writer.WriteField(name, value) - } - addField("urls", link) - addField("savepath", savePath) - addField("tags", "alist-"+id) - addField("autoTMM", "false") - if err != nil { - return err - } - - err = writer.Close() - if err != nil { - return err - } - - u := c.url.JoinPath("/api/v2/torrents/add") - u.User = nil // remove userinfo for requests - req, err := http.NewRequest("POST", u.String(), buf) - if err != nil { - return err - } - req.Header.Add("Content-Type", writer.FormDataContentType()) - - resp, err := c.client.Do(req) - if err != nil { - return err - } - - // check result - body := make([]byte, 2) - _, err = resp.Body.Read(body) - if err != nil { - return err - } - if resp.StatusCode != 200 || string(body) != "Ok" { - return errors.New("failed to add qBittorrent task: " + link) - } - return nil -} - -type TorrentStatus string - -const ( - ERROR TorrentStatus = "error" - MISSINGFILES TorrentStatus = "missingFiles" - UPLOADING TorrentStatus = "uploading" - PAUSEDUP TorrentStatus = "pausedUP" - QUEUEDUP TorrentStatus = "queuedUP" - STALLEDUP TorrentStatus = "stalledUP" - CHECKINGUP TorrentStatus = "checkingUP" - FORCEDUP TorrentStatus = "forcedUP" - ALLOCATING TorrentStatus = "allocating" - DOWNLOADING TorrentStatus = "downloading" - METADL TorrentStatus = "metaDL" - PAUSEDDL TorrentStatus = "pausedDL" - QUEUEDDL TorrentStatus = "queuedDL" - STALLEDDL TorrentStatus = "stalledDL" - CHECKINGDL TorrentStatus = "checkingDL" - FORCEDDL TorrentStatus = "forcedDL" - CHECKINGRESUMEDATA TorrentStatus = "checkingResumeData" - MOVING TorrentStatus = "moving" - UNKNOWN TorrentStatus = "unknown" -) - -// https://github.com/DGuang21/PTGo/blob/main/app/client/client_distributer.go -type TorrentInfo struct { - AddedOn int `json:"added_on"` // 将 torrent 添加到客户端的时间（Unix Epoch） - AmountLeft int64 `json:"amount_left"` // 剩余大小（字节） - AutoTmm bool `json:"auto_tmm"` // 此 torrent 是否由 Automatic Torrent Management 管理 - Availability float64 `json:"availability"` // 当前百分比 - Category string `json:"category"` // - Completed int64 `json:"completed"` // 完成的传输数据量（字节） - CompletionOn int `json:"completion_on"` // Torrent 完成的时间（Unix Epoch） - ContentPath string `json:"content_path"` // torrent 内容的绝对路径（多文件 torrent 的根路径，单文件 torrent 的绝对文件路径） - DlLimit int `json:"dl_limit"` // Torrent 下载速度限制（字节/秒） - Dlspeed int `json:"dlspeed"` // Torrent 下载速度（字节/秒） - Downloaded int64 `json:"downloaded"` // 已经下载大小 - DownloadedSession int64 `json:"downloaded_session"` // 此会话下载的数据量 - Eta int `json:"eta"` // - FLPiecePrio bool `json:"f_l_piece_prio"` // 如果第一个最后一块被优先考虑，则为true - ForceStart bool `json:"force_start"` // 如果为此 torrent 启用了强制启动，则为true - Hash string `json:"hash"` // - LastActivity int `json:"last_activity"` // 上次活跃的时间（Unix Epoch） - MagnetURI string `json:"magnet_uri"` // 与此 torrent 对应的 Magnet URI - MaxRatio float64 `json:"max_ratio"` // 种子/上传停止种子前的最大共享比率 - MaxSeedingTime int `json:"max_seeding_time"` // 停止种子种子前的最长种子时间（秒） - Name string `json:"name"` // - NumComplete int `json:"num_complete"` // - NumIncomplete int `json:"num_incomplete"` // - NumLeechs int `json:"num_leechs"` // 连接到的 leechers 的数量 - NumSeeds int `json:"num_seeds"` // 连接到的种子数 - Priority int `json:"priority"` // 速度优先。如果队列被禁用或 torrent 处于种子模式，则返回 -1 - Progress float64 `json:"progress"` // 进度 - Ratio float64 `json:"ratio"` // Torrent 共享比率 - RatioLimit int `json:"ratio_limit"` // - SavePath string `json:"save_path"` - SeedingTime int `json:"seeding_time"` // Torrent 完成用时（秒） - SeedingTimeLimit int `json:"seeding_time_limit"` // max_seeding_time - SeenComplete int `json:"seen_complete"` // 上次 torrent 完成的时间 - SeqDl bool `json:"seq_dl"` // 如果启用顺序下载，则为true - Size int64 `json:"size"` // - State TorrentStatus `json:"state"` // 参见https://github.com/qbittorrent/qBittorrent/wiki/WebUI-API-(qBittorrent-4.1)#get-torrent-list - SuperSeeding bool `json:"super_seeding"` // 如果启用超级播种，则为true - Tags string `json:"tags"` // Torrent 的逗号连接标签列表 - TimeActive int `json:"time_active"` // 总活动时间（秒） - TotalSize int64 `json:"total_size"` // 此 torrent 中所有文件的总大小（字节）（包括未选择的文件） - Tracker string `json:"tracker"` // 第一个具有工作状态的tracker。如果没有tracker在工作，则返回空字符串。 - TrackersCount int `json:"trackers_count"` // - UpLimit int `json:"up_limit"` // 上传限制 - Uploaded int64 `json:"uploaded"` // 累计上传 - UploadedSession int64 `json:"uploaded_session"` // 当前session累计上传 - Upspeed int `json:"upspeed"` // 上传速度（字节/秒） -} - -type InfoNotFoundError struct { - Id string - Err error -} - -func (i InfoNotFoundError) Error() string { - return "there should be exactly one task with tag \"alist-" + i.Id + "\"" -} - -func NewInfoNotFoundError(id string) InfoNotFoundError { - return InfoNotFoundError{Id: id} -} - -func (c *client) GetInfo(id string) (TorrentInfo, error) { - var infos []TorrentInfo - - err := c.checkAuthorization() - if err != nil { - return TorrentInfo{}, err - } - - v := url.Values{} - v.Set("tag", "alist-"+id) - response, err := c.post("/api/v2/torrents/info", v) - if err != nil { - return TorrentInfo{}, err - } - - body, err := io.ReadAll(response.Body) - if err != nil { - return TorrentInfo{}, err - } - err = utils.Json.Unmarshal(body, &infos) - if err != nil { - return TorrentInfo{}, err - } - if len(infos) != 1 { - return TorrentInfo{}, NewInfoNotFoundError(id) - } - return infos[0], nil -} - -type FileInfo struct { - Index int `json:"index"` - Name string `json:"name"` - Size int64 `json:"size"` - Progress float32 `json:"progress"` - Priority int `json:"priority"` - IsSeed bool `json:"is_seed"` - PieceRange []int `json:"piece_range"` - Availability float32 `json:"availability"` -} - -func (c *client) GetFiles(id string) ([]FileInfo, error) { - var infos []FileInfo - - err := c.checkAuthorization() - if err != nil { - return []FileInfo{}, err - } - - tInfo, err := c.GetInfo(id) - if err != nil { - return []FileInfo{}, err - } - - v := url.Values{} - v.Set("hash", tInfo.Hash) - response, err := c.post("/api/v2/torrents/files", v) - if err != nil { - return []FileInfo{}, err - } - - body, err := io.ReadAll(response.Body) - if err != nil { - return []FileInfo{}, err - } - err = utils.Json.Unmarshal(body, &infos) - if err != nil { - return []FileInfo{}, err - } - return infos, nil -} - -func (c *client) Delete(id string, deleteFiles bool) error { - err := c.checkAuthorization() - if err != nil { - return err - } - - info, err := c.GetInfo(id) - if err != nil { - return err - } - v := url.Values{} - v.Set("hashes", info.Hash) - if deleteFiles { - v.Set("deleteFiles", "true") - } else { - v.Set("deleteFiles", "false") - } - response, err := c.post("/api/v2/torrents/delete", v) - if err != nil { - return err - } - if response.StatusCode != 200 { - return errors.New("failed to delete qbittorrent task") - } - - v = url.Values{} - v.Set("tags", "alist-"+id) - response, err = c.post("/api/v2/torrents/deleteTags", v) - if err != nil { - return err - } - if response.StatusCode != 200 { - return errors.New("failed to delete qbittorrent tag") - } - return nil -} diff --git a/internal/qbittorrent/client_test.go b/internal/qbittorrent/client_test.go deleted file mode 100644 index 21f1dc4104f..00000000000 --- a/internal/qbittorrent/client_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package qbittorrent - -import ( - "net/http" - "net/http/cookiejar" - "net/url" - "testing" -) - -func TestLogin(t *testing.T) { - // test logging in with wrong password - u, err := url.Parse("http://admin:admin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - jar, err := cookiejar.New(nil) - if err != nil { - t.Error(err) - } - var c = &client{ - url: u, - client: http.Client{Jar: jar}, - } - err = c.login() - if err == nil { - t.Error(err) - } - - // test logging in with correct password - u, err = url.Parse("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - c.url = u - err = c.login() - if err != nil { - t.Error(err) - } -} - -// in this test, the `Bypass authentication for clients on localhost` option in qBittorrent webui should be disabled -func TestAuthorized(t *testing.T) { - // init client - u, err := url.Parse("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - jar, err := cookiejar.New(nil) - if err != nil { - t.Error(err) - } - var c = &client{ - url: u, - client: http.Client{Jar: jar}, - } - - // test without logging in, which should be unauthorized - authorized := c.authorized() - if authorized { - t.Error("Should not be authorized") - } - - // test after logging in - err = c.login() - if err != nil { - t.Error(err) - } - authorized = c.authorized() - if !authorized { - t.Error("Should be authorized") - } -} - -func TestNew(t *testing.T) { - _, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - _, err = New("http://admin:wrong_password@127.0.0.1:8080/") - if err == nil { - t.Error("Should get an error") - } -} - -func TestAdd(t *testing.T) { - // init client - c, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - err = c.AddFromLink( - "https://releases.ubuntu.com/22.04/ubuntu-22.04.1-desktop-amd64.iso.torrent", - "D:\\qBittorrentDownload\\alist", - "uuid-1", - ) - if err != nil { - t.Error(err) - } - err = c.AddFromLink( - "magnet:?xt=urn:btih:375ae3280cd80a8e9d7212e11dfaf7c45069dd35&dn=archlinux-2023.02.01-x86_64.iso", - "D:\\qBittorrentDownload\\alist", - "uuid-2", - ) - if err != nil { - t.Error(err) - } -} - -func TestGetInfo(t *testing.T) { - // init client - c, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - _, err = c.GetInfo("uuid-1") - if err != nil { - t.Error(err) - } -} - -func TestGetFiles(t *testing.T) { - // init client - c, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - files, err := c.GetFiles("uuid-1") - if err != nil { - t.Error(err) - } - if len(files) != 1 { - t.Error("should have exactly one file") - } -} - -func TestDelete(t *testing.T) { - // init client - c, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - err = c.AddFromLink( - "https://releases.ubuntu.com/22.04/ubuntu-22.04.1-desktop-amd64.iso.torrent", - "D:\\qBittorrentDownload\\alist", - "uuid-1", - ) - if err != nil { - t.Error(err) - } - err = c.Delete("uuid-1", true) - if err != nil { - t.Error(err) - } -} diff --git a/internal/qbittorrent/monitor.go b/internal/qbittorrent/monitor.go deleted file mode 100644 index bfb1bcf42e1..00000000000 --- a/internal/qbittorrent/monitor.go +++ /dev/null @@ -1,181 +0,0 @@ -package qbittorrent - -import ( - "fmt" - "os" - "path/filepath" - "sync" - "sync/atomic" - "time" - - "github.com/alist-org/alist/v3/internal/stream" - - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -type Monitor struct { - tsk *task.Task[string] - tempDir string - dstDirPath string - seedtime int - finish chan struct{} -} - -func (m *Monitor) Loop() error { - var ( - err error - completed bool - ) - m.finish = make(chan struct{}) - - // wait for qbittorrent to parse torrent and create task - m.tsk.SetStatus("waiting for qbittorrent to parse torrent and create task") - waitCount := 0 - for { - _, err := qbclient.GetInfo(m.tsk.ID) - if err == nil { - break - } - switch err.(type) { - case InfoNotFoundError: - break - default: - return err - } - - waitCount += 1 - if waitCount >= 60 { - return errors.New("torrent parse timeout") - } - timer := time.NewTimer(time.Second) - <-timer.C - } - -outer: - for { - select { - case <-m.tsk.Ctx.Done(): - // delete qbittorrent task and downloaded files when the task exits with error - return qbclient.Delete(m.tsk.ID, true) - case <-time.After(time.Second * 2): - completed, err = m.update() - if completed { - break outer - } - } - } - if err != nil { - return err - } - m.tsk.SetStatus("qbittorrent download completed, transferring") - <-m.finish - m.tsk.SetStatus("completed") - return nil -} - -func (m *Monitor) update() (bool, error) { - info, err := qbclient.GetInfo(m.tsk.ID) - if err != nil { - m.tsk.SetStatus("qbittorrent " + string(info.State)) - return true, err - } - - progress := float64(info.Completed) / float64(info.Size) * 100 - m.tsk.SetProgress(progress) - switch info.State { - case UPLOADING, PAUSEDUP, QUEUEDUP, STALLEDUP, FORCEDUP, CHECKINGUP: - err = m.complete() - return true, errors.WithMessage(err, "failed to transfer file") - case ALLOCATING, DOWNLOADING, METADL, PAUSEDDL, QUEUEDDL, STALLEDDL, CHECKINGDL, FORCEDDL, CHECKINGRESUMEDATA, MOVING: - m.tsk.SetStatus("qbittorrent downloading") - return false, nil - case ERROR, MISSINGFILES, UNKNOWN: - return true, errors.Errorf("failed to download %s, error: %s", m.tsk.ID, info.State) - } - return true, errors.New("unknown error occurred downloading qbittorrent") // should never happen -} - -var TransferTaskManager = task.NewTaskManager(3, func(k *uint64) { - atomic.AddUint64(k, 1) -}) - -func (m *Monitor) complete() error { - // check dstDir again - storage, dstBaseDir, err := op.GetStorageAndActualPath(m.dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - // get files - files, err := qbclient.GetFiles(m.tsk.ID) - if err != nil { - return errors.Wrapf(err, "failed to get files of %s", m.tsk.ID) - } - log.Debugf("files len: %d", len(files)) - // delete qbittorrent task but do not delete the files before transferring to avoid qbittorrent - // accessing downloaded files and throw `cannot access the file because it is being used by another process` error - // err = qbclient.Delete(m.tsk.ID, false) - // if err != nil { - // return err - // } - // upload files - var wg sync.WaitGroup - wg.Add(len(files)) - go func() { - wg.Wait() - m.finish <- struct{}{} - if m.seedtime < 0 { - log.Debugf("do not delete qb task %s", m.tsk.ID) - return - } - log.Debugf("delete qb task %s after %d minutes", m.tsk.ID, m.seedtime) - <-time.After(time.Duration(m.seedtime) * time.Minute) - err := qbclient.Delete(m.tsk.ID, true) - if err != nil { - log.Errorln(err.Error()) - } - err = os.RemoveAll(m.tempDir) - if err != nil { - log.Errorf("failed to remove qbittorrent temp dir: %+v", err.Error()) - } - }() - for _, file := range files { - tempPath := filepath.Join(m.tempDir, file.Name) - dstPath := filepath.Join(dstBaseDir, file.Name) - dstDir := filepath.Dir(dstPath) - fileName := filepath.Base(dstPath) - TransferTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("transfer %s to [%s](%s)", tempPath, storage.GetStorage().MountPath, dstPath), - Func: func(tsk *task.Task[uint64]) error { - defer wg.Done() - size := file.Size - mimetype := utils.GetMimeType(tempPath) - f, err := os.Open(tempPath) - if err != nil { - return errors.Wrapf(err, "failed to open file %s", tempPath) - } - s := stream.FileStream{ - Obj: &model.Object{ - Name: fileName, - Size: size, - Modified: time.Now(), - IsFolder: false, - }, - Reader: f, - Closers: utils.NewClosers(f), - Mimetype: mimetype, - } - ss, err := stream.NewSeekableStream(s, nil) - if err != nil { - return err - } - return op.Put(tsk.Ctx, storage, dstDir, ss, tsk.SetProgress) - }, - })) - } - return nil -} diff --git a/internal/qbittorrent/qbittorrent.go b/internal/qbittorrent/qbittorrent.go deleted file mode 100644 index d011717578e..00000000000 --- a/internal/qbittorrent/qbittorrent.go +++ /dev/null @@ -1,23 +0,0 @@ -package qbittorrent - -import ( - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/pkg/task" -) - -var DownTaskManager = task.NewTaskManager[string](3) -var qbclient Client - -func InitClient() error { - var err error - qbclient = nil - - url := setting.GetStr(conf.QbittorrentUrl) - qbclient, err = New(url) - return err -} - -func IsQbittorrentReady() bool { - return qbclient != nil -} diff --git a/server/handles/task.go b/server/handles/task.go index 15e8067248a..821f7d5663c 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -5,7 +5,6 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/offline_download/tool" - "github.com/alist-org/alist/v3/internal/qbittorrent" "github.com/alist-org/alist/v3/pkg/task" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" @@ -118,10 +117,6 @@ func taskRoute[K comparable](g *gin.RouterGroup, manager *task.Manager[K], k2Str func SetupTaskRoute(g *gin.RouterGroup) { taskRoute(g.Group("/upload"), fs.UploadTaskManager, uint64K2Str, str2Uint64K) taskRoute(g.Group("/copy"), fs.CopyTaskManager, uint64K2Str, str2Uint64K) - taskRoute(g.Group("/qbit_down"), qbittorrent.DownTaskManager, strK2Str, str2StrK) - taskRoute(g.Group("/qbit_transfer"), qbittorrent.TransferTaskManager, uint64K2Str, str2Uint64K) - //taskRoute(g.Group("/aria2_down"), aria2.DownTaskManager, strK2Str, str2StrK) - //taskRoute(g.Group("/aria2_transfer"), aria2.TransferTaskManager, uint64K2Str, str2Uint64K) taskRoute(g.Group("/offline_download"), tool.DownTaskManager, strK2Str, str2StrK) taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager, uint64K2Str, str2Uint64K) } From 11a30c504485c3f17e6fffd0d9875be2cb42c99a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 20 Nov 2023 18:01:51 +0800 Subject: [PATCH 023/659] feat: refactor task module --- go.mod | 3 + go.sum | 12 ++ internal/fs/copy.go | 92 ++++++------ internal/fs/put.go | 41 ++++-- internal/offline_download/aria2/aria2.go | 42 ++---- internal/offline_download/qbit/qbit.go | 22 +-- internal/offline_download/tool/add.go | 50 +++---- internal/offline_download/tool/base.go | 22 +-- internal/offline_download/tool/download.go | 147 +++++++++++++++++++ internal/offline_download/tool/monitor.go | 159 --------------------- internal/offline_download/tool/tools.go | 9 +- internal/offline_download/tool/transfer.go | 66 +++++++++ server/handles/offline_download.go | 14 +- server/handles/task.go | 103 ++++--------- 14 files changed, 405 insertions(+), 377 deletions(-) create mode 100644 internal/offline_download/tool/download.go delete mode 100644 internal/offline_download/tool/monitor.go create mode 100644 internal/offline_download/tool/transfer.go diff --git a/go.mod b/go.mod index c6a6bc21592..e81d863f770 100644 --- a/go.mod +++ b/go.mod @@ -123,6 +123,7 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgx/v5 v5.3.0 // indirect + github.com/jaevor/go-nanoid v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -183,6 +184,8 @@ require ( github.com/u2takey/go-utils v0.3.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect github.com/x448/float16 v0.8.4 // indirect + github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect + github.com/xhofe/tache v0.0.0-20231120085916-722855be0521 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.3.0 // indirect diff --git a/go.sum b/go.sum index 5abe5aefd6b..7d1ccf7b542 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZ github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jaevor/go-nanoid v1.3.0 h1:nD+iepesZS6pr3uOVf20vR9GdGgJW1HPaR46gtrxzkg= +github.com/jaevor/go-nanoid v1.3.0/go.mod h1:SI+jFaPuddYkqkVQoNGHs81navCtH388TcrH0RqFKgY= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -433,6 +435,16 @@ github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXo github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= +github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= +github.com/xhofe/tache v0.0.0-20231110075853-2bd4b52dad9b h1:958N/31ioR0QSg6RarX1aqBsfmlOI2JeYiVzxeGdUAA= +github.com/xhofe/tache v0.0.0-20231110075853-2bd4b52dad9b/go.mod h1:1ISbKrHZNMMrXvgCdaFV0Vkc9Wbo7WV1q7Teovm4Huc= +github.com/xhofe/tache v0.0.0-20231119124711-c417893fc267 h1:MC271sH8UHYqr/IDz9PsqTlyD51HyFvxtQRTemwxR9s= +github.com/xhofe/tache v0.0.0-20231119124711-c417893fc267/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/xhofe/tache v0.0.0-20231120064353-a3585a237e25 h1:XZBuEzDB9Kqni/+zAKxl30iOdp80/GavUsCkPMiQMjg= +github.com/xhofe/tache v0.0.0-20231120064353-a3585a237e25/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/xhofe/tache v0.0.0-20231120085916-722855be0521 h1:m7O+xOqQRysjFngMhQ39RzCFdiCouFLvsrV7N2ScbUY= +github.com/xhofe/tache v0.0.0-20231120085916-722855be0521/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= diff --git a/internal/fs/copy.go b/internal/fs/copy.go index c3e387cb227..911e528ab1a 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -3,24 +3,39 @@ package fs import ( "context" "fmt" - "net/http" - stdpath "path" - "sync/atomic" - "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/pkg/task" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" + "net/http" + stdpath "path" ) -var CopyTaskManager = task.NewTaskManager(3, func(tid *uint64) { - atomic.AddUint64(tid, 1) -}) +type CopyTask struct { + tache.Base + Status string `json:"status"` + srcStorage, dstStorage driver.Driver + srcObjPath, dstDirPath string +} + +func (t *CopyTask) GetName() string { + return fmt.Sprintf("copy [%s](%s) to [%s](%s)", + t.srcStorage.GetStorage().MountPath, t.srcObjPath, t.dstStorage.GetStorage().MountPath, t.dstDirPath) +} + +func (t *CopyTask) GetStatus() string { + return t.Status +} + +func (t *CopyTask) Run() error { + return copyBetween2Storages(t, t.srcStorage, t.dstStorage, t.srcObjPath, t.dstDirPath) +} + +var CopyTaskManager = tache.NewManager[*CopyTask]() // Copy if in the same storage, call move method // if not, add copy task @@ -63,59 +78,52 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool } } // not in the same storage - CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjActualPath, dstStorage.GetStorage().MountPath, dstDirActualPath), - Func: func(task *task.Task[uint64]) error { - return copyBetween2Storages(task, srcStorage, dstStorage, srcObjActualPath, dstDirActualPath) - }, - })) + CopyTaskManager.Add(&CopyTask{ + srcStorage: srcStorage, + dstStorage: dstStorage, + srcObjPath: srcObjActualPath, + dstDirPath: dstDirActualPath, + }) return true, nil } -func copyBetween2Storages(t *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error { - t.SetStatus("getting src object") - srcObj, err := op.Get(t.Ctx, srcStorage, srcObjPath) +func copyBetween2Storages(t *CopyTask, srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error { + t.Status = "getting src object" + srcObj, err := op.Get(t.Ctx(), srcStorage, srcObjPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", srcObjPath) } if srcObj.IsDir() { - t.SetStatus("src object is dir, listing objs") - objs, err := op.List(t.Ctx, srcStorage, srcObjPath, model.ListArgs{}) + t.Status = "src object is dir, listing objs" + objs, err := op.List(t.Ctx(), srcStorage, srcObjPath, model.ListArgs{}) if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", srcObjPath) } for _, obj := range objs { - if utils.IsCanceled(t.Ctx) { + if utils.IsCanceled(t.Ctx()) { return nil } srcObjPath := stdpath.Join(srcObjPath, obj.GetName()) dstObjPath := stdpath.Join(dstDirPath, srcObj.GetName()) - CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjPath, dstStorage.GetStorage().MountPath, dstObjPath), - Func: func(t *task.Task[uint64]) error { - return copyBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstObjPath) - }, - })) + CopyTaskManager.Add(&CopyTask{ + srcStorage: srcStorage, + dstStorage: dstStorage, + srcObjPath: srcObjPath, + dstDirPath: dstObjPath, + }) } - } else { - CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjPath, dstStorage.GetStorage().MountPath, dstDirPath), - Func: func(t *task.Task[uint64]) error { - err := copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath) - log.Debugf("copy file between storages: %+v", err) - return err - }, - })) + t.Status = "src object is dir, added all copy tasks of objs" + return nil } - return nil + return copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath) } -func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcFilePath, dstDirPath string) error { - srcFile, err := op.Get(tsk.Ctx, srcStorage, srcFilePath) +func copyFileBetween2Storages(tsk *CopyTask, srcStorage, dstStorage driver.Driver, srcFilePath, dstDirPath string) error { + srcFile, err := op.Get(tsk.Ctx(), srcStorage, srcFilePath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", srcFilePath) } - link, _, err := op.Link(tsk.Ctx, srcStorage, srcFilePath, model.LinkArgs{ + link, _, err := op.Link(tsk.Ctx(), srcStorage, srcFilePath, model.LinkArgs{ Header: http.Header{}, }) if err != nil { @@ -123,12 +131,12 @@ func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage dri } fs := stream.FileStream{ Obj: srcFile, - Ctx: tsk.Ctx, + Ctx: tsk.Ctx(), } // any link provided is seekable ss, err := stream.NewSeekableStream(fs, link) if err != nil { return errors.WithMessagef(err, "failed get [%s] stream", srcFilePath) } - return op.Put(tsk.Ctx, dstStorage, dstDirPath, ss, tsk.SetProgress, true) + return op.Put(tsk.Ctx(), dstStorage, dstDirPath, ss, tsk.SetProgress, true) } diff --git a/internal/fs/put.go b/internal/fs/put.go index ab6d24bf571..5c154756d14 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -3,18 +3,34 @@ package fs import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/model" - "sync/atomic" - + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" "github.com/pkg/errors" + "github.com/xhofe/tache" ) -var UploadTaskManager = task.NewTaskManager(3, func(tid *uint64) { - atomic.AddUint64(tid, 1) -}) +type UploadTask struct { + tache.Base + storage driver.Driver + dstDirActualPath string + file model.FileStreamer +} + +func (t *UploadTask) GetName() string { + return fmt.Sprintf("upload %s to [%s](%s)", t.file.GetName(), t.storage.GetStorage().MountPath, t.dstDirActualPath) +} + +func (t *UploadTask) GetStatus() string { + return "uploading" +} + +func (t *UploadTask) Run() error { + return op.Put(t.Ctx(), t.storage, t.dstDirActualPath, t.file, t.SetProgress, true) +} + +var UploadTaskManager = tache.NewManager[*UploadTask]() // putAsTask add as a put task and return immediately func putAsTask(dstDirPath string, file model.FileStreamer) error { @@ -33,12 +49,11 @@ func putAsTask(dstDirPath string, file model.FileStreamer) error { //file.SetReader(tempFile) //file.SetTmpFile(tempFile) } - UploadTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("upload %s to [%s](%s)", file.GetName(), storage.GetStorage().MountPath, dstDirActualPath), - Func: func(task *task.Task[uint64]) error { - return op.Put(task.Ctx, storage, dstDirActualPath, file, task.SetProgress, true) - }, - })) + UploadTaskManager.Add(&UploadTask{ + storage: storage, + dstDirActualPath: dstDirActualPath, + file: file, + }) return nil } diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index f2b9628c240..4cdad64b924 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -21,6 +21,10 @@ type Aria2 struct { client rpc.Client } +func (a *Aria2) Name() string { + return "aria2" +} + func (a *Aria2) Items() []model.SettingItem { // aria2 settings return []model.SettingItem{ @@ -58,16 +62,17 @@ func (a *Aria2) AddURL(args *tool.AddUrlArgs) (string, error) { if err != nil { return "", err } + notify.Signals.Store(gid, args.Signal) return gid, nil } -func (a *Aria2) Remove(tid string) error { - _, err := a.client.Remove(tid) +func (a *Aria2) Remove(task *tool.DownloadTask) error { + _, err := a.client.Remove(task.GID) return err } -func (a *Aria2) Status(tid string) (*tool.Status, error) { - info, err := a.client.TellStatus(tid) +func (a *Aria2) Status(task *tool.DownloadTask) (*tool.Status, error) { + info, err := a.client.TellStatus(task.GID) if err != nil { return nil, err } @@ -85,15 +90,15 @@ func (a *Aria2) Status(tid string) (*tool.Status, error) { } s.Progress = float64(downloaded) / float64(total) * 100 if len(info.FollowedBy) != 0 { - s.NewTID = info.FollowedBy[0] - notify.Signals.Delete(tid) - //notify.Signals.Store(gid, m.c) + s.NewGID = info.FollowedBy[0] + notify.Signals.Delete(task.GID) + notify.Signals.Store(s.NewGID, task.Signal) } switch info.Status { case "complete": s.Completed = true case "error": - s.Err = errors.Errorf("failed to download %s, error: %s", tid, info.ErrorMessage) + s.Err = errors.Errorf("failed to download %s, error: %s", task.GID, info.ErrorMessage) case "active": s.Status = "aria2: " + info.Status if info.Seeder == "true" { @@ -102,32 +107,15 @@ func (a *Aria2) Status(tid string) (*tool.Status, error) { case "waiting", "paused": s.Status = "aria2: " + info.Status case "removed": - s.Err = errors.Errorf("failed to download %s, removed", tid) + s.Err = errors.Errorf("failed to download %s, removed", task.GID) default: return nil, errors.Errorf("[aria2] unknown status %s", info.Status) } return s, nil } -func (a *Aria2) GetFiles(tid string) []tool.File { - //files, err := a.client.GetFiles(tid) - //if err != nil { - // return nil - //} - //return utils.MustSliceConvert(files, func(f rpc.FileInfo) tool.File { - // return tool.File{ - // //ReadCloser: nil, - // Name: path.Base(f.Path), - // Size: f.Length, - // Path: "", - // Modified: time.Time{}, - // } - //}) - return nil -} - var _ tool.Tool = (*Aria2)(nil) func init() { - tool.Tools.Add("aria2", &Aria2{}) + tool.Tools.Add(&Aria2{}) } diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go index 388ce22edbd..28a5170ec12 100644 --- a/internal/offline_download/qbit/qbit.go +++ b/internal/offline_download/qbit/qbit.go @@ -13,6 +13,10 @@ type QBittorrent struct { client qbittorrent.Client } +func (a *QBittorrent) Name() string { + return "qBittorrent" +} + func (a *QBittorrent) Items() []model.SettingItem { // qBittorrent settings return []model.SettingItem{ @@ -44,13 +48,13 @@ func (a *QBittorrent) AddURL(args *tool.AddUrlArgs) (string, error) { return args.UID, nil } -func (a *QBittorrent) Remove(tid string) error { - err := a.client.Delete(tid, true) +func (a *QBittorrent) Remove(task *tool.DownloadTask) error { + err := a.client.Delete(task.GID, true) return err } -func (a *QBittorrent) Status(tid string) (*tool.Status, error) { - info, err := a.client.GetInfo(tid) +func (a *QBittorrent) Status(task *tool.DownloadTask) (*tool.Status, error) { + info, err := a.client.GetInfo(task.GID) if err != nil { return nil, err } @@ -62,19 +66,15 @@ func (a *QBittorrent) Status(tid string) (*tool.Status, error) { case qbittorrent.ALLOCATING, qbittorrent.DOWNLOADING, qbittorrent.METADL, qbittorrent.PAUSEDDL, qbittorrent.QUEUEDDL, qbittorrent.STALLEDDL, qbittorrent.CHECKINGDL, qbittorrent.FORCEDDL, qbittorrent.CHECKINGRESUMEDATA, qbittorrent.MOVING: s.Status = "[qBittorrent] downloading" case qbittorrent.ERROR, qbittorrent.MISSINGFILES, qbittorrent.UNKNOWN: - s.Err = errors.Errorf("[qBittorrent] failed to download %s, error: %s", tid, info.State) + s.Err = errors.Errorf("[qBittorrent] failed to download %s, error: %s", task.GID, info.State) default: - s.Err = errors.Errorf("[qBittorrent] unknown error occurred downloading %s", tid) + s.Err = errors.Errorf("[qBittorrent] unknown error occurred downloading %s", task.GID) } return s, nil } -func (a *QBittorrent) GetFiles(tid string) []tool.File { - return nil -} - var _ tool.Tool = (*QBittorrent)(nil) func init() { - tool.Tools.Add("qBittorrent", &QBittorrent{}) + tool.Tools.Add(&QBittorrent{}) } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index ceaf92d3bc3..9ad8d055726 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,21 +2,27 @@ package tool import ( "context" - "fmt" - "path/filepath" - "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" "github.com/google/uuid" "github.com/pkg/errors" + "path/filepath" +) + +type DeletePolicy string + +const ( + DeleteOnUploadSucceed DeletePolicy = "delete_on_upload_succeed" + DeleteOnUploadFailed DeletePolicy = "delete_on_upload_failed" + DeleteNever DeletePolicy = "delete_never" ) type AddURLArgs struct { - URL string - DstDirPath string - Tool string + URL string + DstDirPath string + Tool string + DeletePolicy DeletePolicy } func AddURL(ctx context.Context, args *AddURLArgs) error { @@ -56,29 +62,13 @@ func AddURL(ctx context.Context, args *AddURLArgs) error { uid := uuid.NewString() tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) - signal := make(chan int) - gid, err := tool.AddURL(&AddUrlArgs{ - Url: args.URL, - UID: uid, - TempDir: tempDir, - Signal: signal, - }) - if err != nil { - return errors.Wrapf(err, "[%s] failed to add uri %s", args.Tool, args.URL) + t := &DownloadTask{ + Url: args.URL, + DstDirPath: args.DstDirPath, + TempDir: tempDir, + DeletePolicy: args.DeletePolicy, + tool: tool, } - DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ - ID: gid, - Name: fmt.Sprintf("download %s to [%s](%s)", args.URL, storage.GetStorage().MountPath, dstDirActualPath), - Func: func(tsk *task.Task[string]) error { - m := &Monitor{ - tool: tool, - tsk: tsk, - tempDir: tempDir, - dstDirPath: args.DstDirPath, - signal: signal, - } - return m.Loop() - }, - })) + DownloadTaskManager.Add(t) return nil } diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go index 4689635b54e..1dd8e82bb4e 100644 --- a/internal/offline_download/tool/base.go +++ b/internal/offline_download/tool/base.go @@ -17,13 +17,14 @@ type AddUrlArgs struct { type Status struct { Progress float64 - NewTID string + NewGID string Completed bool Status string Err error } type Tool interface { + Name() string // Items return the setting items the tool need Items() []model.SettingItem Init() (string, error) @@ -31,20 +32,23 @@ type Tool interface { // AddURL add an uri to download, return the task id AddURL(args *AddUrlArgs) (string, error) // Remove the download if task been canceled - Remove(tid string) error + Remove(task *DownloadTask) error // Status return the status of the download task, if an error occurred, return the error in Status.Err - Status(tid string) (*Status, error) + Status(task *DownloadTask) (*Status, error) +} + +type GetFileser interface { // GetFiles return the files of the download task, if nil, means walk the temp dir to get the files - GetFiles(tid string) []File + GetFiles(task *DownloadTask) []File } type File struct { // ReadCloser for http client - io.ReadCloser - Name string - Size int64 - Path string - Modified time.Time + ReadCloser io.ReadCloser + Name string + Size int64 + Path string + Modified time.Time } func (f *File) GetReadCloser() (io.ReadCloser, error) { diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go new file mode 100644 index 00000000000..7b536762e37 --- /dev/null +++ b/internal/offline_download/tool/download.go @@ -0,0 +1,147 @@ +package tool + +import ( + "fmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" + "sync" + "time" +) + +type DownloadTask struct { + tache.Base + Url string `json:"url"` + DstDirPath string `json:"dst_dir_path"` + TempDir string `json:"temp_dir"` + DeletePolicy DeletePolicy `json:"delete_policy"` + + Status string `json:"status"` + Signal chan int `json:"-"` + GID string `json:"-"` + finish chan struct{} + tool Tool + callStatusRetried int +} + +func (t *DownloadTask) Run() error { + t.Signal = make(chan int) + t.finish = make(chan struct{}) + defer func() { + t.Signal = nil + t.finish = nil + }() + gid, err := t.tool.AddURL(&AddUrlArgs{ + Url: t.Url, + UID: t.ID, + TempDir: t.TempDir, + Signal: t.Signal, + }) + if err != nil { + return err + } + t.GID = gid + var ( + ok bool + ) +outer: + for { + select { + case <-t.CtxDone(): + err := t.tool.Remove(t) + return err + case <-t.Signal: + ok, err = t.Update() + if ok { + break outer + } + case <-time.After(time.Second * 3): + ok, err = t.Update() + if ok { + break outer + } + } + } + if err != nil { + return err + } + t.Status = "aria2 download completed, maybe transferring" + t.finish <- struct{}{} + t.Status = "offline download completed" + return nil +} + +// Update download status, return true if download completed +func (t *DownloadTask) Update() (bool, error) { + info, err := t.tool.Status(t) + if err != nil { + t.callStatusRetried++ + log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) + return false, nil + } + if t.callStatusRetried > 5 { + return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) + } + t.callStatusRetried = 0 + t.SetProgress(info.Progress) + t.Status = fmt.Sprintf("[%s]: %s", t.tool.Name(), info.Status) + if info.NewGID != "" { + log.Debugf("followen by: %+v", info.NewGID) + t.GID = info.NewGID + return false, nil + } + // if download completed + if info.Completed { + err := t.Complete() + return true, errors.WithMessage(err, "failed to transfer file") + } + // if download failed + if info.Err != nil { + return true, errors.Errorf("failed to download %s, error: %s", t.ID, info.Err.Error()) + } + return false, nil +} + +func (t *DownloadTask) Complete() error { + var ( + files []File + err error + ) + if getFileser, ok := t.tool.(GetFileser); ok { + files = getFileser.GetFiles(t) + } else { + files, err = GetFiles(t.TempDir) + if err != nil { + return errors.Wrapf(err, "failed to get files") + } + } + // upload files + var wg sync.WaitGroup + wg.Add(len(files)) + go func() { + wg.Wait() + t.finish <- struct{}{} + }() + for i, _ := range files { + file := files[i] + TransferTaskManager.Add(&TransferTask{ + file: file, + dstDirPath: t.DstDirPath, + wg: &wg, + tempDir: t.TempDir, + }) + } + return nil +} + +func (t *DownloadTask) GetName() string { + return fmt.Sprintf("download %s to (%s)", t.Url, t.DstDirPath) +} + +func (t *DownloadTask) GetStatus() string { + return t.Status +} + +var ( + DownloadTaskManager *tache.Manager[*DownloadTask] = tache.NewManager[*DownloadTask]() +) diff --git a/internal/offline_download/tool/monitor.go b/internal/offline_download/tool/monitor.go deleted file mode 100644 index 984bda17cb7..00000000000 --- a/internal/offline_download/tool/monitor.go +++ /dev/null @@ -1,159 +0,0 @@ -package tool - -import ( - "fmt" - "os" - "path/filepath" - "sync" - "sync/atomic" - "time" - - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -type Monitor struct { - tool Tool - tsk *task.Task[string] - tempDir string - retried int - dstDirPath string - finish chan struct{} - signal chan int -} - -func (m *Monitor) Loop() error { - m.finish = make(chan struct{}) - var ( - err error - ok bool - ) -outer: - for { - select { - case <-m.tsk.Ctx.Done(): - err := m.tool.Remove(m.tsk.ID) - return err - case <-m.signal: - ok, err = m.Update() - if ok { - break outer - } - case <-time.After(time.Second * 2): - ok, err = m.Update() - if ok { - break outer - } - } - } - if err != nil { - return err - } - m.tsk.SetStatus("aria2 download completed, transferring") - <-m.finish - m.tsk.SetStatus("completed") - return nil -} - -// Update download status, return true if download completed -func (m *Monitor) Update() (bool, error) { - info, err := m.tool.Status(m.tsk.ID) - if err != nil { - m.retried++ - log.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) - return false, nil - } - if m.retried > 5 { - return true, errors.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) - } - m.retried = 0 - m.tsk.SetProgress(info.Progress) - m.tsk.SetStatus("tool: " + info.Status) - if info.NewTID != "" { - log.Debugf("followen by: %+v", info.NewTID) - DownTaskManager.RawTasks().Delete(m.tsk.ID) - m.tsk.ID = info.NewTID - DownTaskManager.RawTasks().Store(m.tsk.ID, m.tsk) - return false, nil - } - // if download completed - if info.Completed { - err := m.Complete() - return true, errors.WithMessage(err, "failed to transfer file") - } - // if download failed - if info.Err != nil { - return true, errors.Errorf("failed to download %s, error: %s", m.tsk.ID, info.Err.Error()) - } - return false, nil -} - -var TransferTaskManager = task.NewTaskManager(3, func(k *uint64) { - atomic.AddUint64(k, 1) -}) - -func (m *Monitor) Complete() error { - // check dstDir again - storage, dstDirActualPath, err := op.GetStorageAndActualPath(m.dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - var files []File - if f := m.tool.GetFiles(m.tsk.ID); f != nil { - files = f - } else { - files, err = GetFiles(m.tempDir) - if err != nil { - return errors.Wrapf(err, "failed to get files") - } - } - // upload files - var wg sync.WaitGroup - wg.Add(len(files)) - go func() { - wg.Wait() - err := os.RemoveAll(m.tempDir) - m.finish <- struct{}{} - if err != nil { - log.Errorf("failed to remove aria2 temp dir: %+v", err.Error()) - } - }() - for i, _ := range files { - file := files[i] - TransferTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, storage.GetStorage().MountPath, dstDirActualPath), - Func: func(tsk *task.Task[uint64]) error { - defer wg.Done() - mimetype := utils.GetMimeType(file.Path) - rc, err := file.GetReadCloser() - if err != nil { - return errors.Wrapf(err, "failed to open file %s", file.Path) - } - s := &stream.FileStream{ - Ctx: nil, - Obj: &model.Object{ - Name: filepath.Base(file.Path), - Size: file.Size, - Modified: file.Modified, - IsFolder: false, - }, - Reader: rc, - Mimetype: mimetype, - Closers: utils.NewClosers(rc), - } - relDir, err := filepath.Rel(m.tempDir, filepath.Dir(file.Path)) - if err != nil { - log.Errorf("find relation directory error: %v", err) - } - newDistDir := filepath.Join(dstDirActualPath, relDir) - return op.Put(tsk.Ctx, storage, newDistDir, s, tsk.SetProgress) - }, - })) - } - return nil -} diff --git a/internal/offline_download/tool/tools.go b/internal/offline_download/tool/tools.go index b7eacbd2b9b..9de7d526ab0 100644 --- a/internal/offline_download/tool/tools.go +++ b/internal/offline_download/tool/tools.go @@ -2,14 +2,11 @@ package tool import ( "fmt" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/task" ) var ( - Tools = make(ToolsManager) - DownTaskManager = task.NewTaskManager[string](3) + Tools = make(ToolsManager) ) type ToolsManager map[string]Tool @@ -21,8 +18,8 @@ func (t ToolsManager) Get(name string) (Tool, error) { return nil, fmt.Errorf("tool %s not found", name) } -func (t ToolsManager) Add(name string, tool Tool) { - t[name] = tool +func (t ToolsManager) Add(tool Tool) { + t[tool.Name()] = tool } func (t ToolsManager) Names() []string { diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go new file mode 100644 index 00000000000..f7d1791d72f --- /dev/null +++ b/internal/offline_download/tool/transfer.go @@ -0,0 +1,66 @@ +package tool + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" + "path/filepath" + "sync" +) + +type TransferTask struct { + tache.Base + file File + dstDirPath string + wg *sync.WaitGroup + tempDir string +} + +func (t *TransferTask) Run() error { + defer t.wg.Done() + // check dstDir again + storage, dstDirActualPath, err := op.GetStorageAndActualPath(t.dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + mimetype := utils.GetMimeType(t.file.Path) + rc, err := t.file.GetReadCloser() + if err != nil { + return errors.Wrapf(err, "failed to open file %s", t.file.Path) + } + s := &stream.FileStream{ + Ctx: nil, + Obj: &model.Object{ + Name: filepath.Base(t.file.Path), + Size: t.file.Size, + Modified: t.file.Modified, + IsFolder: false, + }, + Reader: rc, + Mimetype: mimetype, + Closers: utils.NewClosers(rc), + } + relDir, err := filepath.Rel(t.tempDir, filepath.Dir(t.file.Path)) + if err != nil { + log.Errorf("find relation directory error: %v", err) + } + newDistDir := filepath.Join(dstDirActualPath, relDir) + return op.Put(t.Ctx(), storage, newDistDir, s, t.SetProgress) +} + +func (t *TransferTask) GetName() string { + return fmt.Sprintf("transfer %s to [%s]", t.file.Path, t.dstDirPath) +} + +func (t *TransferTask) GetStatus() string { + return "transferring" +} + +var ( + TransferTaskManager *tache.Manager[*TransferTask] = tache.NewManager[*TransferTask]() +) diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index cf9c1775bae..fdee063df18 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -74,9 +74,10 @@ func OfflineDownloadTools(c *gin.Context) { } type AddOfflineDownloadReq struct { - Urls []string `json:"urls"` - Path string `json:"path"` - Tool string `json:"tool"` + Urls []string `json:"urls"` + Path string `json:"path"` + Tool string `json:"tool"` + DeletePolicy string `json:"delete_policy"` } func AddOfflineDownload(c *gin.Context) { @@ -98,9 +99,10 @@ func AddOfflineDownload(c *gin.Context) { } for _, url := range req.Urls { err := tool.AddURL(c, &tool.AddURLArgs{ - URL: url, - DstDirPath: reqPath, - Tool: req.Tool, + URL: url, + DstDirPath: reqPath, + Tool: req.Tool, + DeletePolicy: tool.DeletePolicy(req.DeletePolicy), }) if err != nil { common.ErrorResp(c, err, 500) diff --git a/server/handles/task.go b/server/handles/task.go index 821f7d5663c..1193ce05884 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -1,122 +1,77 @@ package handles import ( - "strconv" - "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/offline_download/tool" - "github.com/alist-org/alist/v3/pkg/task" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "github.com/xhofe/tache" ) type TaskInfo struct { - ID string `json:"id"` - Name string `json:"name"` - State string `json:"state"` - Status string `json:"status"` - Progress float64 `json:"progress"` - Error string `json:"error"` -} - -type K2Str[K comparable] func(k K) string - -func uint64K2Str(k uint64) string { - return strconv.FormatUint(k, 10) + ID string `json:"id"` + Name string `json:"name"` + State tache.State `json:"state"` + Status string `json:"status"` + Progress float64 `json:"progress"` + Error string `json:"error"` } -func strK2Str(str string) string { - return str -} - -func getTaskInfo[K comparable](task *task.Task[K], k2Str K2Str[K]) TaskInfo { +func getTaskInfo[T tache.TaskWithInfo](task T) TaskInfo { return TaskInfo{ - ID: k2Str(task.ID), - Name: task.Name, + ID: task.GetID(), + Name: task.GetName(), State: task.GetState(), Status: task.GetStatus(), Progress: task.GetProgress(), - Error: task.GetErrMsg(), + Error: task.GetErr().Error(), } } -func getTaskInfos[K comparable](tasks []*task.Task[K], k2Str K2Str[K]) []TaskInfo { +func getTaskInfos[T tache.TaskWithInfo](tasks []T) []TaskInfo { var infos []TaskInfo for _, t := range tasks { - infos = append(infos, getTaskInfo(t, k2Str)) + infos = append(infos, getTaskInfo(t)) } return infos } -type Str2K[K comparable] func(str string) (K, error) - -func str2Uint64K(str string) (uint64, error) { - return strconv.ParseUint(str, 10, 64) -} - -func str2StrK(str string) (string, error) { - return str, nil -} - -func taskRoute[K comparable](g *gin.RouterGroup, manager *task.Manager[K], k2Str K2Str[K], str2K Str2K[K]) { +func taskRoute[T tache.TaskWithInfo](g *gin.RouterGroup, manager *tache.Manager[T]) { g.GET("/undone", func(c *gin.Context) { - common.SuccessResp(c, getTaskInfos(manager.ListUndone(), k2Str)) + common.SuccessResp(c, getTaskInfos(manager.GetByState(tache.StatePending, tache.StateRunning, + tache.StateCanceling, tache.StateErrored, tache.StateFailing, tache.StateWaitingRetry, tache.StateBeforeRetry))) }) g.GET("/done", func(c *gin.Context) { - common.SuccessResp(c, getTaskInfos(manager.ListDone(), k2Str)) + common.SuccessResp(c, getTaskInfos(manager.GetByState(tache.StateCanceled, tache.StateFailed, tache.StateSucceeded))) }) g.POST("/cancel", func(c *gin.Context) { tid := c.Query("tid") - id, err := str2K(tid) - if err != nil { - common.ErrorResp(c, err, 400) - return - } - if err := manager.Cancel(id); err != nil { - common.ErrorResp(c, err, 500) - } else { - common.SuccessResp(c) - } + manager.Cancel(tid) + common.SuccessResp(c) }) g.POST("/delete", func(c *gin.Context) { tid := c.Query("tid") - id, err := str2K(tid) - if err != nil { - common.ErrorResp(c, err, 400) - return - } - if err := manager.Remove(id); err != nil { - common.ErrorResp(c, err, 500) - } else { - common.SuccessResp(c) - } + manager.Remove(tid) + common.SuccessResp(c) }) g.POST("/retry", func(c *gin.Context) { tid := c.Query("tid") - id, err := str2K(tid) - if err != nil { - common.ErrorResp(c, err, 400) - return - } - if err := manager.Retry(id); err != nil { - common.ErrorResp(c, err, 500) - } else { - common.SuccessResp(c) - } + manager.Retry(tid) + common.SuccessResp(c) }) g.POST("/clear_done", func(c *gin.Context) { - manager.ClearDone() + manager.RemoveByState(tache.StateCanceled, tache.StateFailed, tache.StateSucceeded) common.SuccessResp(c) }) g.POST("/clear_succeeded", func(c *gin.Context) { - manager.ClearSucceeded() + manager.RemoveByState(tache.StateSucceeded) common.SuccessResp(c) }) } func SetupTaskRoute(g *gin.RouterGroup) { - taskRoute(g.Group("/upload"), fs.UploadTaskManager, uint64K2Str, str2Uint64K) - taskRoute(g.Group("/copy"), fs.CopyTaskManager, uint64K2Str, str2Uint64K) - taskRoute(g.Group("/offline_download"), tool.DownTaskManager, strK2Str, str2StrK) - taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager, uint64K2Str, str2Uint64K) + taskRoute(g.Group("/upload"), fs.UploadTaskManager) + taskRoute(g.Group("/copy"), fs.CopyTaskManager) + taskRoute(g.Group("/offline_download"), tool.DownloadTaskManager) + taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager) } From 7583c4d734a0c58ecfa897e4eb5c4f02adcce632 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 21 Nov 2023 15:51:08 +0800 Subject: [PATCH 024/659] feat: customize workers and retry of task (close #5493 fix #5274) --- cmd/server.go | 8 ++- internal/bootstrap/task.go | 15 +++++ internal/conf/config.go | 76 +++++++++++++++------- internal/fs/copy.go | 2 +- internal/fs/put.go | 2 +- internal/offline_download/tool/download.go | 2 +- internal/offline_download/tool/transfer.go | 2 +- server/handles/task.go | 6 +- 8 files changed, 82 insertions(+), 31 deletions(-) create mode 100644 internal/bootstrap/task.go diff --git a/cmd/server.go b/cmd/server.go index 0678e3e1188..d03a9d8099c 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "fmt" "net" "net/http" @@ -36,6 +37,7 @@ the address is defined in config file`, } bootstrap.InitOfflineDownloadTools() bootstrap.LoadStorages() + bootstrap.InitTaskManager() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) } @@ -49,7 +51,7 @@ the address is defined in config file`, httpSrv = &http.Server{Addr: httpBase, Handler: r} go func() { err := httpSrv.ListenAndServe() - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { utils.Log.Fatalf("failed to start http: %s", err.Error()) } }() @@ -60,7 +62,7 @@ the address is defined in config file`, httpsSrv = &http.Server{Addr: httpsBase, Handler: r} go func() { err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { utils.Log.Fatalf("failed to start https: %s", err.Error()) } }() @@ -84,7 +86,7 @@ the address is defined in config file`, } } err = unixSrv.Serve(listener) - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { utils.Log.Fatalf("failed to start unix: %s", err.Error()) } }() diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go new file mode 100644 index 00000000000..5d52e9d2ef8 --- /dev/null +++ b/internal/bootstrap/task.go @@ -0,0 +1,15 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/xhofe/tache" +) + +func InitTaskManager() { + fs.UploadTaskManager = tache.NewManager[*fs.UploadTask](tache.WithWorks(conf.Conf.Tasks.Upload.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) + fs.CopyTaskManager = tache.NewManager[*fs.CopyTask](tache.WithWorks(conf.Conf.Tasks.Copy.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Copy.MaxRetry)) + tool.DownloadTaskManager = tache.NewManager[*tool.DownloadTask](tache.WithWorks(conf.Conf.Tasks.Download.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Download.MaxRetry)) + tool.TransferTaskManager = tache.NewManager[*tool.TransferTask](tache.WithWorks(conf.Conf.Tasks.Transfer.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Transfer.MaxRetry)) +} diff --git a/internal/conf/config.go b/internal/conf/config.go index de26e1fe0c4..2754064ca4b 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -8,15 +8,15 @@ import ( ) type Database struct { - Type string `json:"type" env:"DB_TYPE"` - Host string `json:"host" env:"DB_HOST"` - Port int `json:"port" env:"DB_PORT"` - User string `json:"user" env:"DB_USER"` - Password string `json:"password" env:"DB_PASS"` - Name string `json:"name" env:"DB_NAME"` - DBFile string `json:"db_file" env:"DB_FILE"` - TablePrefix string `json:"table_prefix" env:"DB_TABLE_PREFIX"` - SSLMode string `json:"ssl_mode" env:"DB_SSL_MODE"` + Type string `json:"type" env:"TYPE"` + Host string `json:"host" env:"HOST"` + Port int `json:"port" env:"PORT"` + User string `json:"user" env:"USER"` + Password string `json:"password" env:"PASS"` + Name string `json:"name" env:"NAME"` + DBFile string `json:"db_file" env:"FILE"` + TablePrefix string `json:"table_prefix" env:"TABLE_PREFIX"` + SSLMode string `json:"ssl_mode" env:"SSL_MODE"` } type Scheme struct { @@ -39,21 +39,34 @@ type LogConfig struct { Compress bool `json:"compress" env:"COMPRESS"` } +type TaskConfig struct { + Workers int `json:"workers" env:"WORKERS"` + MaxRetry int `json:"max_retry" env:"MAX_RETRY"` +} + +type TasksConfig struct { + Download TaskConfig `json:"download" envPrefix:"DOWNLOAD_"` + Transfer TaskConfig `json:"transfer" envPrefix:"TRANSFER_"` + Upload TaskConfig `json:"upload" envPrefix:"UPLOAD_"` + Copy TaskConfig `json:"copy" envPrefix:"COPY_"` +} + type Config struct { - Force bool `json:"force" env:"FORCE"` - SiteURL string `json:"site_url" env:"SITE_URL"` - Cdn string `json:"cdn" env:"CDN"` - JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` - TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` - Database Database `json:"database"` - Scheme Scheme `json:"scheme"` - TempDir string `json:"temp_dir" env:"TEMP_DIR"` - BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` - DistDir string `json:"dist_dir"` - Log LogConfig `json:"log"` - DelayedStart int `json:"delayed_start" env:"DELAYED_START"` - MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` - TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` + Force bool `json:"force" env:"FORCE"` + SiteURL string `json:"site_url" env:"SITE_URL"` + Cdn string `json:"cdn" env:"CDN"` + JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` + TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` + Database Database `json:"database" envPrefix:"DB_"` + Scheme Scheme `json:"scheme"` + TempDir string `json:"temp_dir" env:"TEMP_DIR"` + BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` + DistDir string `json:"dist_dir"` + Log LogConfig `json:"log"` + DelayedStart int `json:"delayed_start" env:"DELAYED_START"` + MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` + TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` + Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"` } func DefaultConfig() *Config { @@ -90,5 +103,22 @@ func DefaultConfig() *Config { }, MaxConnections: 0, TlsInsecureSkipVerify: true, + Tasks: TasksConfig{ + Download: TaskConfig{ + Workers: 5, + MaxRetry: 1, + }, + Transfer: TaskConfig{ + Workers: 5, + MaxRetry: 2, + }, + Upload: TaskConfig{ + Workers: 5, + }, + Copy: TaskConfig{ + Workers: 5, + MaxRetry: 2, + }, + }, } } diff --git a/internal/fs/copy.go b/internal/fs/copy.go index 911e528ab1a..43e163966ff 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -35,7 +35,7 @@ func (t *CopyTask) Run() error { return copyBetween2Storages(t, t.srcStorage, t.dstStorage, t.srcObjPath, t.dstDirPath) } -var CopyTaskManager = tache.NewManager[*CopyTask]() +var CopyTaskManager *tache.Manager[*CopyTask] // Copy if in the same storage, call move method // if not, add copy task diff --git a/internal/fs/put.go b/internal/fs/put.go index 5c154756d14..43d41acf0fd 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -30,7 +30,7 @@ func (t *UploadTask) Run() error { return op.Put(t.Ctx(), t.storage, t.dstDirActualPath, t.file, t.SetProgress, true) } -var UploadTaskManager = tache.NewManager[*UploadTask]() +var UploadTaskManager *tache.Manager[*UploadTask] // putAsTask add as a put task and return immediately func putAsTask(dstDirPath string, file model.FileStreamer) error { diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 7b536762e37..36ab6c82d4a 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -143,5 +143,5 @@ func (t *DownloadTask) GetStatus() string { } var ( - DownloadTaskManager *tache.Manager[*DownloadTask] = tache.NewManager[*DownloadTask]() + DownloadTaskManager *tache.Manager[*DownloadTask] ) diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index f7d1791d72f..c39e4ba0880 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -62,5 +62,5 @@ func (t *TransferTask) GetStatus() string { } var ( - TransferTaskManager *tache.Manager[*TransferTask] = tache.NewManager[*TransferTask]() + TransferTaskManager *tache.Manager[*TransferTask] ) diff --git a/server/handles/task.go b/server/handles/task.go index 1193ce05884..acfa1b02d04 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -18,13 +18,17 @@ type TaskInfo struct { } func getTaskInfo[T tache.TaskWithInfo](task T) TaskInfo { + errMsg := "" + if task.GetErr() != nil { + errMsg = task.GetErr().Error() + } return TaskInfo{ ID: task.GetID(), Name: task.GetName(), State: task.GetState(), Status: task.GetStatus(), Progress: task.GetProgress(), - Error: task.GetErr().Error(), + Error: errMsg, } } From b2890f05ab5a6bbbec14c84d23b77768df73538c Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 21 Nov 2023 15:54:42 +0800 Subject: [PATCH 025/659] feat: retry all failed task (close #5242) --- server/handles/task.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/handles/task.go b/server/handles/task.go index acfa1b02d04..0429116a44b 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -71,6 +71,10 @@ func taskRoute[T tache.TaskWithInfo](g *gin.RouterGroup, manager *tache.Manager[ manager.RemoveByState(tache.StateSucceeded) common.SuccessResp(c) }) + g.POST("/retry_failed", func(c *gin.Context) { + manager.RetryAllFailed() + common.SuccessResp(c) + }) } func SetupTaskRoute(g *gin.RouterGroup) { From d7f66138ebb9d9104dc32ba485047f795777c4e8 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 22 Nov 2023 15:09:39 +0800 Subject: [PATCH 026/659] docs: add sponsor `VidHub ` [skip ci] --- README.md | 7 ++++--- README_cn.md | 7 ++++--- README_ja.md | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 3f8fc4eebbd..757bc740b37 100644 --- a/README.md +++ b/README.md @@ -112,9 +112,10 @@ https://alist.nn.ci/guide/sponsor.html ### Special sponsors -- [亚洲云 - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商](https://www.asiayun.com/aff/QQCOOQKZ) (sponsored Chinese API server) -- [找资源 - 阿里云盘资源搜索引擎](https://zhaoziyuan.pw/) -- [JetBrains: Essential tools for software developers and teams](https://www.jetbrains.com/) +- [VidHub](https://okaapps.com/product/1659622164?ref=alist) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. +- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) +- [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 +- [JetBrains](https://www.jetbrains.com/) - Essential tools for software developers and teams ## Contributors diff --git a/README_cn.md b/README_cn.md index 6c7100d0eca..848e21a8e62 100644 --- a/README_cn.md +++ b/README_cn.md @@ -110,9 +110,10 @@ AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我 ### 特别赞助 -- [亚洲云 - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商](https://www.asiayun.com/aff/QQCOOQKZ) (国内API服务器赞助) -- [找资源 - 阿里云盘资源搜索引擎](https://zhaoziyuan.pw/) -- [JetBrains: Essential tools for software developers and teams](https://www.jetbrains.com/) +- [VidHub](https://zh.okaapps.com/product/1659622164?ref=alist) - 苹果生态下优雅的网盘视频播放器，iPhone，iPad，Mac，Apple TV全平台支持。 +- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (国内API服务器赞助) +- [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 +- [JetBrains](https://www.jetbrains.com/) - Essential tools for software developers and teams ## 贡献者 diff --git a/README_ja.md b/README_ja.md index 0efcf4e33b7..fc639fdbc2a 100644 --- a/README_ja.md +++ b/README_ja.md @@ -112,9 +112,10 @@ https://alist.nn.ci/guide/sponsor.html ### スペシャルスポンサー -- [亚洲云 - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商](https://www.asiayun.com/aff/QQCOOQKZ) (sponsored Chinese API server) -- [找资源 - 阿里云盘资源搜索引擎](https://zhaoziyuan.pw/) -- [JetBrains: Essential tools for software developers and teams](https://www.jetbrains.com/) +- [VidHub](https://okaapps.com/product/1659622164?ref=alist) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. +- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) +- [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 +- [JetBrains](https://www.jetbrains.com/) - Essential tools for software developers and teams ## コントリビューター From 12800704381cb369b393310da3662802e1278e83 Mon Sep 17 00:00:00 2001 From: zhangxiang <31364579+msterzhang@users.noreply.github.com> Date: Thu, 23 Nov 2023 21:40:16 +0800 Subject: [PATCH 027/659] feat: add chaoxing and vtencent driver (#5526 close #3347) * add chaoxing and vtencent * add vtencent put file * add sha1 to transfer files instantly * simplified upload file code * setting onlyproxy * fix get files modifyDate bug --- drivers/all.go | 2 + drivers/chaoxing/driver.go | 297 ++++++++++++++++++++++++++++++++++ drivers/chaoxing/meta.go | 47 ++++++ drivers/chaoxing/types.go | 263 ++++++++++++++++++++++++++++++ drivers/chaoxing/util.go | 179 ++++++++++++++++++++ drivers/vtencent/drive.go | 203 +++++++++++++++++++++++ drivers/vtencent/meta.go | 39 +++++ drivers/vtencent/signature.go | 33 ++++ drivers/vtencent/types.go | 252 +++++++++++++++++++++++++++++ drivers/vtencent/util.go | 289 +++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 2 - 12 files changed, 1605 insertions(+), 3 deletions(-) create mode 100644 drivers/chaoxing/driver.go create mode 100644 drivers/chaoxing/meta.go create mode 100644 drivers/chaoxing/types.go create mode 100644 drivers/chaoxing/util.go create mode 100644 drivers/vtencent/drive.go create mode 100644 drivers/vtencent/meta.go create mode 100644 drivers/vtencent/signature.go create mode 100644 drivers/vtencent/types.go create mode 100644 drivers/vtencent/util.go diff --git a/drivers/all.go b/drivers/all.go index 4f7fa839be0..40666028f11 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -18,6 +18,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_netdisk" _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" + _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/crypt" _ "github.com/alist-org/alist/v3/drivers/dropbox" @@ -46,6 +47,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/url_tree" _ "github.com/alist-org/alist/v3/drivers/uss" _ "github.com/alist-org/alist/v3/drivers/virtual" + _ "github.com/alist-org/alist/v3/drivers/vtencent" _ "github.com/alist-org/alist/v3/drivers/webdav" _ "github.com/alist-org/alist/v3/drivers/weiyun" _ "github.com/alist-org/alist/v3/drivers/wopan" diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go new file mode 100644 index 00000000000..143235fa481 --- /dev/null +++ b/drivers/chaoxing/driver.go @@ -0,0 +1,297 @@ +package chaoxing + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "google.golang.org/appengine/log" +) + +type ChaoXing struct { + model.Storage + Addition + cron *cron.Cron + config driver.Config + conf Conf +} + +func (d *ChaoXing) Config() driver.Config { + return d.config +} + +func (d *ChaoXing) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ChaoXing) refreshCookie() error { + cookie, err := d.Login() + if err != nil { + d.Status = err.Error() + op.MustSaveDriverStorage(d) + return nil + } + d.Addition.Cookie = cookie + op.MustSaveDriverStorage(d) + return nil +} + +func (d *ChaoXing) Init(ctx context.Context) error { + err := d.refreshCookie() + if err != nil { + log.Errorf(ctx, err.Error()) + } + d.cron = cron.NewCron(time.Hour * 12) + d.cron.Do(func() { + err = d.refreshCookie() + if err != nil { + log.Errorf(ctx, err.Error()) + } + }) + return nil +} + +func (d *ChaoXing) Drop(ctx context.Context) error { + d.cron.Stop() + return nil +} + +func (d *ChaoXing) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.GetFiles(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *ChaoXing) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var resp DownResp + ua := d.conf.ua + fileId := strings.Split(file.GetID(), "$")[1] + _, err := d.requestDownload("/screen/note_note/files/status/"+fileId, http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua) + }, &resp) + if err != nil { + return nil, err + } + u := resp.Download + return &model.Link{ + URL: u, + Header: http.Header{ + "Cookie": []string{d.Cookie}, + "Referer": []string{d.conf.referer}, + "User-Agent": []string{ua}, + }, + Concurrency: 2, + PartSize: 10 * utils.MB, + }, nil +} + +func (d *ChaoXing) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "name": dirName, + "pid": parentDir.GetID(), + } + var resp ListFileResp + _, err := d.request("/pc/resource/addResourceFolder", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return err + } + if resp.Result != 1 { + msg := fmt.Sprintf("error:%s", resp.Msg) + return errors.New(msg) + } + return nil +} + +func (d *ChaoXing) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderIds": srcObj.GetID(), + "targetId": dstDir.GetID(), + } + if !srcObj.IsDir() { + query = map[string]string{ + "bbsid": d.Addition.Bbsid, + "recIds": strings.Split(srcObj.GetID(), "$")[0], + "targetId": dstDir.GetID(), + } + } + var resp ListFileResp + _, err := d.request("/pc/resource/moveResource", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return err + } + if !resp.Status { + msg := fmt.Sprintf("error:%s", resp.Msg) + return errors.New(msg) + } + return nil +} + +func (d *ChaoXing) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderId": srcObj.GetID(), + "name": newName, + } + path := "/pc/resource/updateResourceFolderName" + if !srcObj.IsDir() { + // path = "/pc/resource/updateResourceFileName" + // query = map[string]string{ + // "bbsid": d.Addition.Bbsid, + // "recIds": strings.Split(srcObj.GetID(), "$")[0], + // "name": newName, + // } + return errors.New("此网盘不支持修改文件名") + } + var resp ListFileResp + _, err := d.request(path, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return err + } + if resp.Result != 1 { + msg := fmt.Sprintf("error:%s", resp.Msg) + return errors.New(msg) + } + return nil +} + +func (d *ChaoXing) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + // TODO copy obj, optional + return errs.NotImplement +} + +func (d *ChaoXing) Remove(ctx context.Context, obj model.Obj) error { + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderIds": obj.GetID(), + } + path := "/pc/resource/deleteResourceFolder" + var resp ListFileResp + if !obj.IsDir() { + path = "/pc/resource/deleteResourceFile" + query = map[string]string{ + "bbsid": d.Addition.Bbsid, + "recIds": strings.Split(obj.GetID(), "$")[0], + } + } + _, err := d.request(path, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return err + } + if resp.Result != 1 { + msg := fmt.Sprintf("error:%s", resp.Msg) + return errors.New(msg) + } + return nil +} + +func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + var resp UploadDataRsp + _, err := d.request("https://noteyd.chaoxing.com/pc/files/getUploadConfig", http.MethodGet, func(req *resty.Request) { + }, &resp) + if err != nil { + return err + } + if resp.Result != 1 { + return errors.New("get upload data error") + } + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + filePart, err := writer.CreateFormFile("file", stream.GetName()) + if err != nil { + return err + } + _, err = io.Copy(filePart, stream) + if err != nil { + return err + } + err = writer.WriteField("_token", resp.Msg.Token) + if err != nil { + return err + } + err = writer.WriteField("puid", fmt.Sprintf("%d", resp.Msg.Puid)) + if err != nil { + fmt.Println("Error writing param2 to request body:", err) + return err + } + err = writer.Close() + if err != nil { + return err + } + req, err := http.NewRequest("POST", "https://pan-yz.chaoxing.com/upload", body) + if err != nil { + return err + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len())) + resps, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resps.Body.Close() + bodys, err := io.ReadAll(resps.Body) + if err != nil { + return err + } + var fileRsp UploadFileDataRsp + err = json.Unmarshal(bodys, &fileRsp) + if err != nil { + return err + } + if fileRsp.Msg != "success" { + return errors.New(fileRsp.Msg) + } + uploadDoneParam := UploadDoneParam{Key: fileRsp.ObjectID, Cataid: "100000019", Param: fileRsp.Data} + params, err := json.Marshal(uploadDoneParam) + if err != nil { + return err + } + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "pid": dstDir.GetID(), + "type": "yunpan", + "params": url.QueryEscape("[" + string(params) + "]"), + } + var respd ListFileResp + _, err = d.request("/pc/resource/addResource", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &respd) + if err != nil { + return err + } + if respd.Result != 1 { + msg := fmt.Sprintf("error:%v", resp.Msg) + return errors.New(msg) + } + return nil +} + +var _ driver.Driver = (*ChaoXing)(nil) diff --git a/drivers/chaoxing/meta.go b/drivers/chaoxing/meta.go new file mode 100644 index 00000000000..42f4164c353 --- /dev/null +++ b/drivers/chaoxing/meta.go @@ -0,0 +1,47 @@ +package chaoxing + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +// 此程序挂载的是超星小组网盘，需要代理才能只用； +// 登录超星后进入个人空间，进入小组，新建小组，点击进去。 +// url中就有bbsid的参数，系统限制单文件大小2G，没有总容量限制 +type Addition struct { + // 超星用户名及密码 + UserName string `json:"user_name" required:"true"` + Password string `json:"password" required:"true"` + // 从自己新建的小组url里获取 + Bbsid string `json:"bbsid" required:"true"` + driver.RootID + // 可不填，程序会自动登录获取 + Cookie string `json:"cookie"` +} + +type Conf struct { + ua string + referer string + api string + DowloadApi string +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ChaoXing{ + config: driver.Config{ + Name: "超星小组盘", + OnlyProxy: true, + OnlyLocal: true, + DefaultRoot: "-1", + NoOverwriteUpload: true, + }, + conf: Conf{ + ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch", + referer: "https://chaoxing.com/", + api: "https://groupweb.chaoxing.com", + DowloadApi: "https://noteyd.chaoxing.com", + }, + } + }) +} diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go new file mode 100644 index 00000000000..a1ce13c3019 --- /dev/null +++ b/drivers/chaoxing/types.go @@ -0,0 +1,263 @@ +package chaoxing + +import ( + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Resp struct { + Result int `json:"result"` +} + +type UserAuth struct { + GroupAuth struct { + AddData int `json:"addData"` + AddDataFolder int `json:"addDataFolder"` + AddLebel int `json:"addLebel"` + AddManager int `json:"addManager"` + AddMem int `json:"addMem"` + AddTopicFolder int `json:"addTopicFolder"` + AnonymousAddReply int `json:"anonymousAddReply"` + AnonymousAddTopic int `json:"anonymousAddTopic"` + BatchOperation int `json:"batchOperation"` + DelData int `json:"delData"` + DelDataFolder int `json:"delDataFolder"` + DelMem int `json:"delMem"` + DelTopicFolder int `json:"delTopicFolder"` + Dismiss int `json:"dismiss"` + ExamEnc string `json:"examEnc"` + GroupChat int `json:"groupChat"` + IsShowCircleChatButton int `json:"isShowCircleChatButton"` + IsShowCircleCloudButton int `json:"isShowCircleCloudButton"` + IsShowCompanyButton int `json:"isShowCompanyButton"` + Join int `json:"join"` + MemberShowRankSet int `json:"memberShowRankSet"` + ModifyDataFolder int `json:"modifyDataFolder"` + ModifyExpose int `json:"modifyExpose"` + ModifyName int `json:"modifyName"` + ModifyShowPic int `json:"modifyShowPic"` + ModifyTopicFolder int `json:"modifyTopicFolder"` + ModifyVisibleState int `json:"modifyVisibleState"` + OnlyMgrScoreSet int `json:"onlyMgrScoreSet"` + Quit int `json:"quit"` + SendNotice int `json:"sendNotice"` + ShowActivityManage int `json:"showActivityManage"` + ShowActivitySet int `json:"showActivitySet"` + ShowAttentionSet int `json:"showAttentionSet"` + ShowAutoClearStatus int `json:"showAutoClearStatus"` + ShowBarcode int `json:"showBarcode"` + ShowChatRoomSet int `json:"showChatRoomSet"` + ShowCircleActivitySet int `json:"showCircleActivitySet"` + ShowCircleSet int `json:"showCircleSet"` + ShowCmem int `json:"showCmem"` + ShowDataFolder int `json:"showDataFolder"` + ShowDelReason int `json:"showDelReason"` + ShowForward int `json:"showForward"` + ShowGroupChat int `json:"showGroupChat"` + ShowGroupChatSet int `json:"showGroupChatSet"` + ShowGroupSquareSet int `json:"showGroupSquareSet"` + ShowLockAddSet int `json:"showLockAddSet"` + ShowManager int `json:"showManager"` + ShowManagerIdentitySet int `json:"showManagerIdentitySet"` + ShowNeedDelReasonSet int `json:"showNeedDelReasonSet"` + ShowNotice int `json:"showNotice"` + ShowOnlyManagerReplySet int `json:"showOnlyManagerReplySet"` + ShowRank int `json:"showRank"` + ShowRank2 int `json:"showRank2"` + ShowRecycleBin int `json:"showRecycleBin"` + ShowReplyByClass int `json:"showReplyByClass"` + ShowReplyNeedCheck int `json:"showReplyNeedCheck"` + ShowSignbanSet int `json:"showSignbanSet"` + ShowSpeechSet int `json:"showSpeechSet"` + ShowTopicCheck int `json:"showTopicCheck"` + ShowTopicNeedCheck int `json:"showTopicNeedCheck"` + ShowTransferSet int `json:"showTransferSet"` + } `json:"groupAuth"` + OperationAuth struct { + Add int `json:"add"` + AddTopicToFolder int `json:"addTopicToFolder"` + ChoiceSet int `json:"choiceSet"` + DelTopicFromFolder int `json:"delTopicFromFolder"` + Delete int `json:"delete"` + Reply int `json:"reply"` + ScoreSet int `json:"scoreSet"` + TopSet int `json:"topSet"` + Update int `json:"update"` + } `json:"operationAuth"` +} + +type File struct { + Cataid int `json:"cataid"` + Cfid int `json:"cfid"` + Content struct { + Cfid int `json:"cfid"` + Pid int `json:"pid"` + FolderName string `json:"folderName"` + ShareType int `json:"shareType"` + Preview string `json:"preview"` + Filetype string `json:"filetype"` + PreviewURL string `json:"previewUrl"` + IsImg bool `json:"isImg"` + ParentPath string `json:"parentPath"` + Icon string `json:"icon"` + Suffix string `json:"suffix"` + Duration int `json:"duration"` + Pantype string `json:"pantype"` + Puid int `json:"puid"` + Filepath string `json:"filepath"` + Crc string `json:"crc"` + Isfile bool `json:"isfile"` + Residstr string `json:"residstr"` + ObjectID string `json:"objectId"` + Extinfo string `json:"extinfo"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + ResTypeValue int `json:"resTypeValue"` + UploadDateFormat string `json:"uploadDateFormat"` + DisableOpt bool `json:"disableOpt"` + DownPath string `json:"downPath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + Restype string `json:"restype"` + Size int `json:"size"` + UploadDate string `json:"uploadDate"` + FileSize string `json:"fileSize"` + Name string `json:"name"` + FileID string `json:"fileId"` + } `json:"content"` + CreatorID int `json:"creatorId"` + DesID string `json:"des_id"` + ID int `json:"id"` + Inserttime int64 `json:"inserttime"` + Key string `json:"key"` + Norder int `json:"norder"` + OwnerID int `json:"ownerId"` + OwnerType int `json:"ownerType"` + Path string `json:"path"` + Rid int `json:"rid"` + Status int `json:"status"` + Topsign int `json:"topsign"` +} + +type ListFileResp struct { + Msg string `json:"msg"` + Result int `json:"result"` + Status bool `json:"status"` + UserAuth UserAuth `json:"userAuth"` + List []File `json:"list"` +} + +type DownResp struct { + Msg string `json:"msg"` + Duration int `json:"duration"` + Download string `json:"download"` + FileStatus string `json:"fileStatus"` + URL string `json:"url"` + Status bool `json:"status"` +} + +type UploadDataRsp struct { + Result int `json:"result"` + Msg struct { + Puid int `json:"puid"` + Token string `json:"token"` + } `json:"msg"` +} + +type UploadFileDataRsp struct { + Result bool `json:"result"` + Msg string `json:"msg"` + Crc string `json:"crc"` + ObjectID string `json:"objectId"` + Resid int64 `json:"resid"` + Puid int `json:"puid"` + Data struct { + DisableOpt bool `json:"disableOpt"` + Resid int64 `json:"resid"` + Crc string `json:"crc"` + Puid int `json:"puid"` + Isfile bool `json:"isfile"` + Pantype string `json:"pantype"` + Size int `json:"size"` + Name string `json:"name"` + ObjectID string `json:"objectId"` + Restype string `json:"restype"` + UploadDate time.Time `json:"uploadDate"` + ModifyDate time.Time `json:"modifyDate"` + UploadDateFormat string `json:"uploadDateFormat"` + Residstr string `json:"residstr"` + Suffix string `json:"suffix"` + Preview string `json:"preview"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + Duration int `json:"duration"` + IsImg bool `json:"isImg"` + PreviewURL string `json:"previewUrl"` + Filetype string `json:"filetype"` + Filepath string `json:"filepath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + ResTypeValue int `json:"resTypeValue"` + Extinfo string `json:"extinfo"` + } `json:"data"` +} + + +type UploadDoneParam struct { + Cataid string `json:"cataid"` + Key string `json:"key"` + Param struct { + DisableOpt bool `json:"disableOpt"` + Resid int64 `json:"resid"` + Crc string `json:"crc"` + Puid int `json:"puid"` + Isfile bool `json:"isfile"` + Pantype string `json:"pantype"` + Size int `json:"size"` + Name string `json:"name"` + ObjectID string `json:"objectId"` + Restype string `json:"restype"` + UploadDate time.Time `json:"uploadDate"` + ModifyDate time.Time `json:"modifyDate"` + UploadDateFormat string `json:"uploadDateFormat"` + Residstr string `json:"residstr"` + Suffix string `json:"suffix"` + Preview string `json:"preview"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + Duration int `json:"duration"` + IsImg bool `json:"isImg"` + PreviewURL string `json:"previewUrl"` + Filetype string `json:"filetype"` + Filepath string `json:"filepath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + ResTypeValue int `json:"resTypeValue"` + Extinfo string `json:"extinfo"` + } `json:"param"` +} + +func fileToObj(f File) *model.Object { + if len(f.Content.FolderName) > 0 { + return &model.Object{ + ID: fmt.Sprintf("%d", f.ID), + Name: f.Content.FolderName, + Size: 0, + Modified: time.UnixMilli(f.Inserttime), + IsFolder: true, + } + } + paserTime, err := time.Parse("2006-01-02 15:04", f.Content.UploadDate) + if err != nil { + paserTime = time.Now() + } + return &model.Object{ + ID: fmt.Sprintf("%d$%s", f.ID, f.Content.FileID), + Name: f.Content.Name, + Size: int64(f.Content.Size), + Modified: paserTime, + IsFolder: false, + } +} diff --git a/drivers/chaoxing/util.go b/drivers/chaoxing/util.go new file mode 100644 index 00000000000..2e34994dd90 --- /dev/null +++ b/drivers/chaoxing/util.go @@ -0,0 +1,179 @@ +package chaoxing + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "errors" + "fmt" + "mime/multipart" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" +) + +func (d *ChaoXing) requestDownload(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + u := d.conf.DowloadApi + pathname + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "Accept": "application/json, text/plain, */*", + "Referer": d.conf.referer, + }) + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Resp + req.SetError(&e) + res, err := req.Execute(method, u) + if err != nil { + return nil, err + } + return res.Body(), nil +} + +func (d *ChaoXing) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + u := d.conf.api + pathname + if strings.Contains(pathname, "getUploadConfig") { + u = pathname + } + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "Accept": "application/json, text/plain, */*", + "Referer": d.conf.referer, + }) + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Resp + req.SetError(&e) + res, err := req.Execute(method, u) + if err != nil { + return nil, err + } + return res.Body(), nil +} + +func (d *ChaoXing) GetFiles(parent string) ([]File, error) { + files := make([]File, 0) + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderId": parent, + "recType": "1", + } + var resp ListFileResp + _, err := d.request("/pc/resource/getResourceList", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return nil, err + } + if resp.Result != 1 { + msg:=fmt.Sprintf("error code is:%d", resp.Result) + return nil, errors.New(msg) + } + if len(resp.List) > 0 { + files = append(files, resp.List...) + } + querys := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderId": parent, + "recType": "2", + } + var resps ListFileResp + _, err = d.request("/pc/resource/getResourceList", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(querys) + }, &resps) + if err != nil { + return nil, err + } + if len(resps.List) > 0 { + files = append(files, resps.List...) + } + return files, nil +} + +func EncryptByAES(message, key string) (string, error) { + aesKey := []byte(key) + plainText := []byte(message) + block, err := aes.NewCipher(aesKey) + if err != nil { + return "", err + } + iv := aesKey[:aes.BlockSize] + mode := cipher.NewCBCEncrypter(block, iv) + padding := aes.BlockSize - len(plainText)%aes.BlockSize + paddedText := append(plainText, byte(padding)) + for i := 0; i < padding-1; i++ { + paddedText = append(paddedText, byte(padding)) + } + ciphertext := make([]byte, len(paddedText)) + mode.CryptBlocks(ciphertext, paddedText) + encrypted := base64.StdEncoding.EncodeToString(ciphertext) + return encrypted, nil +} + +func CookiesToString(cookies []*http.Cookie) string { + var cookieStr string + for _, cookie := range cookies { + cookieStr += cookie.Name + "=" + cookie.Value + "; " + } + if len(cookieStr) > 2 { + cookieStr = cookieStr[:len(cookieStr)-2] + } + return cookieStr +} + +func (d *ChaoXing) Login() (string, error) { + transferKey := "u2oh6Vu^HWe4_AES" + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + uname, err := EncryptByAES(d.Addition.UserName, transferKey) + if err != nil { + return "", err + } + password, err := EncryptByAES(d.Addition.Password, transferKey) + if err != nil { + return "", err + } + err = writer.WriteField("uname", uname) + if err != nil { + return "", err + } + err = writer.WriteField("password", password) + if err != nil { + return "", err + } + err = writer.WriteField("t", "true") + if err != nil { + return "", err + } + err = writer.Close() + if err != nil { + return "", err + } + // Create the request + req, err := http.NewRequest("POST", "https://passport2.chaoxing.com/fanyalogin", body) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len())) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + return CookiesToString(resp.Cookies()), nil + +} diff --git a/drivers/vtencent/drive.go b/drivers/vtencent/drive.go new file mode 100644 index 00000000000..b6dd13b27d9 --- /dev/null +++ b/drivers/vtencent/drive.go @@ -0,0 +1,203 @@ +package vtencent + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type Vtencent struct { + model.Storage + Addition + cron *cron.Cron + config driver.Config + conf Conf +} + +func (d *Vtencent) Config() driver.Config { + return d.config +} + +func (d *Vtencent) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Vtencent) Init(ctx context.Context) error { + tfUid, err := d.LoadUser() + if err != nil { + d.Status = err.Error() + op.MustSaveDriverStorage(d) + return nil + } + d.Addition.TfUid = tfUid + op.MustSaveDriverStorage(d) + d.cron = cron.NewCron(time.Hour * 12) + d.cron.Do(func() { + _, err := d.LoadUser() + if err != nil { + d.Status = err.Error() + op.MustSaveDriverStorage(d) + } + }) + return nil +} + +func (d *Vtencent) Drop(ctx context.Context) error { + d.cron.Stop() + return nil +} + +func (d *Vtencent) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.GetFiles(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *Vtencent) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + form := fmt.Sprintf(`{"MaterialIds":["%s"]}`, file.GetID()) + var dat map[string]interface{} + if err := json.Unmarshal([]byte(form), &dat); err != nil { + return nil, err + } + var resps RspDown + api := "https://api.vs.tencent.com/SaaS/Material/DescribeMaterialDownloadUrl" + rsp, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(dat) + }, &resps) + if err != nil { + return nil, err + } + if err := json.Unmarshal(rsp, &resps); err != nil { + return nil, err + } + if len(resps.Data.DownloadURLInfoSet) == 0 { + return nil, err + } + u := resps.Data.DownloadURLInfoSet[0].DownloadURL + return &model.Link{ + URL: u, + Header: http.Header{ + "Referer": []string{d.conf.referer}, + "User-Agent": []string{d.conf.ua}, + }, + Concurrency: 2, + PartSize: 10 * utils.MB, + }, nil +} + +func (d *Vtencent) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + classId, err := strconv.Atoi(parentDir.GetID()) + if err != nil { + return err + } + _, err = d.request("https://api.vs.tencent.com/PaaS/Material/CreateClass", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "Owner": base.Json{ + "Type": "PERSON", + "Id": d.TfUid, + }, + "ParentClassId": classId, + "Name": dirName, + "VerifySign": ""}) + }, nil) + return err +} + +func (d *Vtencent) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + srcType := "MATERIAL" + if srcObj.IsDir() { + srcType = "CLASS" + } + form := fmt.Sprintf(`{"SourceInfos":[ + {"Owner":{"Id":"%s","Type":"PERSON"}, + "Resource":{"Type":"%s","Id":"%s"}} + ], + "Destination":{"Owner":{"Id":"%s","Type":"PERSON"}, + "Resource":{"Type":"CLASS","Id":"%s"}} + }`, d.TfUid, srcType, srcObj.GetID(), d.TfUid, dstDir.GetID()) + var dat map[string]interface{} + if err := json.Unmarshal([]byte(form), &dat); err != nil { + return err + } + _, err := d.request("https://api.vs.tencent.com/PaaS/Material/MoveResource", http.MethodPost, func(req *resty.Request) { + req.SetBody(dat) + }, nil) + return err +} + +func (d *Vtencent) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + api := "https://api.vs.tencent.com/PaaS/Material/ModifyMaterial" + form := fmt.Sprintf(`{ + "Owner":{"Type":"PERSON","Id":"%s"}, + "MaterialId":"%s","Name":"%s"}`, d.TfUid, srcObj.GetID(), newName) + if srcObj.IsDir() { + classId, err := strconv.Atoi(srcObj.GetID()) + if err != nil { + return err + } + api = "https://api.vs.tencent.com/PaaS/Material/ModifyClass" + form = fmt.Sprintf(`{"Owner":{"Type":"PERSON","Id":"%s"}, + "ClassId":%d,"Name":"%s"}`, d.TfUid, classId, newName) + } + var dat map[string]interface{} + if err := json.Unmarshal([]byte(form), &dat); err != nil { + return err + } + _, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(dat) + }, nil) + return err +} + +func (d *Vtencent) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + // TODO copy obj, optional + return errs.NotImplement +} + +func (d *Vtencent) Remove(ctx context.Context, obj model.Obj) error { + srcType := "MATERIAL" + if obj.IsDir() { + srcType = "CLASS" + } + form := fmt.Sprintf(`{ + "SourceInfos":[ + {"Owner":{"Type":"PERSON","Id":"%s"}, + "Resource":{"Type":"%s","Id":"%s"}} + ] + }`, d.TfUid, srcType, obj.GetID()) + var dat map[string]interface{} + if err := json.Unmarshal([]byte(form), &dat); err != nil { + return err + } + _, err := d.request("https://api.vs.tencent.com/PaaS/Material/DeleteResource", http.MethodPost, func(req *resty.Request) { + req.SetBody(dat) + }, nil) + return err +} + +func (d *Vtencent) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + err := d.FileUpload(ctx, dstDir, stream, up) + return err +} + +//func (d *Vtencent) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Vtencent)(nil) diff --git a/drivers/vtencent/meta.go b/drivers/vtencent/meta.go new file mode 100644 index 00000000000..e78db685c97 --- /dev/null +++ b/drivers/vtencent/meta.go @@ -0,0 +1,39 @@ +package vtencent + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Cookie string `json:"cookie" required:"true"` + TfUid string `json:"tf_uid"` + OrderBy string `json:"order_by" type:"select" options:"Name,Size,UpdateTime,CreatTime"` + OrderDirection string `json:"order_direction" type:"select" options:"Asc,Desc"` +} + +type Conf struct { + ua string + referer string + origin string +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Vtencent{ + config: driver.Config{ + Name: "腾讯智能创作平台", + OnlyProxy: true, + OnlyLocal: true, + DefaultRoot: "9", + NoOverwriteUpload: true, + }, + conf: Conf{ + ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch", + referer: "https://app.v.tencent.com/", + origin: "https://app.v.tencent.com", + }, + } + }) +} diff --git a/drivers/vtencent/signature.go b/drivers/vtencent/signature.go new file mode 100644 index 00000000000..14fda9bdc21 --- /dev/null +++ b/drivers/vtencent/signature.go @@ -0,0 +1,33 @@ +package vtencent + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/hex" +) + +func QSignatureKey(timeKey string, signPath string, key string) string { + signKey := hmac.New(sha1.New, []byte(key)) + signKey.Write([]byte(timeKey)) + signKeyBytes := signKey.Sum(nil) + signKeyHex := hex.EncodeToString(signKeyBytes) + sha := sha1.New() + sha.Write([]byte(signPath)) + shaBytes := sha.Sum(nil) + shaHex := hex.EncodeToString(shaBytes) + + O := "sha1\n" + timeKey + "\n" + shaHex + "\n" + dataSignKey := hmac.New(sha1.New, []byte(signKeyHex)) + dataSignKey.Write([]byte(O)) + dataSignKeyBytes := dataSignKey.Sum(nil) + dataSignKeyHex := hex.EncodeToString(dataSignKeyBytes) + return dataSignKeyHex +} + +func QTwoSignatureKey(timeKey string, key string) string { + signKey := hmac.New(sha1.New, []byte(key)) + signKey.Write([]byte(timeKey)) + signKeyBytes := signKey.Sum(nil) + signKeyHex := hex.EncodeToString(signKeyBytes) + return signKeyHex +} diff --git a/drivers/vtencent/types.go b/drivers/vtencent/types.go new file mode 100644 index 00000000000..b967481e253 --- /dev/null +++ b/drivers/vtencent/types.go @@ -0,0 +1,252 @@ +package vtencent + +import ( + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type RespErr struct { + Code string `json:"Code"` + Message string `json:"Message"` +} + +type Reqfiles struct { + ScrollToken string `json:"ScrollToken"` + Text string `json:"Text"` + Offset int `json:"Offset"` + Limit int `json:"Limit"` + Sort struct { + Field string `json:"Field"` + Order string `json:"Order"` + } `json:"Sort"` + CreateTimeRanges []any `json:"CreateTimeRanges"` + MaterialTypes []any `json:"MaterialTypes"` + ReviewStatuses []any `json:"ReviewStatuses"` + Tags []any `json:"Tags"` + SearchScopes []struct { + Owner struct { + Type string `json:"Type"` + ID string `json:"Id"` + } `json:"Owner"` + ClassID int `json:"ClassId"` + SearchOneDepth bool `json:"SearchOneDepth"` + } `json:"SearchScopes"` +} + +type File struct { + Type string `json:"Type"` + ClassInfo struct { + ClassID int `json:"ClassId"` + Name string `json:"Name"` + UpdateTime time.Time `json:"UpdateTime"` + CreateTime time.Time `json:"CreateTime"` + FileInboxID string `json:"FileInboxId"` + Owner struct { + Type string `json:"Type"` + ID string `json:"Id"` + } `json:"Owner"` + ClassPath string `json:"ClassPath"` + ParentClassID int `json:"ParentClassId"` + AttachmentInfo struct { + SubClassCount int `json:"SubClassCount"` + MaterialCount int `json:"MaterialCount"` + Size int64 `json:"Size"` + } `json:"AttachmentInfo"` + ClassPreviewURLSet []string `json:"ClassPreviewUrlSet"` + } `json:"ClassInfo"` + MaterialInfo struct { + BasicInfo struct { + MaterialID string `json:"MaterialId"` + MaterialType string `json:"MaterialType"` + Name string `json:"Name"` + CreateTime time.Time `json:"CreateTime"` + UpdateTime time.Time `json:"UpdateTime"` + ClassPath string `json:"ClassPath"` + ClassID int `json:"ClassId"` + TagInfoSet []any `json:"TagInfoSet"` + TagSet []any `json:"TagSet"` + PreviewURL string `json:"PreviewUrl"` + MediaURL string `json:"MediaUrl"` + UnifiedMediaPreviewURL string `json:"UnifiedMediaPreviewUrl"` + Owner struct { + Type string `json:"Type"` + ID string `json:"Id"` + } `json:"Owner"` + PermissionSet any `json:"PermissionSet"` + PermissionInfoSet []any `json:"PermissionInfoSet"` + TfUID string `json:"TfUid"` + GroupID string `json:"GroupId"` + VersionMaterialIDSet []any `json:"VersionMaterialIdSet"` + FileType string `json:"FileType"` + CmeMaterialPlayList []any `json:"CmeMaterialPlayList"` + Status string `json:"Status"` + DownloadSwitch string `json:"DownloadSwitch"` + } `json:"BasicInfo"` + MediaInfo struct { + Width int `json:"Width"` + Height int `json:"Height"` + Size int `json:"Size"` + Duration float64 `json:"Duration"` + Fps int `json:"Fps"` + BitRate int `json:"BitRate"` + Codec string `json:"Codec"` + MediaType string `json:"MediaType"` + FavoriteStatus string `json:"FavoriteStatus"` + } `json:"MediaInfo"` + MaterialStatus struct { + ContentReviewStatus string `json:"ContentReviewStatus"` + EditorUsableStatus string `json:"EditorUsableStatus"` + UnifiedPreviewStatus string `json:"UnifiedPreviewStatus"` + EditPreviewImageSpiritStatus string `json:"EditPreviewImageSpiritStatus"` + TranscodeStatus string `json:"TranscodeStatus"` + AdaptiveStreamingStatus string `json:"AdaptiveStreamingStatus"` + StreamConnectable string `json:"StreamConnectable"` + AiAnalysisStatus string `json:"AiAnalysisStatus"` + AiRecognitionStatus string `json:"AiRecognitionStatus"` + } `json:"MaterialStatus"` + ImageMaterial struct { + Height int `json:"Height"` + Width int `json:"Width"` + Size int `json:"Size"` + MaterialURL string `json:"MaterialUrl"` + Resolution string `json:"Resolution"` + VodFileID string `json:"VodFileId"` + OriginalURL string `json:"OriginalUrl"` + } `json:"ImageMaterial"` + VideoMaterial struct { + MetaData struct { + Size int `json:"Size"` + Container string `json:"Container"` + Bitrate int `json:"Bitrate"` + Height int `json:"Height"` + Width int `json:"Width"` + Duration float64 `json:"Duration"` + Rotate int `json:"Rotate"` + VideoStreamInfoSet []struct { + Bitrate int `json:"Bitrate"` + Height int `json:"Height"` + Width int `json:"Width"` + Codec string `json:"Codec"` + Fps int `json:"Fps"` + } `json:"VideoStreamInfoSet"` + AudioStreamInfoSet []struct { + Bitrate int `json:"Bitrate"` + SamplingRate int `json:"SamplingRate"` + Codec string `json:"Codec"` + } `json:"AudioStreamInfoSet"` + } `json:"MetaData"` + ImageSpriteInfo any `json:"ImageSpriteInfo"` + MaterialURL string `json:"MaterialUrl"` + CoverURL string `json:"CoverUrl"` + Resolution string `json:"Resolution"` + VodFileID string `json:"VodFileId"` + OriginalURL string `json:"OriginalUrl"` + AudioWaveformURL string `json:"AudioWaveformUrl"` + SubtitleURL string `json:"SubtitleUrl"` + TranscodeInfoSet []any `json:"TranscodeInfoSet"` + ImageSpriteInfoSet []any `json:"ImageSpriteInfoSet"` + } `json:"VideoMaterial"` + } `json:"MaterialInfo"` +} + +type RspFiles struct { + Code string `json:"Code"` + Message string `json:"Message"` + EnglishMessage string `json:"EnglishMessage"` + Data struct { + TotalCount int `json:"TotalCount"` + ResourceInfoSet []File `json:"ResourceInfoSet"` + ScrollToken string `json:"ScrollToken"` + } `json:"Data"` +} + +type RspDown struct { + Code string `json:"Code"` + Message string `json:"Message"` + EnglishMessage string `json:"EnglishMessage"` + Data struct { + DownloadURLInfoSet []struct { + MaterialID string `json:"MaterialId"` + DownloadURL string `json:"DownloadUrl"` + } `json:"DownloadUrlInfoSet"` + } `json:"Data"` +} + +type RspCreatrMaterial struct { + Code string `json:"Code"` + Message string `json:"Message"` + EnglishMessage string `json:"EnglishMessage"` + Data struct { + UploadContext string `json:"UploadContext"` + VodUploadSign string `json:"VodUploadSign"` + QuickUpload bool `json:"QuickUpload"` + } `json:"Data"` +} + +type RspApplyUploadUGC struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Video struct { + StorageSignature string `json:"storageSignature"` + StoragePath string `json:"storagePath"` + } `json:"video"` + StorageAppID int `json:"storageAppId"` + StorageBucket string `json:"storageBucket"` + StorageRegion string `json:"storageRegion"` + StorageRegionV5 string `json:"storageRegionV5"` + Domain string `json:"domain"` + VodSessionKey string `json:"vodSessionKey"` + TempCertificate struct { + SecretID string `json:"secretId"` + SecretKey string `json:"secretKey"` + Token string `json:"token"` + ExpiredTime int `json:"expiredTime"` + } `json:"tempCertificate"` + AppID int `json:"appId"` + Timestamp int `json:"timestamp"` + StorageRegionV50 string `json:"StorageRegionV5"` + MiniProgramAccelerateHost string `json:"MiniProgramAccelerateHost"` + } `json:"data"` +} + +type RspCommitUploadUGC struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Video struct { + URL string `json:"url"` + VerifyContent string `json:"verify_content"` + } `json:"video"` + FileID string `json:"fileId"` + } `json:"data"` +} + +type RspFinishUpload struct { + Code string `json:"Code"` + Message string `json:"Message"` + EnglishMessage string `json:"EnglishMessage"` + Data struct { + MaterialID string `json:"MaterialId"` + } `json:"Data"` +} + +func fileToObj(f File) *model.Object { + obj := &model.Object{} + if f.Type == "CLASS" { + obj.Name = f.ClassInfo.Name + obj.ID = strconv.Itoa(f.ClassInfo.ClassID) + obj.IsFolder = true + obj.Modified = f.ClassInfo.CreateTime + obj.Size = 0 + } else if f.Type == "MATERIAL" { + obj.Name = f.MaterialInfo.BasicInfo.Name + obj.ID = f.MaterialInfo.BasicInfo.MaterialID + obj.IsFolder = false + obj.Modified = f.MaterialInfo.BasicInfo.CreateTime + obj.Size = int64(f.MaterialInfo.MediaInfo.Size) + } + return obj +} diff --git a/drivers/vtencent/util.go b/drivers/vtencent/util.go new file mode 100644 index 00000000000..ad69793e694 --- /dev/null +++ b/drivers/vtencent/util.go @@ -0,0 +1,289 @@ +package vtencent + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "path" + "strconv" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/go-resty/resty/v2" +) + +func (d *Vtencent) request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "cookie": d.Cookie, + "content-type": "application/json", + "origin": d.conf.origin, + "referer": d.conf.referer, + }) + if callback != nil { + callback(req) + } else { + req.SetBody("{}") + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + code := utils.Json.Get(res.Body(), "Code").ToString() + if code != "Success" { + switch code { + case "AuthFailure.SessionInvalid": + if err != nil { + return nil, errors.New(code) + } + default: + return nil, errors.New(code) + } + return d.request(url, method, callback, resp) + } + return res.Body(), nil +} + +func (d *Vtencent) ugcRequest(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "cookie": d.Cookie, + "content-type": "application/json", + "origin": d.conf.origin, + "referer": d.conf.referer, + }) + if callback != nil { + callback(req) + } else { + req.SetBody("{}") + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + code := utils.Json.Get(res.Body(), "Code").ToInt() + if code != 0 { + message := utils.Json.Get(res.Body(), "message").ToString() + if len(message) == 0 { + message = utils.Json.Get(res.Body(), "msg").ToString() + } + return nil, errors.New(message) + } + return res.Body(), nil +} + +func (d *Vtencent) LoadUser() (string, error) { + api := "https://api.vs.tencent.com/SaaS/Account/DescribeAccount" + res, err := d.request(api, http.MethodPost, func(req *resty.Request) {}, nil) + if err != nil { + return "", err + } + return utils.Json.Get(res, "Data", "TfUid").ToString(), nil +} + +func (d *Vtencent) GetFiles(dirId string) ([]File, error) { + api := "https://api.vs.tencent.com/PaaS/Material/SearchResource" + form := fmt.Sprintf(`{ + "Text":"", + "Text":"", + "Offset":0, + "Limit":20000, + "Sort":{"Field":"%s","Order":"%s"}, + "CreateTimeRanges":[], + "MaterialTypes":[], + "ReviewStatuses":[], + "Tags":[], + "SearchScopes":[{"Owner":{"Type":"PERSON","Id":"%s"},"ClassId":%s,"SearchOneDepth":true}] + }`, d.Addition.OrderBy, d.Addition.OrderDirection, d.TfUid, dirId) + var resps RspFiles + _, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return []File{}, err + } + return resps.Data.ResourceInfoSet, nil +} + +func (d *Vtencent) CreateUploadMaterial(classId int, fileName string, UploadSummaryKey string) (RspCreatrMaterial, error) { + api := "https://api.vs.tencent.com/PaaS/Material/CreateUploadMaterial" + form := base.Json{"Owner": base.Json{"Type": "PERSON", "Id": d.TfUid}, + "MaterialType": "VIDEO", "Name": fileName, "ClassId": classId, + "UploadSummaryKey": UploadSummaryKey} + var resps RspCreatrMaterial + _, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspCreatrMaterial{}, err + } + return resps, nil +} + +func (d *Vtencent) ApplyUploadUGC(signature string, stream model.FileStreamer) (RspApplyUploadUGC, error) { + api := "https://vod2.qcloud.com/v3/index.php?Action=ApplyUploadUGC" + form := base.Json{ + "signature": signature, + "videoName": stream.GetName(), + "videoType": strings.ReplaceAll(path.Ext(stream.GetName()), ".", ""), + "videoSize": stream.GetSize(), + } + var resps RspApplyUploadUGC + _, err := d.ugcRequest(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspApplyUploadUGC{}, err + } + return resps, nil +} + +func (d *Vtencent) CommitUploadUGC(signature string, vodSessionKey string) (RspCommitUploadUGC, error) { + api := "https://vod2.qcloud.com/v3/index.php?Action=CommitUploadUGC" + form := base.Json{ + "signature": signature, + "vodSessionKey": vodSessionKey, + } + var resps RspCommitUploadUGC + rsp, err := d.ugcRequest(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspCommitUploadUGC{}, err + } + if len(resps.Data.Video.URL) == 0 { + return RspCommitUploadUGC{}, errors.New(string(rsp)) + } + return resps, nil +} + +func (d *Vtencent) FinishUploadMaterial(SummaryKey string, VodVerifyKey string, UploadContext, VodFileId string) (RspFinishUpload, error) { + api := "https://api.vs.tencent.com/PaaS/Material/FinishUploadMaterial" + form := base.Json{ + "UploadContext": UploadContext, + "VodVerifyKey": VodVerifyKey, + "VodFileId": VodFileId, + "UploadFullKey": SummaryKey} + var resps RspFinishUpload + rsp, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspFinishUpload{}, err + } + if len(resps.Data.MaterialID) == 0 { + return RspFinishUpload{}, errors.New(string(rsp)) + } + return resps, nil +} + +func (d *Vtencent) FinishHashUploadMaterial(SummaryKey string, UploadContext string) (RspFinishUpload, error) { + api := "https://api.vs.tencent.com/PaaS/Material/FinishUploadMaterial" + var resps RspFinishUpload + form := base.Json{ + "UploadContext": UploadContext, + "UploadFullKey": SummaryKey} + rsp, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspFinishUpload{}, err + } + if len(resps.Data.MaterialID) == 0 { + return RspFinishUpload{}, errors.New(string(rsp)) + } + return resps, nil +} + +func (d *Vtencent) FileUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + classId, err := strconv.Atoi(dstDir.GetID()) + if err != nil { + return err + } + const chunkLength int64 = 1024 * 1024 * 10 + reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: chunkLength}) + if err != nil { + return err + } + chunkHash, err := utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + rspCreatrMaterial, err := d.CreateUploadMaterial(classId, stream.GetName(), chunkHash) + if err != nil { + return err + } + if rspCreatrMaterial.Data.QuickUpload { + SummaryKey := stream.GetHash().GetHash(utils.SHA1) + if len(SummaryKey) < utils.SHA1.Width { + if SummaryKey, err = utils.HashReader(utils.SHA1, stream); err != nil { + return err + } + } + UploadContext := rspCreatrMaterial.Data.UploadContext + _, err = d.FinishHashUploadMaterial(SummaryKey, UploadContext) + if err != nil { + return err + } + return nil + } + hash := sha1.New() + rspUGC, err := d.ApplyUploadUGC(rspCreatrMaterial.Data.VodUploadSign, stream) + if err != nil { + return err + } + params := rspUGC.Data + certificate := params.TempCertificate + cfg := &aws.Config{ + HTTPClient: base.HttpClient, + // S3ForcePathStyle: aws.Bool(true), + Credentials: credentials.NewStaticCredentials(certificate.SecretID, certificate.SecretKey, certificate.Token), + Region: aws.String(params.StorageRegionV5), + Endpoint: aws.String(fmt.Sprintf("cos.%s.myqcloud.com", params.StorageRegionV5)), + } + ss, err := session.NewSession(cfg) + if err != nil { + return err + } + uploader := s3manager.NewUploader(ss) + input := &s3manager.UploadInput{ + Bucket: aws.String(fmt.Sprintf("%s-%d", params.StorageBucket, params.StorageAppID)), + Key: ¶ms.Video.StoragePath, + Body: io.TeeReader(stream, io.MultiWriter(hash, driver.NewProgress(stream.GetSize(), up))), + } + _, err = uploader.UploadWithContext(ctx, input) + if err != nil { + return err + } + rspCommitUGC, err := d.CommitUploadUGC(rspCreatrMaterial.Data.VodUploadSign, rspUGC.Data.VodSessionKey) + if err != nil { + return err + } + VodVerifyKey := rspCommitUGC.Data.Video.VerifyContent + VodFileId := rspCommitUGC.Data.FileID + UploadContext := rspCreatrMaterial.Data.UploadContext + SummaryKey := hex.EncodeToString(hash.Sum(nil)) + _, err = d.FinishUploadMaterial(SummaryKey, VodVerifyKey, UploadContext, VodFileId) + if err != nil { + return err + } + return nil +} diff --git a/go.mod b/go.mod index e81d863f770..2fdd60f57bf 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( golang.org/x/net v0.17.0 golang.org/x/oauth2 v0.12.0 golang.org/x/time v0.3.0 + google.golang.org/appengine v1.6.7 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 gorm.io/driver/sqlite v1.4.4 @@ -194,7 +195,6 @@ require ( golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/api v0.134.0 // indirect - google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/grpc v1.57.0 // indirect google.golang.org/protobuf v1.31.0 // indirect diff --git a/go.sum b/go.sum index 7d1ccf7b542..55f23a459af 100644 --- a/go.sum +++ b/go.sum @@ -28,8 +28,6 @@ github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY= -github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.46.7 h1:IjvAWeiJZlbETOemOwvheN5L17CvKvKW0T1xOC6d3Sc= github.com/aws/aws-sdk-go v1.46.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= From 0fbb986ba9f6520d6cf3f3a0e4c1365c257fddac Mon Sep 17 00:00:00 2001 From: BlueSkyXN <63384277+BlueSkyXN@users.noreply.github.com> Date: Thu, 23 Nov 2023 21:49:16 +0800 Subject: [PATCH 028/659] fix(aliyundrive_open): mitigation measures for 15-minute limit (#5560 close #5547) * fix(aliyundrive_open):Mitigation measures for AliOpen's 15-minute limit. I conducted small-scale tests, which seem to have no significant negative impact. If the 15-minute issue still occurs, further measures will be needed. Methods like local proxy can be attempted. * chore(aliyundrive_open): change cache of the link to 1 minute --------- Co-authored-by: Andy Hsu --- drivers/aliyundrive_open/driver.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index bc41e56b6fd..994361285b9 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -80,7 +80,7 @@ func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), - "expire_sec": 14400, + "expire_sec": 900, }) }) if err != nil { @@ -93,7 +93,7 @@ func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link } url = utils.Json.Get(res, "streamsUrl", d.LIVPDownloadFormat).ToString() } - exp := time.Hour + exp := time.Minute return &model.Link{ URL: url, Expiration: &exp, @@ -207,7 +207,7 @@ func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (inte case "video_preview": uri = "/adrive/v1.0/openFile/getVideoPreviewPlayInfo" data["category"] = "live_transcoding" - data["url_expire_sec"] = 14400 + data["url_expire_sec"] = 900 default: return nil, errs.NotSupport } From fe34d30d17a96a0b32043532ce2b282ffeff7d0a Mon Sep 17 00:00:00 2001 From: textrix Date: Thu, 23 Nov 2023 22:50:16 +0900 Subject: [PATCH 029/659] feat(crypt): add show hidden option (#5554) --- drivers/crypt/driver.go | 6 ++++++ drivers/crypt/meta.go | 2 ++ 2 files changed, 8 insertions(+) diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index d8783b6ea14..649f47e58c1 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -124,6 +124,9 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ //filter illegal files continue } + if !d.ShowHidden && strings.HasPrefix(name, ".") { + continue + } objRes := model.Object{ Name: name, Size: 0, @@ -145,6 +148,9 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ //filter illegal files continue } + if !d.ShowHidden && strings.HasPrefix(name, ".") { + continue + } objRes := model.Object{ Name: name, Size: size, diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index ffa4af71bdc..180773a3f48 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -21,6 +21,8 @@ type Addition struct { FileNameEncoding string `json:"filename_encoding" type:"select" required:"true" options:"base64,base32,base32768" default:"base64" help:"for advanced user only!"` Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + + ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ From d455a232efc2e3196ad2ae99d83ecd04f4c5fcff Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 23 Nov 2023 22:29:04 +0800 Subject: [PATCH 030/659] fix(vtencent): hack file with size 0 but actual size is not 0 - allow use another proxy for vtencent and chaoxing --- drivers/chaoxing/meta.go | 6 +++--- drivers/vtencent/drive.go | 9 +++++++-- drivers/vtencent/meta.go | 4 ++-- internal/driver/config.go | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/drivers/chaoxing/meta.go b/drivers/chaoxing/meta.go index 42f4164c353..c0500629cf3 100644 --- a/drivers/chaoxing/meta.go +++ b/drivers/chaoxing/meta.go @@ -5,7 +5,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" ) -// 此程序挂载的是超星小组网盘，需要代理才能只用； +// 此程序挂载的是超星小组网盘，需要代理才能使用； // 登录超星后进入个人空间，进入小组，新建小组，点击进去。 // url中就有bbsid的参数，系统限制单文件大小2G，没有总容量限制 type Addition struct { @@ -30,9 +30,9 @@ func init() { op.RegisterDriver(func() driver.Driver { return &ChaoXing{ config: driver.Config{ - Name: "超星小组盘", + Name: "ChaoXingGroupDrive", OnlyProxy: true, - OnlyLocal: true, + OnlyLocal: false, DefaultRoot: "-1", NoOverwriteUpload: true, }, diff --git a/drivers/vtencent/drive.go b/drivers/vtencent/drive.go index b6dd13b27d9..676431439a9 100644 --- a/drivers/vtencent/drive.go +++ b/drivers/vtencent/drive.go @@ -90,7 +90,7 @@ func (d *Vtencent) Link(ctx context.Context, file model.Obj, args model.LinkArgs return nil, err } u := resps.Data.DownloadURLInfoSet[0].DownloadURL - return &model.Link{ + link := &model.Link{ URL: u, Header: http.Header{ "Referer": []string{d.conf.referer}, @@ -98,7 +98,12 @@ func (d *Vtencent) Link(ctx context.Context, file model.Obj, args model.LinkArgs }, Concurrency: 2, PartSize: 10 * utils.MB, - }, nil + } + if file.GetSize() == 0 { + link.Concurrency = 0 + link.PartSize = 0 + } + return link, nil } func (d *Vtencent) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { diff --git a/drivers/vtencent/meta.go b/drivers/vtencent/meta.go index e78db685c97..3bb6cf74639 100644 --- a/drivers/vtencent/meta.go +++ b/drivers/vtencent/meta.go @@ -23,9 +23,9 @@ func init() { op.RegisterDriver(func() driver.Driver { return &Vtencent{ config: driver.Config{ - Name: "腾讯智能创作平台", + Name: "VTencent", OnlyProxy: true, - OnlyLocal: true, + OnlyLocal: false, DefaultRoot: "9", NoOverwriteUpload: true, }, diff --git a/internal/driver/config.go b/internal/driver/config.go index 35ff6e4f2ed..c9e3f949af0 100644 --- a/internal/driver/config.go +++ b/internal/driver/config.go @@ -11,7 +11,7 @@ type Config struct { DefaultRoot string `json:"default_root"` CheckStatus bool `json:"-"` Alert string `json:"alert"` //info,success,warning,danger - NoOverwriteUpload bool `json:"-"` + NoOverwriteUpload bool `json:"-"` // whether to support overwrite upload } func (c Config) MustProxy() bool { From b6134dc5152fb39b9833c9e7c9717d47d9f5d601 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 24 Nov 2023 15:02:36 +0800 Subject: [PATCH 031/659] feat: allow keep files in offline download (close #4678) --- internal/offline_download/tool/add.go | 1 + internal/offline_download/tool/download.go | 9 +++---- internal/offline_download/tool/transfer.go | 28 ++++++++++++++++++---- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 9ad8d055726..687121222eb 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -16,6 +16,7 @@ const ( DeleteOnUploadSucceed DeletePolicy = "delete_on_upload_succeed" DeleteOnUploadFailed DeletePolicy = "delete_on_upload_failed" DeleteNever DeletePolicy = "delete_never" + DeleteAlways DeletePolicy = "delete_always" ) type AddURLArgs struct { diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 36ab6c82d4a..f4ff164c6fb 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -125,10 +125,11 @@ func (t *DownloadTask) Complete() error { for i, _ := range files { file := files[i] TransferTaskManager.Add(&TransferTask{ - file: file, - dstDirPath: t.DstDirPath, - wg: &wg, - tempDir: t.TempDir, + file: file, + dstDirPath: t.DstDirPath, + wg: &wg, + tempDir: t.TempDir, + deletePolicy: t.DeletePolicy, }) } return nil diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index c39e4ba0880..0744b333089 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -9,16 +9,18 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" + "os" "path/filepath" "sync" ) type TransferTask struct { tache.Base - file File - dstDirPath string - wg *sync.WaitGroup - tempDir string + file File + dstDirPath string + wg *sync.WaitGroup + tempDir string + deletePolicy DeletePolicy } func (t *TransferTask) Run() error { @@ -61,6 +63,24 @@ func (t *TransferTask) GetStatus() string { return "transferring" } +func (t *TransferTask) OnSucceeded() { + if t.deletePolicy == DeleteOnUploadSucceed || t.deletePolicy == DeleteAlways { + err := os.Remove(t.file.Path) + if err != nil { + log.Errorf("failed to delete file %s, error: %s", t.file.Path, err.Error()) + } + } +} + +func (t *TransferTask) OnFailed() { + if t.deletePolicy == DeleteOnUploadFailed || t.deletePolicy == DeleteAlways { + err := os.Remove(t.file.Path) + if err != nil { + log.Errorf("failed to delete file %s, error: %s", t.file.Path, err.Error()) + } + } +} + var ( TransferTaskManager *tache.Manager[*TransferTask] ) From 34746e951cba8a48aa697c75ce24c2fee55c3808 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 24 Nov 2023 16:26:05 +0800 Subject: [PATCH 032/659] feat(offline_download): add simple http tool (close #4002) --- internal/errs/errors.go | 4 + internal/offline_download/all.go | 1 + internal/offline_download/aria2/aria2.go | 5 ++ internal/offline_download/http/client.go | 85 ++++++++++++++++++++++ internal/offline_download/http/util.go | 21 ++++++ internal/offline_download/qbit/qbit.go | 5 ++ internal/offline_download/tool/base.go | 3 + internal/offline_download/tool/download.go | 7 ++ 8 files changed, 131 insertions(+) create mode 100644 internal/offline_download/http/client.go create mode 100644 internal/offline_download/http/util.go diff --git a/internal/errs/errors.go b/internal/errs/errors.go index b48718778a6..cd681e607b3 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -29,3 +29,7 @@ func NewErr(err error, format string, a ...any) error { func IsNotFoundError(err error) bool { return errors.Is(pkgerr.Cause(err), ObjectNotFound) || errors.Is(pkgerr.Cause(err), StorageNotFound) } + +func IsNotSupportError(err error) bool { + return errors.Is(pkgerr.Cause(err), NotSupport) +} diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 0c7853cb13f..2229a855468 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -2,5 +2,6 @@ package offline_download import ( _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" + _ "github.com/alist-org/alist/v3/internal/offline_download/http" _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" ) diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index 4cdad64b924..ea6404a6229 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -3,6 +3,7 @@ package aria2 import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/errs" "strconv" "time" @@ -21,6 +22,10 @@ type Aria2 struct { client rpc.Client } +func (a *Aria2) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + func (a *Aria2) Name() string { return "aria2" } diff --git a/internal/offline_download/http/client.go b/internal/offline_download/http/client.go new file mode 100644 index 00000000000..0db05f35c15 --- /dev/null +++ b/internal/offline_download/http/client.go @@ -0,0 +1,85 @@ +package http + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/pkg/utils" + "net/http" + "net/url" + "os" + "path" + "path/filepath" +) + +type SimpleHttp struct { + client http.Client +} + +func (s SimpleHttp) Name() string { + return "SimpleHttp" +} + +func (s SimpleHttp) Items() []model.SettingItem { + return nil +} + +func (s SimpleHttp) Init() (string, error) { + return "ok", nil +} + +func (s SimpleHttp) IsReady() bool { + return true +} + +func (s SimpleHttp) AddURL(args *tool.AddUrlArgs) (string, error) { + panic("should not be called") +} + +func (s SimpleHttp) Remove(task *tool.DownloadTask) error { + panic("should not be called") +} + +func (s SimpleHttp) Status(task *tool.DownloadTask) (*tool.Status, error) { + panic("should not be called") +} + +func (s SimpleHttp) Run(task *tool.DownloadTask) error { + u := task.Url + // parse url + _u, err := url.Parse(u) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(task.Ctx(), http.MethodGet, u, nil) + if err != nil { + return err + } + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return fmt.Errorf("http status code %d", resp.StatusCode) + } + filename := path.Base(_u.Path) + if n, err := parseFilenameFromContentDisposition(resp.Header.Get("Content-Disposition")); err == nil { + filename = n + } + // save to temp dir + _ = os.MkdirAll(task.TempDir, os.ModePerm) + filePath := filepath.Join(task.TempDir, filename) + file, err := os.Create(filePath) + if err != nil { + return err + } + defer file.Close() + fileSize := resp.ContentLength + err = utils.CopyWithCtx(task.Ctx(), file, resp.Body, fileSize, task.SetProgress) + return err +} + +func init() { + tool.Tools.Add(&SimpleHttp{}) +} diff --git a/internal/offline_download/http/util.go b/internal/offline_download/http/util.go new file mode 100644 index 00000000000..eefefec24ac --- /dev/null +++ b/internal/offline_download/http/util.go @@ -0,0 +1,21 @@ +package http + +import ( + "fmt" + "mime" +) + +func parseFilenameFromContentDisposition(contentDisposition string) (string, error) { + if contentDisposition == "" { + return "", fmt.Errorf("Content-Disposition is empty") + } + _, params, err := mime.ParseMediaType(contentDisposition) + if err != nil { + return "", err + } + filename := params["filename"] + if filename == "" { + return "", fmt.Errorf("filename not found in Content-Disposition: [%s]", contentDisposition) + } + return filename, nil +} diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go index 28a5170ec12..c2e92d2dce0 100644 --- a/internal/offline_download/qbit/qbit.go +++ b/internal/offline_download/qbit/qbit.go @@ -2,6 +2,7 @@ package qbit import ( "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/setting" @@ -13,6 +14,10 @@ type QBittorrent struct { client qbittorrent.Client } +func (a *QBittorrent) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + func (a *QBittorrent) Name() string { return "qBittorrent" } diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go index 1dd8e82bb4e..3b9fb07a999 100644 --- a/internal/offline_download/tool/base.go +++ b/internal/offline_download/tool/base.go @@ -35,6 +35,9 @@ type Tool interface { Remove(task *DownloadTask) error // Status return the status of the download task, if an error occurred, return the error in Status.Err Status(task *DownloadTask) (*Status, error) + + // Run for simple http download + Run(task *DownloadTask) error } type GetFileser interface { diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index f4ff164c6fb..fd91df03be7 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -2,6 +2,7 @@ package tool import ( "fmt" + "github.com/alist-org/alist/v3/internal/errs" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" @@ -25,6 +26,12 @@ type DownloadTask struct { } func (t *DownloadTask) Run() error { + if err := t.tool.Run(t); !errs.IsNotSupportError(err) { + if err == nil { + return t.Complete() + } + return err + } t.Signal = make(chan int) t.finish = make(chan struct{}) defer func() { From 6100647310594868e931f3de1188ddd8bde93b78 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 24 Nov 2023 16:46:48 +0800 Subject: [PATCH 033/659] fix: reflected XSS vulnerability plist api --- server/handles/helper.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/handles/helper.go b/server/handles/helper.go index 40eba3c449c..bd41c42c3bf 100644 --- a/server/handles/helper.go +++ b/server/handles/helper.go @@ -45,6 +45,8 @@ func Plist(c *gin.Context) { } fullName := c.Param("name") Url := link.String() + Url = strings.ReplaceAll(Url, "<", "[") + Url = strings.ReplaceAll(Url, ">", "]") nameEncode := linkNameSplit[1] fullName, err = url.PathUnescape(nameEncode) if err != nil { From 3f405de6a982866db1ed6adcbb23c7e4c3f05e8a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 24 Nov 2023 19:17:37 +0800 Subject: [PATCH 034/659] feat: customize allow `origins`, `headers` and `methods` --- internal/conf/config.go | 12 ++++++++++++ server/router.go | 7 ++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/internal/conf/config.go b/internal/conf/config.go index 2754064ca4b..92761ea7e90 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -51,6 +51,12 @@ type TasksConfig struct { Copy TaskConfig `json:"copy" envPrefix:"COPY_"` } +type Cors struct { + AllowOrigins []string `json:"allow_origins" env:"ALLOW_ORIGINS"` + AllowMethods []string `json:"allow_methods" env:"ALLOW_METHODS"` + AllowHeaders []string `json:"allow_headers" env:"ALLOW_HEADERS"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -67,6 +73,7 @@ type Config struct { MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"` + Cors Cors `json:"cors" envPrefix:"CORS_"` } func DefaultConfig() *Config { @@ -120,5 +127,10 @@ func DefaultConfig() *Config { MaxRetry: 2, }, }, + Cors: Cors{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{"*"}, + AllowHeaders: []string{"*"}, + }, } } diff --git a/server/router.go b/server/router.go index 7f179231ecd..588cc071bfb 100644 --- a/server/router.go +++ b/server/router.go @@ -163,8 +163,9 @@ func _fs(g *gin.RouterGroup) { func Cors(r *gin.Engine) { config := cors.DefaultConfig() - config.AllowAllOrigins = true - config.AllowHeaders = []string{"*"} - config.AllowMethods = []string{"*"} + //config.AllowAllOrigins = true + config.AllowOrigins = conf.Conf.Cors.AllowOrigins + config.AllowHeaders = conf.Conf.Cors.AllowHeaders + config.AllowMethods = conf.Conf.Cors.AllowMethods r.Use(cors.New(config)) } From d26887d211c87b68da00827096ba85944e998256 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 24 Nov 2023 19:22:19 +0800 Subject: [PATCH 035/659] fix: `content-type` conflicts with #5420 --- server/common/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/common/proxy.go b/server/common/proxy.go index a4f04abfe2c..4ca4ba7f6ab 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -19,7 +19,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. attachFileName(w, file) contentType := link.Header.Get("Content-Type") if contentType != "" { - w.Header().Add("Content-Type", contentType) + w.Header().Set("Content-Type", contentType) } http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) return nil From 68af284dad2ba332e3b91e64a422f9103dec603a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 25 Nov 2023 14:15:17 +0800 Subject: [PATCH 036/659] fix: task popped but not execute (close #5565) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2fdd60f57bf..70fc32b9ed4 100644 --- a/go.mod +++ b/go.mod @@ -186,7 +186,7 @@ require ( github.com/ugorji/go/codec v1.2.11 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect - github.com/xhofe/tache v0.0.0-20231120085916-722855be0521 // indirect + github.com/xhofe/tache v0.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.3.0 // indirect diff --git a/go.sum b/go.sum index 55f23a459af..c3b55004204 100644 --- a/go.sum +++ b/go.sum @@ -443,6 +443,8 @@ github.com/xhofe/tache v0.0.0-20231120064353-a3585a237e25 h1:XZBuEzDB9Kqni/+zAKx github.com/xhofe/tache v0.0.0-20231120064353-a3585a237e25/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/tache v0.0.0-20231120085916-722855be0521 h1:m7O+xOqQRysjFngMhQ39RzCFdiCouFLvsrV7N2ScbUY= github.com/xhofe/tache v0.0.0-20231120085916-722855be0521/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/xhofe/tache v0.1.0 h1:W0uoyLWCmUEQudXwB93owdlGSlN8gwZmiiDlKFCerKA= +github.com/xhofe/tache v0.1.0/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= From d5f381ef6f81ca5317e6e215dbd5bd60f1ed6a2a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 25 Nov 2023 14:22:13 +0800 Subject: [PATCH 037/659] chore: upgrade golang version --- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- Dockerfile | 4 ++-- go.mod | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70fe145c018..eeff969f581 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - go-version: [ '1.20' ] + go-version: [ '1.21' ] name: Build runs-on: ${{ matrix.platform }} steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6697363d2f0..50a4719993f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.20' ] + go-version: [ '1.21' ] name: Release runs-on: ${{ matrix.platform }} steps: diff --git a/Dockerfile b/Dockerfile index 97d1b9e8811..542d502c53b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM alpine:3.18 as builder +FROM alpine:edge as builder LABEL stage=go-builder WORKDIR /app/ COPY ./ ./ RUN apk add --no-cache bash curl gcc git go musl-dev; \ bash build.sh release docker -FROM alpine:3.18 +FROM alpine:edge LABEL MAINTAINER="i@nn.ci" VOLUME /opt/alist/data/ WORKDIR /opt/alist/ diff --git a/go.mod b/go.mod index 70fc32b9ed4..b058dad303d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/alist-org/alist/v3 -go 1.20 +go 1.21 require ( github.com/SheltonZhu/115driver v1.0.21 From b88067ea2f8a3cb357747ad4bb1daaa0f279df26 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 25 Nov 2023 14:31:48 +0800 Subject: [PATCH 038/659] ci: fix docker build error: 'pread64' undeclared here --- build.sh | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/build.sh b/build.sh index 7ca010a7a14..3f0cfb0a5e0 100644 --- a/build.sh +++ b/build.sh @@ -85,6 +85,7 @@ BuildDev() { } BuildDocker() { + echo "replace github.com/mattn/go-sqlite3 => github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed" >>go.mod go build -o ./bin/alist -ldflags="$ldflags" -tags=jsoniter . } diff --git a/go.sum b/go.sum index c3b55004204..d144f2bd4fb 100644 --- a/go.sum +++ b/go.sum @@ -256,6 +256,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed h1:lM1oz49yOQhEQsJh3lRnQ/voNTO+Lurx8fRy2Gmb2c8= +github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= From 1420492d811e15191ed7df3045d26c71348b46e2 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 25 Nov 2023 15:11:29 +0800 Subject: [PATCH 039/659] ci: go get after replacing go mod --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 3f0cfb0a5e0..c072572bd3f 100644 --- a/build.sh +++ b/build.sh @@ -86,6 +86,7 @@ BuildDev() { BuildDocker() { echo "replace github.com/mattn/go-sqlite3 => github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed" >>go.mod + go get gorm.io/driver/sqlite@v1.4.4 go build -o ./bin/alist -ldflags="$ldflags" -tags=jsoniter . } From f23567199bf5cda9f5eda9bb13b75d5fae035401 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 25 Nov 2023 15:12:25 +0800 Subject: [PATCH 040/659] chore: go mod tidy --- go.mod | 2 +- go.sum | 25 +++++++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index b058dad303d..ee5a255a83f 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 + github.com/xhofe/tache v0.1.0 golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/image v0.11.0 @@ -186,7 +187,6 @@ require ( github.com/ugorji/go/codec v1.2.11 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect - github.com/xhofe/tache v0.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.3.0 // indirect diff --git a/go.sum b/go.sum index d144f2bd4fb..87f5866630e 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= @@ -11,6 +12,7 @@ github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/ github.com/SheltonZhu/115driver v1.0.21 h1:Pz6r14VwIiuSyHj+OmJe57FHhbmWB/6IfnXAFL2iXbU= github.com/SheltonZhu/115driver v1.0.21/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= +github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaVLk4gI/Co0paF92FmxSR6U1zY= @@ -90,6 +92,7 @@ github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FD github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= @@ -109,6 +112,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= @@ -144,6 +148,7 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -174,6 +179,7 @@ github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVI github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= @@ -189,11 +195,14 @@ github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= +github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -249,6 +258,7 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -256,8 +266,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed h1:lM1oz49yOQhEQsJh3lRnQ/voNTO+Lurx8fRy2Gmb2c8= -github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= @@ -346,6 +354,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= +github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= @@ -374,6 +383,7 @@ github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUc github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= @@ -387,6 +397,7 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -431,20 +442,13 @@ github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZ github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= +github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= -github.com/xhofe/tache v0.0.0-20231110075853-2bd4b52dad9b h1:958N/31ioR0QSg6RarX1aqBsfmlOI2JeYiVzxeGdUAA= -github.com/xhofe/tache v0.0.0-20231110075853-2bd4b52dad9b/go.mod h1:1ISbKrHZNMMrXvgCdaFV0Vkc9Wbo7WV1q7Teovm4Huc= -github.com/xhofe/tache v0.0.0-20231119124711-c417893fc267 h1:MC271sH8UHYqr/IDz9PsqTlyD51HyFvxtQRTemwxR9s= -github.com/xhofe/tache v0.0.0-20231119124711-c417893fc267/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= -github.com/xhofe/tache v0.0.0-20231120064353-a3585a237e25 h1:XZBuEzDB9Kqni/+zAKxl30iOdp80/GavUsCkPMiQMjg= -github.com/xhofe/tache v0.0.0-20231120064353-a3585a237e25/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= -github.com/xhofe/tache v0.0.0-20231120085916-722855be0521 h1:m7O+xOqQRysjFngMhQ39RzCFdiCouFLvsrV7N2ScbUY= -github.com/xhofe/tache v0.0.0-20231120085916-722855be0521/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/tache v0.1.0 h1:W0uoyLWCmUEQudXwB93owdlGSlN8gwZmiiDlKFCerKA= github.com/xhofe/tache v0.1.0/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -453,6 +457,7 @@ github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQ go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= From d142fc3449daf0a50d2083102664ce42f592ea27 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 25 Nov 2023 16:09:38 +0800 Subject: [PATCH 041/659] ci: upgrade golang version --- .github/workflows/release_linux_musl.yml | 2 +- .github/workflows/release_linux_musl_arm.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_linux_musl.yml b/.github/workflows/release_linux_musl.yml index dd298c49b43..0288427524b 100644 --- a/.github/workflows/release_linux_musl.yml +++ b/.github/workflows/release_linux_musl.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.20' ] + go-version: [ '1.21' ] name: Release runs-on: ${{ matrix.platform }} steps: diff --git a/.github/workflows/release_linux_musl_arm.yml b/.github/workflows/release_linux_musl_arm.yml index f5e69c1c81b..abd96987c7f 100644 --- a/.github/workflows/release_linux_musl_arm.yml +++ b/.github/workflows/release_linux_musl_arm.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.20' ] + go-version: [ '1.21' ] name: Release runs-on: ${{ matrix.platform }} steps: From 54e75d72877b1b02867189c66c0d510de833fe77 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 25 Nov 2023 20:27:23 +0800 Subject: [PATCH 042/659] feat: enabled `sign_all` by default --- internal/bootstrap/data/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 21a432ddebd..09bf79b0418 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -132,7 +132,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.CustomizeHead, Value: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.CustomizeBody, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.LinkExpiration, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, - {Key: conf.SignAll, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.SignAll, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.PrivacyRegs, Value: `(?:(?:\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]) ([[:xdigit:]]{1,4}(?::[[:xdigit:]]{1,4}){7}|::|:(?::[[:xdigit:]]{1,4}){1,6}|[[:xdigit:]]{1,4}:(?::[[:xdigit:]]{1,4}){1,5}|(?:[[:xdigit:]]{1,4}:){2}(?::[[:xdigit:]]{1,4}){1,4}|(?:[[:xdigit:]]{1,4}:){3}(?::[[:xdigit:]]{1,4}){1,3}|(?:[[:xdigit:]]{1,4}:){4}(?::[[:xdigit:]]{1,4}){1,2}|(?:[[:xdigit:]]{1,4}:){5}:[[:xdigit:]]{1,4}|(?:[[:xdigit:]]{1,4}:){1,6}:) (?U)access_token=(.*)&`, From f4dcf4599c8c5cf6ab1044b42e155ac2c81d1e6f Mon Sep 17 00:00:00 2001 From: Kuingsmile Date: Mon, 27 Nov 2023 02:53:52 -0800 Subject: [PATCH 043/659] fix: add error handling for webdav mkcol according to RFC 4918 (#5581) * feat: add error handling for mkcol method in webdav.go * feat: update rfc reference * fix: fix issue with uncorrect error handling --- server/webdav/webdav.go | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index f2e3fd8a409..509a7f1cf6d 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -382,6 +382,21 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in if r.ContentLength > 0 { return http.StatusUnsupportedMediaType, nil } + + // RFC 4918 9.3.1 + //405 (Method Not Allowed) - MKCOL can only be executed on an unmapped URL + if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil { + return http.StatusMethodNotAllowed, err + } + // RFC 4918 9.3.1 + // 409 (Conflict) The server MUST NOT create those intermediate collections automatically. + reqDir := path.Dir(reqPath) + if _, err := fs.Get(ctx, reqDir, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusConflict, err + } + return http.StatusMethodNotAllowed, err + } if err := fs.MakeDir(ctx, reqPath); err != nil { if os.IsNotExist(err) { return http.StatusConflict, err @@ -521,12 +536,12 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus } } reqPath, status, err := h.stripPrefix(r.URL.Path) - reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return status, err } + reqPath, err = user.JoinPath(reqPath) if err != nil { - return status, err + return 403, err } ld = LockDetails{ Root: reqPath, From b99e709bdb43e65efc3be08ebcbeb4d86fffdaeb Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 29 Nov 2023 22:51:03 +0800 Subject: [PATCH 044/659] fix(teambition): international upload (close #5360) --- drivers/teambition/driver.go | 22 +++++++++++++++++----- drivers/teambition/help.go | 18 ++++++++++++++++++ drivers/teambition/util.go | 11 ++++++----- 3 files changed, 41 insertions(+), 10 deletions(-) create mode 100644 drivers/teambition/help.go diff --git a/drivers/teambition/driver.go b/drivers/teambition/driver.go index d4fcc401bad..c75d2ac00b6 100644 --- a/drivers/teambition/driver.go +++ b/drivers/teambition/driver.go @@ -3,12 +3,12 @@ package teambition import ( "context" "errors" + "github.com/alist-org/alist/v3/pkg/utils" "net/http" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -128,11 +128,23 @@ func (d *Teambition) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if d.UseS3UploadMethod { return d.newUpload(ctx, dstDir, stream, up) } - res, err := d.request("/api/v2/users/me", http.MethodGet, nil, nil) - if err != nil { - return err + var ( + token string + err error + ) + if d.isInternational() { + res, err := d.request("/projects", http.MethodGet, nil, nil) + if err != nil { + return err + } + token = getBetweenStr(string(res), "strikerAuth":"", "","phoneForLogin") + } else { + res, err := d.request("/api/v2/users/me", http.MethodGet, nil, nil) + if err != nil { + return err + } + token = utils.Json.Get(res, "strikerAuth").ToString() } - token := utils.Json.Get(res, "strikerAuth").ToString() var newFile *FileUpload if stream.GetSize() <= 20971520 { // post upload diff --git a/drivers/teambition/help.go b/drivers/teambition/help.go new file mode 100644 index 00000000000..8581c3e827c --- /dev/null +++ b/drivers/teambition/help.go @@ -0,0 +1,18 @@ +package teambition + +import "strings" + +func getBetweenStr(str, start, end string) string { + n := strings.Index(str, start) + if n == -1 { + return "" + } + n = n + len(start) + str = string([]byte(str)[n:]) + m := strings.Index(str, end) + if m == -1 { + return "" + } + str = string([]byte(str)[:m]) + return str +} diff --git a/drivers/teambition/util.go b/drivers/teambition/util.go index c39ffb18286..79de7007c78 100644 --- a/drivers/teambition/util.go +++ b/drivers/teambition/util.go @@ -126,19 +126,20 @@ func (d *Teambition) upload(ctx context.Context, file model.FileStreamer, token prefix = "us-tcs" } var newFile FileUpload - _, err := base.RestyClient.R(). + res, err := base.RestyClient.R(). SetContext(ctx). SetResult(&newFile).SetHeader("Authorization", token). SetMultipartFormData(map[string]string{ - "name": file.GetName(), - "type": file.GetMimetype(), - "size": strconv.FormatInt(file.GetSize(), 10), - //"lastModifiedDate": "", + "name": file.GetName(), + "type": file.GetMimetype(), + "size": strconv.FormatInt(file.GetSize(), 10), + "lastModifiedDate": time.Now().Format("Mon Jan 02 2006 15:04:05 GMT+0800 (中国标准时间)"), }).SetMultipartField("file", file.GetName(), file.GetMimetype(), file). Post(fmt.Sprintf("https://%s.teambition.net/upload", prefix)) if err != nil { return nil, err } + log.Debugf("[teambition] upload response: %s", res.String()) return &newFile, nil } From f475eb4401aadd8193a1b8e603bfb368bdab2be6 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 30 Nov 2023 12:37:25 +0800 Subject: [PATCH 045/659] fix: incorrect go-version on auto-lang --- .github/workflows/auto_lang.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto_lang.yml b/.github/workflows/auto_lang.yml index 4c20efcbf4e..d1221f763f8 100644 --- a/.github/workflows/auto_lang.yml +++ b/.github/workflows/auto_lang.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.20' ] + go-version: [ '1.21' ] name: auto generate lang.json runs-on: ${{ matrix.platform }} steps: From 66b7fe1e1b5cb5c87464575ea322adc5b06d9bb7 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 30 Nov 2023 20:44:05 +0800 Subject: [PATCH 046/659] fix: task cannot be retried manually (close #5599) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ee5a255a83f..e6ced69b695 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 - github.com/xhofe/tache v0.1.0 + github.com/xhofe/tache v0.1.1 golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/image v0.11.0 diff --git a/go.sum b/go.sum index 87f5866630e..36c2b49a252 100644 --- a/go.sum +++ b/go.sum @@ -451,6 +451,8 @@ github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3K github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.0 h1:W0uoyLWCmUEQudXwB93owdlGSlN8gwZmiiDlKFCerKA= github.com/xhofe/tache v0.1.0/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/xhofe/tache v0.1.1 h1:O5QY4cVjIGELx3UGh6LbVAc18MWGXgRNQjMt72x6w/8= +github.com/xhofe/tache v0.1.1/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= From e4a6b758dcda5b4d3f46e2218db0f589c4480623 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 2 Dec 2023 16:44:00 +0800 Subject: [PATCH 047/659] docs: remove jetbrains in special sponsor [skip ci] --- README.md | 1 - README_cn.md | 1 - README_ja.md | 1 - 3 files changed, 3 deletions(-) diff --git a/README.md b/README.md index 757bc740b37..5f4d8ef4270 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,6 @@ https://alist.nn.ci/guide/sponsor.html - [VidHub](https://okaapps.com/product/1659622164?ref=alist) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) - [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 -- [JetBrains](https://www.jetbrains.com/) - Essential tools for software developers and teams ## Contributors diff --git a/README_cn.md b/README_cn.md index 848e21a8e62..6af8aeaf1af 100644 --- a/README_cn.md +++ b/README_cn.md @@ -113,7 +113,6 @@ AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我 - [VidHub](https://zh.okaapps.com/product/1659622164?ref=alist) - 苹果生态下优雅的网盘视频播放器，iPhone，iPad，Mac，Apple TV全平台支持。 - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (国内API服务器赞助) - [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 -- [JetBrains](https://www.jetbrains.com/) - Essential tools for software developers and teams ## 贡献者 diff --git a/README_ja.md b/README_ja.md index fc639fdbc2a..b873947fc86 100644 --- a/README_ja.md +++ b/README_ja.md @@ -115,7 +115,6 @@ https://alist.nn.ci/guide/sponsor.html - [VidHub](https://okaapps.com/product/1659622164?ref=alist) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) - [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 -- [JetBrains](https://www.jetbrains.com/) - Essential tools for software developers and teams ## コントリビューター From 8bdfc7ac8e0ef07897ee5e72069f4659e2a1c6be Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 3 Dec 2023 14:20:01 +0800 Subject: [PATCH 048/659] fix(offline_download): don't wait for transfer task (close #5595) --- internal/offline_download/tool/download.go | 15 +-------------- internal/offline_download/tool/transfer.go | 3 --- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index fd91df03be7..0e1a9ca8335 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -6,7 +6,6 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" - "sync" "time" ) @@ -20,7 +19,6 @@ type DownloadTask struct { Status string `json:"status"` Signal chan int `json:"-"` GID string `json:"-"` - finish chan struct{} tool Tool callStatusRetried int } @@ -33,10 +31,8 @@ func (t *DownloadTask) Run() error { return err } t.Signal = make(chan int) - t.finish = make(chan struct{}) defer func() { t.Signal = nil - t.finish = nil }() gid, err := t.tool.AddURL(&AddUrlArgs{ Url: t.Url, @@ -72,9 +68,7 @@ outer: if err != nil { return err } - t.Status = "aria2 download completed, maybe transferring" - t.finish <- struct{}{} - t.Status = "offline download completed" + t.Status = "offline download completed, maybe transferring" return nil } @@ -123,18 +117,11 @@ func (t *DownloadTask) Complete() error { } } // upload files - var wg sync.WaitGroup - wg.Add(len(files)) - go func() { - wg.Wait() - t.finish <- struct{}{} - }() for i, _ := range files { file := files[i] TransferTaskManager.Add(&TransferTask{ file: file, dstDirPath: t.DstDirPath, - wg: &wg, tempDir: t.TempDir, deletePolicy: t.DeletePolicy, }) diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 0744b333089..0ef58df5019 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -11,20 +11,17 @@ import ( "github.com/xhofe/tache" "os" "path/filepath" - "sync" ) type TransferTask struct { tache.Base file File dstDirPath string - wg *sync.WaitGroup tempDir string deletePolicy DeletePolicy } func (t *TransferTask) Run() error { - defer t.wg.Done() // check dstDir again storage, dstDirActualPath, err := op.GetStorageAndActualPath(t.dstDirPath) if err != nil { From 026e944cbbe5ac4f9dc3f3db5a062f25a489d9a0 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 3 Dec 2023 14:44:20 +0800 Subject: [PATCH 049/659] feat: add task info to resp of add task api (close #5579) --- internal/fs/copy.go | 23 ++++++++++++----------- internal/fs/fs.go | 9 +++++---- internal/fs/put.go | 15 ++++++++------- internal/offline_download/tool/add.go | 17 +++++++++-------- server/handles/fsmanage.go | 17 ++++++++--------- server/handles/fsup.go | 23 +++++++++++++++++++---- server/handles/offline_download.go | 9 +++++++-- server/handles/task.go | 16 +++++++++++----- 8 files changed, 79 insertions(+), 50 deletions(-) diff --git a/internal/fs/copy.go b/internal/fs/copy.go index 43e163966ff..25f068f0c40 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -39,23 +39,23 @@ var CopyTaskManager *tache.Manager[*CopyTask] // Copy if in the same storage, call move method // if not, add copy task -func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (bool, error) { +func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (tache.TaskWithInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { - return false, errors.WithMessage(err, "failed get src storage") + return nil, errors.WithMessage(err, "failed get src storage") } dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { - return false, errors.WithMessage(err, "failed get dst storage") + return nil, errors.WithMessage(err, "failed get dst storage") } // copy if in the same storage, just call driver.Copy if srcStorage.GetStorage() == dstStorage.GetStorage() { - return false, op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + return nil, op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) } if ctx.Value(conf.NoTaskKey) != nil { srcObj, err := op.Get(ctx, srcStorage, srcObjActualPath) if err != nil { - return false, errors.WithMessagef(err, "failed get src [%s] file", srcObjPath) + return nil, errors.WithMessagef(err, "failed get src [%s] file", srcObjPath) } if !srcObj.IsDir() { // copy file directly @@ -63,7 +63,7 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool Header: http.Header{}, }) if err != nil { - return false, errors.WithMessagef(err, "failed get [%s] link", srcObjPath) + return nil, errors.WithMessagef(err, "failed get [%s] link", srcObjPath) } fs := stream.FileStream{ Obj: srcObj, @@ -72,19 +72,20 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool // any link provided is seekable ss, err := stream.NewSeekableStream(fs, link) if err != nil { - return false, errors.WithMessagef(err, "failed get [%s] stream", srcObjPath) + return nil, errors.WithMessagef(err, "failed get [%s] stream", srcObjPath) } - return false, op.Put(ctx, dstStorage, dstDirActualPath, ss, nil, false) + return nil, op.Put(ctx, dstStorage, dstDirActualPath, ss, nil, false) } } // not in the same storage - CopyTaskManager.Add(&CopyTask{ + t := &CopyTask{ srcStorage: srcStorage, dstStorage: dstStorage, srcObjPath: srcObjActualPath, dstDirPath: dstDirActualPath, - }) - return true, nil + } + CopyTaskManager.Add(t) + return t, nil } func copyBetween2Storages(t *CopyTask, srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error { diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 2b23142a662..23e8a87a6fd 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -6,6 +6,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" ) // the param named path of functions in this package is a mount path @@ -68,7 +69,7 @@ func Move(ctx context.Context, srcPath, dstDirPath string, lazyCache ...bool) er return err } -func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (bool, error) { +func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (tache.TaskWithInfo, error) { res, err := _copy(ctx, srcObjPath, dstDirPath, lazyCache...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) @@ -100,12 +101,12 @@ func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer return err } -func PutAsTask(dstDirPath string, file model.FileStreamer) error { - err := putAsTask(dstDirPath, file) +func PutAsTask(dstDirPath string, file model.FileStreamer) (tache.TaskWithInfo, error) { + t, err := putAsTask(dstDirPath, file) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) } - return err + return t, err } type GetStoragesArgs struct { diff --git a/internal/fs/put.go b/internal/fs/put.go index 43d41acf0fd..807b15e07d6 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -33,28 +33,29 @@ func (t *UploadTask) Run() error { var UploadTaskManager *tache.Manager[*UploadTask] // putAsTask add as a put task and return immediately -func putAsTask(dstDirPath string, file model.FileStreamer) error { +func putAsTask(dstDirPath string, file model.FileStreamer) (tache.TaskWithInfo, error) { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { - return errors.WithMessage(err, "failed get storage") + return nil, errors.WithMessage(err, "failed get storage") } if storage.Config().NoUpload { - return errors.WithStack(errs.UploadNotSupported) + return nil, errors.WithStack(errs.UploadNotSupported) } if file.NeedStore() { _, err := file.CacheFullInTempFile() if err != nil { - return errors.Wrapf(err, "failed to create temp file") + return nil, errors.Wrapf(err, "failed to create temp file") } //file.SetReader(tempFile) //file.SetTmpFile(tempFile) } - UploadTaskManager.Add(&UploadTask{ + t := &UploadTask{ storage: storage, dstDirActualPath: dstDirActualPath, file: file, - }) - return nil + } + UploadTaskManager.Add(t) + return t, nil } // putDirect put the file and return after finish diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 687121222eb..3da05c8df68 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -7,6 +7,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/google/uuid" "github.com/pkg/errors" + "github.com/xhofe/tache" "path/filepath" ) @@ -26,38 +27,38 @@ type AddURLArgs struct { DeletePolicy DeletePolicy } -func AddURL(ctx context.Context, args *AddURLArgs) error { +func AddURL(ctx context.Context, args *AddURLArgs) (tache.TaskWithInfo, error) { // get tool tool, err := Tools.Get(args.Tool) if err != nil { - return errors.Wrapf(err, "failed get tool") + return nil, errors.Wrapf(err, "failed get tool") } // check tool is ready if !tool.IsReady() { // try to init tool if _, err := tool.Init(); err != nil { - return errors.Wrapf(err, "failed init tool %s", args.Tool) + return nil, errors.Wrapf(err, "failed init tool %s", args.Tool) } } // check storage storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) if err != nil { - return errors.WithMessage(err, "failed get storage") + return nil, errors.WithMessage(err, "failed get storage") } // check is it could upload if storage.Config().NoUpload { - return errors.WithStack(errs.UploadNotSupported) + return nil, errors.WithStack(errs.UploadNotSupported) } // check path is valid obj, err := op.Get(ctx, storage, dstDirActualPath) if err != nil { if !errs.IsObjectNotFound(err) { - return errors.WithMessage(err, "failed get object") + return nil, errors.WithMessage(err, "failed get object") } } else { if !obj.IsDir() { // can't add to a file - return errors.WithStack(errs.NotFolder) + return nil, errors.WithStack(errs.NotFolder) } } @@ -71,5 +72,5 @@ func AddURL(ctx context.Context, args *AddURLArgs) error { tool: tool, } DownloadTaskManager.Add(t) - return nil + return t, nil } diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 2733509e9a5..3d446eda957 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -2,6 +2,7 @@ package handles import ( "fmt" + "github.com/xhofe/tache" "io" stdpath "path" @@ -120,22 +121,20 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, err, 403) return } - var addedTask []string + var addedTasks []tache.TaskWithInfo for i, name := range req.Names { - ok, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) - if ok { - addedTask = append(addedTask, name) + t, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + if t != nil { + addedTasks = append(addedTasks, t) } if err != nil { common.ErrorResp(c, err, 500) return } } - if len(addedTask) > 0 { - common.SuccessResp(c, fmt.Sprintf("Added %d tasks", len(addedTask))) - } else { - common.SuccessResp(c) - } + common.SuccessResp(c, gin.H{ + "tasks": getTaskInfos(addedTasks), + }) } type RenameReq struct { diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 15de86600dd..ef9baa11dc5 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -1,6 +1,7 @@ package handles import ( + "github.com/xhofe/tache" "io" "net/url" stdpath "path" @@ -57,8 +58,9 @@ func FsStream(c *gin.Context) { Mimetype: c.GetHeader("Content-Type"), WebPutAsTask: asTask, } + var t tache.TaskWithInfo if asTask { - err = fs.PutAsTask(dir, s) + t, err = fs.PutAsTask(dir, s) } else { err = fs.PutDirectly(c, dir, s, true) } @@ -67,7 +69,13 @@ func FsStream(c *gin.Context) { common.ErrorResp(c, err, 500) return } - common.SuccessResp(c) + if t == nil { + common.SuccessResp(c) + return + } + common.SuccessResp(c, gin.H{ + "task": getTaskInfo(t), + }) } func FsForm(c *gin.Context) { @@ -115,11 +123,12 @@ func FsForm(c *gin.Context) { Mimetype: file.Header.Get("Content-Type"), WebPutAsTask: asTask, } + var t tache.TaskWithInfo if asTask { s.Reader = struct { io.Reader }{f} - err = fs.PutAsTask(dir, &s) + t, err = fs.PutAsTask(dir, &s) } else { ss, err := stream.NewSeekableStream(s, nil) if err != nil { @@ -132,5 +141,11 @@ func FsForm(c *gin.Context) { common.ErrorResp(c, err, 500) return } - common.SuccessResp(c) + if t == nil { + common.SuccessResp(c) + return + } + common.SuccessResp(c, gin.H{ + "task": getTaskInfo(t), + }) } diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index fdee063df18..0b019e9e48c 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -7,6 +7,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "github.com/xhofe/tache" ) type SetAria2Req struct { @@ -97,8 +98,9 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 403) return } + var tasks []tache.TaskWithInfo for _, url := range req.Urls { - err := tool.AddURL(c, &tool.AddURLArgs{ + t, err := tool.AddURL(c, &tool.AddURLArgs{ URL: url, DstDirPath: reqPath, Tool: req.Tool, @@ -108,6 +110,9 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 500) return } + tasks = append(tasks, t) } - common.SuccessResp(c) + common.SuccessResp(c, gin.H{ + "tasks": getTaskInfos(tasks), + }) } diff --git a/server/handles/task.go b/server/handles/task.go index 0429116a44b..9c9486b9de2 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -3,6 +3,7 @@ package handles import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/xhofe/tache" @@ -33,11 +34,7 @@ func getTaskInfo[T tache.TaskWithInfo](task T) TaskInfo { } func getTaskInfos[T tache.TaskWithInfo](tasks []T) []TaskInfo { - var infos []TaskInfo - for _, t := range tasks { - infos = append(infos, getTaskInfo(t)) - } - return infos + return utils.MustSliceConvert(tasks, getTaskInfo[T]) } func taskRoute[T tache.TaskWithInfo](g *gin.RouterGroup, manager *tache.Manager[T]) { @@ -48,6 +45,15 @@ func taskRoute[T tache.TaskWithInfo](g *gin.RouterGroup, manager *tache.Manager[ g.GET("/done", func(c *gin.Context) { common.SuccessResp(c, getTaskInfos(manager.GetByState(tache.StateCanceled, tache.StateFailed, tache.StateSucceeded))) }) + g.POST("/info", func(c *gin.Context) { + tid := c.Query("tid") + task, ok := manager.GetByID(tid) + if !ok { + common.ErrorStrResp(c, "task not found", 404) + return + } + common.SuccessResp(c, getTaskInfo(task)) + }) g.POST("/cancel", func(c *gin.Context) { tid := c.Query("tid") manager.Cancel(tid) From 296be88b5f1c20c3df36051a10270125fafc7337 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 10 Dec 2023 13:17:56 +0800 Subject: [PATCH 050/659] fix: incorrect key of oidc username (close #5670) --- drivers/aliyundrive_open/driver.go | 4 ++-- server/handles/ssologin.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index 994361285b9..4029ad57c37 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -80,7 +80,7 @@ func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), - "expire_sec": 900, + "expire_sec": 14400, }) }) if err != nil { @@ -207,7 +207,7 @@ func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (inte case "video_preview": uri = "/adrive/v1.0/openFile/getVideoPreviewPlayInfo" data["category"] = "live_transcoding" - data["url_expire_sec"] = 900 + data["url_expire_sec"] = 14400 default: return nil, errs.NotSupport } diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 52486b97839..b71179b6496 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -231,7 +231,7 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorResp(c, err, 400) return } - userID := utils.Json.Get(payload, conf.SSOOIDCUsernameKey).ToString() + userID := utils.Json.Get(payload, setting.GetStr(conf.SSOOIDCUsernameKey, "name")).ToString() if userID == "" { common.ErrorStrResp(c, "cannot get username from OIDC provider", 400) return From 83c22693306f36853d6c6c2b50f567d9b58cb834 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 11 Dec 2023 15:20:29 +0800 Subject: [PATCH 051/659] fix(qbit): seed time doesn't take effect (close #5663) --- internal/offline_download/qbit/qbit.go | 2 +- internal/offline_download/tool/download.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go index c2e92d2dce0..807ebfef2dc 100644 --- a/internal/offline_download/qbit/qbit.go +++ b/internal/offline_download/qbit/qbit.go @@ -54,7 +54,7 @@ func (a *QBittorrent) AddURL(args *tool.AddUrlArgs) (string, error) { } func (a *QBittorrent) Remove(task *tool.DownloadTask) error { - err := a.client.Delete(task.GID, true) + err := a.client.Delete(task.GID, false) return err } diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 0e1a9ca8335..975530e7f6a 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -2,7 +2,9 @@ package tool import ( "fmt" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/setting" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" @@ -69,6 +71,18 @@ outer: return err } t.Status = "offline download completed, maybe transferring" + // hack for qBittorrent + if t.tool.Name() == "qBittorrent" { + seedTime := setting.GetInt(conf.QbittorrentSeedtime, 0) + if seedTime >= 0 { + t.Status = "offline download completed, waiting for seeding" + <-time.After(time.Minute * time.Duration(seedTime)) + err := t.tool.Remove(t) + if err != nil { + log.Errorln(err.Error()) + } + } + } return nil } From 74b20dedc3d757d264fff4aaba38e320c6d01f56 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Thu, 14 Dec 2023 21:31:36 +0800 Subject: [PATCH 052/659] fix: retry multipart file reset (#5693 close #5628) --- drivers/base/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/base/client.go b/drivers/base/client.go index bc08d6fb717..8bf8f421eea 100644 --- a/drivers/base/client.go +++ b/drivers/base/client.go @@ -33,6 +33,7 @@ func NewRestyClient() *resty.Client { client := resty.New(). SetHeader("user-agent", UserAgent). SetRetryCount(3). + SetRetryResetReaders(true). SetTimeout(DefaultTimeout). SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) return client From 734d4b0354ed72a6f3b2013e2bd27feb4b11683f Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 15 Dec 2023 17:07:02 +0800 Subject: [PATCH 053/659] ci: add `darwin/arm64` target to dev build --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index c072572bd3f..7071337d465 100644 --- a/build.sh +++ b/build.sh @@ -75,7 +75,7 @@ BuildDev() { export CGO_ENABLED=1 go build -o ./dist/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . done - xgo -targets=windows/amd64,darwin/amd64 -out "$appName" -ldflags="$ldflags" -tags=jsoniter . + xgo -targets=windows/amd64,darwin/amd64,darwin/arm64 -out "$appName" -ldflags="$ldflags" -tags=jsoniter . mv alist-* dist cd dist cp ./alist-windows-amd64.exe ./alist-windows-amd64-upx.exe From 6d4ab57a0e49f6a1a705e351ddb0d46134f91498 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 15 Dec 2023 18:22:16 +0800 Subject: [PATCH 054/659] build: enable cgo for `win/arm64` [skip ci] --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 7071337d465..4ea19ecc273 100644 --- a/build.sh +++ b/build.sh @@ -49,6 +49,7 @@ BuildWinArm64() { export GOARCH=arm64 export CC=$(pwd)/wrapper/zcc-arm64 export CXX=$(pwd)/wrapper/zcxx-arm64 + export CGO_ENABLED=1 go build -o "$1" -ldflags="$ldflags" -tags=jsoniter . } From de56f926cfd2d203451e3920f0ca64df617b0682 Mon Sep 17 00:00:00 2001 From: linepro6 <53991395+linepro6@users.noreply.github.com> Date: Sat, 16 Dec 2023 16:56:45 +0800 Subject: [PATCH 055/659] feat(139): support new personal cloud api (#5690) Co-authored-by: Andy Hsu --- drivers/139/driver.go | 640 ++++++++++++++++++++++++++++-------------- drivers/139/meta.go | 2 +- drivers/139/types.go | 45 +++ drivers/139/util.go | 151 ++++++++++ 4 files changed, 622 insertions(+), 216 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 69ab68f705e..9d8cbd523e6 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -35,25 +35,40 @@ func (d *Yun139) Init(ctx context.Context) error { if d.Authorization == "" { return fmt.Errorf("authorization is empty") } - decode, err := base64.StdEncoding.DecodeString(d.Authorization) - if err != nil { + switch d.Addition.Type { + case MetaPersonalNew: + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = "/" + } + return nil + case MetaPersonal: + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = "root" + } + fallthrough + case MetaFamily: + decode, err := base64.StdEncoding.DecodeString(d.Authorization) + if err != nil { + return err + } + decodeStr := string(decode) + splits := strings.Split(decodeStr, ":") + if len(splits) < 2 { + return fmt.Errorf("authorization is invalid, splits < 2") + } + d.Account = splits[1] + _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ + "qryUserExternInfoReq": base.Json{ + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + }, + }, nil) return err + default: + return errs.NotImplement } - decodeStr := string(decode) - splits := strings.Split(decodeStr, ":") - if len(splits) < 2 { - return fmt.Errorf("authorization is invalid, splits < 2") - } - d.Account = splits[1] - _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ - "qryUserExternInfoReq": base.Json{ - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, - }, - }, nil) - return err } func (d *Yun139) Drop(ctx context.Context) error { @@ -61,35 +76,65 @@ func (d *Yun139) Drop(ctx context.Context) error { } func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if d.isFamily() { - return d.familyGetFiles(dir.GetID()) - } else { + switch d.Addition.Type { + case MetaPersonalNew: + return d.personalGetFiles(dir.GetID()) + case MetaPersonal: return d.getFiles(dir.GetID()) + case MetaFamily: + return d.familyGetFiles(dir.GetID()) + default: + return nil, errs.NotImplement } } func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - u, err := d.getLink(file.GetID()) + var url string + var err error + switch d.Addition.Type { + case MetaPersonalNew: + url, err = d.personalGetLink(file.GetID()) + case MetaPersonal: + fallthrough + case MetaFamily: + url, err = d.getLink(file.GetID()) + default: + return nil, errs.NotImplement + } if err != nil { return nil, err } - return &model.Link{URL: u}, nil + return &model.Link{URL: url}, nil } func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - data := base.Json{ - "createCatalogExtReq": base.Json{ - "parentCatalogID": parentDir.GetID(), - "newCatalogName": dirName, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, + var err error + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "parentFileId": parentDir.GetID(), + "name": dirName, + "description": "", + "type": "folder", + "fileRenameMode": "force_rename", + } + pathname := "/hcy/file/create" + _, err = d.personalPost(pathname, data, nil) + case MetaPersonal: + data := base.Json{ + "createCatalogExtReq": base.Json{ + "parentCatalogID": parentDir.GetID(), + "newCatalogName": dirName, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, }, - }, - } - pathname := "/orchestration/personalCloud/catalog/v1.0/createCatalogExt" - if d.isFamily() { - data = base.Json{ + } + pathname := "/orchestration/personalCloud/catalog/v1.0/createCatalogExt" + _, err = d.post(pathname, data, nil) + case MetaFamily: + data := base.Json{ "cloudID": d.CloudID, "commonAccountInfo": base.Json{ "account": d.Account, @@ -97,147 +142,198 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin }, "docLibName": dirName, } - pathname = "/orchestration/familyCloud/cloudCatalog/v1.0/createCloudDoc" + pathname := "/orchestration/familyCloud/cloudCatalog/v1.0/createCloudDoc" + _, err = d.post(pathname, data, nil) + default: + err = errs.NotImplement } - _, err := d.post(pathname, data, nil) return err } func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - if d.isFamily() { - return nil, errs.NotImplement - } - var contentInfoList []string - var catalogInfoList []string - if srcObj.IsDir() { - catalogInfoList = append(catalogInfoList, srcObj.GetID()) - } else { - contentInfoList = append(contentInfoList, srcObj.GetID()) - } - data := base.Json{ - "createBatchOprTaskReq": base.Json{ - "taskType": 3, - "actionType": "304", - "taskInfo": base.Json{ - "contentInfoList": contentInfoList, - "catalogInfoList": catalogInfoList, - "newCatalogID": dstDir.GetID(), - }, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "fileIds": []string{srcObj.GetID()}, + "toParentFileId": dstDir.GetID(), + } + pathname := "/hcy/file/batchMove" + _, err := d.personalPost(pathname, data, nil) + if err != nil { + return nil, err + } + return srcObj, nil + case MetaPersonal: + var contentInfoList []string + var catalogInfoList []string + if srcObj.IsDir() { + catalogInfoList = append(catalogInfoList, srcObj.GetID()) + } else { + contentInfoList = append(contentInfoList, srcObj.GetID()) + } + data := base.Json{ + "createBatchOprTaskReq": base.Json{ + "taskType": 3, + "actionType": "304", + "taskInfo": base.Json{ + "contentInfoList": contentInfoList, + "catalogInfoList": catalogInfoList, + "newCatalogID": dstDir.GetID(), + }, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, }, - }, - } - pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" - _, err := d.post(pathname, data, nil) - if err != nil { - return nil, err + } + pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" + _, err := d.post(pathname, data, nil) + if err != nil { + return nil, err + } + return srcObj, nil + default: + return nil, errs.NotImplement } - return srcObj, nil } func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - if d.isFamily() { - return errs.NotImplement - } - var data base.Json - var pathname string - if srcObj.IsDir() { - data = base.Json{ - "catalogID": srcObj.GetID(), - "catalogName": newName, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, + var err error + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "fileId": srcObj.GetID(), + "name": newName, + "description": "", } - pathname = "/orchestration/personalCloud/catalog/v1.0/updateCatalogInfo" - } else { - data = base.Json{ - "contentID": srcObj.GetID(), - "contentName": newName, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, + pathname := "/hcy/file/update" + _, err = d.personalPost(pathname, data, nil) + case MetaPersonal: + var data base.Json + var pathname string + if srcObj.IsDir() { + data = base.Json{ + "catalogID": srcObj.GetID(), + "catalogName": newName, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname = "/orchestration/personalCloud/catalog/v1.0/updateCatalogInfo" + } else { + data = base.Json{ + "contentID": srcObj.GetID(), + "contentName": newName, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname = "/orchestration/personalCloud/content/v1.0/updateContentInfo" } - pathname = "/orchestration/personalCloud/content/v1.0/updateContentInfo" + _, err = d.post(pathname, data, nil) + default: + err = errs.NotImplement } - _, err := d.post(pathname, data, nil) return err } func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if d.isFamily() { - return errs.NotImplement - } - var contentInfoList []string - var catalogInfoList []string - if srcObj.IsDir() { - catalogInfoList = append(catalogInfoList, srcObj.GetID()) - } else { - contentInfoList = append(contentInfoList, srcObj.GetID()) - } - data := base.Json{ - "createBatchOprTaskReq": base.Json{ - "taskType": 3, - "actionType": 309, - "taskInfo": base.Json{ - "contentInfoList": contentInfoList, - "catalogInfoList": catalogInfoList, - "newCatalogID": dstDir.GetID(), - }, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, + var err error + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "fileIds": []string{srcObj.GetID()}, + "toParentFileId": dstDir.GetID(), + } + pathname := "/hcy/file/batchCopy" + _, err := d.personalPost(pathname, data, nil) + return err + case MetaPersonal: + var contentInfoList []string + var catalogInfoList []string + if srcObj.IsDir() { + catalogInfoList = append(catalogInfoList, srcObj.GetID()) + } else { + contentInfoList = append(contentInfoList, srcObj.GetID()) + } + data := base.Json{ + "createBatchOprTaskReq": base.Json{ + "taskType": 3, + "actionType": 309, + "taskInfo": base.Json{ + "contentInfoList": contentInfoList, + "catalogInfoList": catalogInfoList, + "newCatalogID": dstDir.GetID(), + }, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, }, - }, + } + pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" + _, err = d.post(pathname, data, nil) + default: + err = errs.NotImplement } - pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" - _, err := d.post(pathname, data, nil) return err } func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { - var contentInfoList []string - var catalogInfoList []string - if obj.IsDir() { - catalogInfoList = append(catalogInfoList, obj.GetID()) - } else { - contentInfoList = append(contentInfoList, obj.GetID()) - } - data := base.Json{ - "createBatchOprTaskReq": base.Json{ - "taskType": 2, - "actionType": 201, - "taskInfo": base.Json{ - "newCatalogID": "", - "contentInfoList": contentInfoList, - "catalogInfoList": catalogInfoList, - }, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, - }, - } - pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" - if d.isFamily() { - data = base.Json{ - "catalogList": catalogInfoList, - "contentList": contentInfoList, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "fileIds": []string{obj.GetID()}, + } + pathname := "/hcy/recyclebin/batchTrash" + _, err := d.personalPost(pathname, data, nil) + return err + case MetaPersonal: + fallthrough + case MetaFamily: + var contentInfoList []string + var catalogInfoList []string + if obj.IsDir() { + catalogInfoList = append(catalogInfoList, obj.GetID()) + } else { + contentInfoList = append(contentInfoList, obj.GetID()) + } + data := base.Json{ + "createBatchOprTaskReq": base.Json{ + "taskType": 2, + "actionType": 201, + "taskInfo": base.Json{ + "newCatalogID": "", + "contentInfoList": contentInfoList, + "catalogInfoList": catalogInfoList, + }, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, }, - "sourceCatalogType": 1002, - "taskType": 2, } - pathname = "/orchestration/familyCloud/batchOprTask/v1.0/createBatchOprTask" + pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" + if d.isFamily() { + data = base.Json{ + "catalogList": catalogInfoList, + "contentList": contentInfoList, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + "sourceCatalogType": 1002, + "taskType": 2, + } + pathname = "/orchestration/familyCloud/batchOprTask/v1.0/createBatchOprTask" + } + _, err := d.post(pathname, data, nil) + return err + default: + return errs.NotImplement } - _, err := d.post(pathname, data, nil) - return err } const ( @@ -257,94 +353,208 @@ func getPartSize(size int64) int64 { } func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - data := base.Json{ - "manualRename": 2, - "operation": 0, - "fileCount": 1, - "totalSize": 0, // 去除上传大小限制 - "uploadContentList": []base.Json{{ - "contentName": stream.GetName(), - "contentSize": 0, // 去除上传大小限制 - // "digest": "5a3231986ce7a6b46e408612d385bafa" - }}, - "parentCatalogID": dstDir.GetID(), - "newCatalogName": "", - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, - } - pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" - if d.isFamily() { - data = d.newJson(base.Json{ - "fileCount": 1, - "manualRename": 2, - "operation": 0, - "path": "", - "seqNo": "", - "totalSize": 0, - "uploadContentList": []base.Json{{ - "contentName": stream.GetName(), - "contentSize": 0, - // "digest": "5a3231986ce7a6b46e408612d385bafa" + switch d.Addition.Type { + case MetaPersonalNew: + var err error + fullHash := stream.GetHash().GetHash(utils.SHA256) + if len(fullHash) <= 0 { + tmpF, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + fullHash, err = utils.HashFile(utils.SHA256, tmpF) + if err != nil { + return err + } + } + // return errs.NotImplement + data := base.Json{ + "contentHash": fullHash, + "contentHashAlgorithm": "SHA256", + "contentType": "application/octet-stream", + "parallelUpload": false, + "partInfos": []base.Json{{ + "parallelHashCtx": base.Json{ + "partOffset": 0, + }, + "partNumber": 1, + "partSize": stream.GetSize(), }}, - }) - pathname = "/orchestration/familyCloud/content/v1.0/getFileUploadURL" - return errs.NotImplement - } - var resp UploadResp - _, err := d.post(pathname, data, &resp) - if err != nil { - return err - } - - // Progress - p := driver.NewProgress(stream.GetSize(), up) - - var partSize = getPartSize(stream.GetSize()) - part := (stream.GetSize() + partSize - 1) / partSize - if part == 0 { - part = 1 - } - for i := int64(0); i < part; i++ { - if utils.IsCanceled(ctx) { - return ctx.Err() + "size": stream.GetSize(), + "parentFileId": dstDir.GetID(), + "name": stream.GetName(), + "type": "file", + "fileRenameMode": "auto_rename", + } + pathname := "/hcy/file/create" + var resp PersonalUploadResp + _, err = d.personalPost(pathname, data, &resp) + if err != nil { + return err } - start := i * partSize - byteSize := stream.GetSize() - start - if byteSize > partSize { - byteSize = partSize + if resp.Data.Exist || resp.Data.RapidUpload { + return nil } - limitReader := io.LimitReader(stream, byteSize) + // Progress + p := driver.NewProgress(stream.GetSize(), up) + // Update Progress - r := io.TeeReader(limitReader, p) - req, err := http.NewRequest("POST", resp.Data.UploadResult.RedirectionURL, r) + r := io.TeeReader(stream, p) + + req, err := http.NewRequest("PUT", resp.Data.PartInfos[0].UploadUrl, r) if err != nil { return err } - req = req.WithContext(ctx) - req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) - req.Header.Set("contentSize", strconv.FormatInt(stream.GetSize(), 10)) - req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) - req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) - req.Header.Set("rangeType", "0") - req.ContentLength = byteSize + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprint(stream.GetSize())) + req.Header.Set("Origin", "https://yun.139.com") + req.Header.Set("Referer", "https://yun.139.com/") + req.ContentLength = stream.GetSize() res, err := base.HttpClient.Do(req) if err != nil { return err } + _ = res.Body.Close() log.Debugf("%+v", res) if res.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code: %d", res.StatusCode) } + + data = base.Json{ + "contentHash": fullHash, + "contentHashAlgorithm": "SHA256", + "fileId": resp.Data.FileId, + "uploadId": resp.Data.UploadId, + } + _, err = d.personalPost("/hcy/file/complete", data, nil) + if err != nil { + return err + } + return nil + case MetaPersonal: + fallthrough + case MetaFamily: + data := base.Json{ + "manualRename": 2, + "operation": 0, + "fileCount": 1, + "totalSize": 0, // 去除上传大小限制 + "uploadContentList": []base.Json{{ + "contentName": stream.GetName(), + "contentSize": 0, // 去除上传大小限制 + // "digest": "5a3231986ce7a6b46e408612d385bafa" + }}, + "parentCatalogID": dstDir.GetID(), + "newCatalogName": "", + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" + if d.isFamily() { + // data = d.newJson(base.Json{ + // "fileCount": 1, + // "manualRename": 2, + // "operation": 0, + // "path": "", + // "seqNo": "", + // "totalSize": 0, + // "uploadContentList": []base.Json{{ + // "contentName": stream.GetName(), + // "contentSize": 0, + // // "digest": "5a3231986ce7a6b46e408612d385bafa" + // }}, + // }) + // pathname = "/orchestration/familyCloud/content/v1.0/getFileUploadURL" + return errs.NotImplement + } + var resp UploadResp + _, err := d.post(pathname, data, &resp) + if err != nil { + return err + } + + // Progress + p := driver.NewProgress(stream.GetSize(), up) + + var partSize = getPartSize(stream.GetSize()) + part := (stream.GetSize() + partSize - 1) / partSize + if part == 0 { + part = 1 + } + for i := int64(0); i < part; i++ { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + + start := i * partSize + byteSize := stream.GetSize() - start + if byteSize > partSize { + byteSize = partSize + } + + limitReader := io.LimitReader(stream, byteSize) + // Update Progress + r := io.TeeReader(limitReader, p) + req, err := http.NewRequest("POST", resp.Data.UploadResult.RedirectionURL, r) + if err != nil { + return err + } + + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) + req.Header.Set("contentSize", strconv.FormatInt(stream.GetSize(), 10)) + req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) + req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) + req.Header.Set("rangeType", "0") + req.ContentLength = byteSize + + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + _ = res.Body.Close() + log.Debugf("%+v", res) + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + } + + return nil + default: + return errs.NotImplement } +} - return nil +func (d *Yun139) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch d.Addition.Type { + case MetaPersonalNew: + var resp base.Json + var uri string + data := base.Json{ + "category": "video", + "fileId": args.Obj.GetID(), + } + switch args.Method { + case "video_preview": + uri = "/hcy/videoPreview/getPreviewInfo" + default: + return nil, errs.NotSupport + } + _, err := d.personalPost(uri, data, &resp) + if err != nil { + return nil, err + } + return resp["data"], nil + default: + return nil, errs.NotImplement + } } var _ driver.Driver = (*Yun139)(nil) diff --git a/drivers/139/meta.go b/drivers/139/meta.go index 273ba14876d..416e63a796c 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -9,7 +9,7 @@ type Addition struct { //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` driver.RootID - Type string `json:"type" type:"select" options:"personal,family" default:"personal"` + Type string `json:"type" type:"select" options:"personal,family,personal_new" default:"personal"` CloudID string `json:"cloud_id"` } diff --git a/drivers/139/types.go b/drivers/139/types.go index 217aeb9f497..841aa9d3871 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -1,5 +1,11 @@ package _139 +const ( + MetaPersonal string = "personal" + MetaFamily string = "family" + MetaPersonalNew string = "personal_new" +) + type BaseResp struct { Success bool `json:"success"` Code string `json:"code"` @@ -185,3 +191,42 @@ type QueryContentListResp struct { RecallContent interface{} `json:"recallContent"` } `json:"data"` } + +type PersonalThumbnail struct { + Style string `json:"style"` + Url string `json:"url"` +} + +type PersonalFileItem struct { + FileId string `json:"fileId"` + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Thumbnails []PersonalThumbnail `json:"thumbnailUrls"` +} + +type PersonalListResp struct { + BaseResp + Data struct { + Items []PersonalFileItem `json:"items"` + NextPageCursor string `json:"nextPageCursor"` + } +} + +type PersonalPartInfo struct { + PartNumber int `json:"partNumber"` + UploadUrl string `json:"uploadUrl"` +} + +type PersonalUploadResp struct { + BaseResp + Data struct { + FileId string `json:"fileId"` + PartInfos []PersonalPartInfo `json:"partInfos"` + Exist bool `json:"exist"` + RapidUpload bool `json:"rapidUpload"` + UploadId string `json:"uploadId"` + } +} diff --git a/drivers/139/util.go b/drivers/139/util.go index 0f26b149955..a3627b6c8ae 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -252,3 +252,154 @@ func unicode(str string) string { textUnquoted := textQuoted[1 : len(textQuoted)-1] return textUnquoted } + +func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + url := "https://personal-kd-njs.yun.139.com" + pathname + req := base.RestyClient.R() + randStr := random.String(16) + ts := time.Now().Format("2006-01-02 15:04:05") + if callback != nil { + callback(req) + } + body, err := utils.Json.Marshal(req.Body) + if err != nil { + return nil, err + } + sign := calSign(string(body), ts, randStr) + svcType := "1" + if d.isFamily() { + svcType = "2" + } + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "Authorization": "Basic " + d.Authorization, + "Caller": "web", + "Cms-Device": "default", + "Mcloud-Channel": "1000101", + "Mcloud-Client": "10701", + "Mcloud-Route": "001", + "Mcloud-Sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), + "Mcloud-Version": "7.13.0", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/w/", + "x-DeviceInfo": "||9|7.13.0|chrome|120.0.0.0|||windows 10||zh-CN|||", + "x-huawei-channelSrc": "10000034", + "x-inner-ntwk": "2", + "x-m4c-caller": "PC", + "x-m4c-src": "10002", + "x-SvcType": svcType, + "X-Yun-Api-Version": "v1", + "X-Yun-App-Channel": "10000034", + "X-Yun-Channel-Source": "10000034", + "X-Yun-Client-Info": "||9|7.13.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", + "X-Yun-Module-Type": "100", + "X-Yun-Svc-Type": "1", + }) + + var e BaseResp + req.SetResult(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + log.Debugln(res.String()) + if !e.Success { + return nil, errors.New(e.Message) + } + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + return nil, err + } + } + return res.Body(), nil +} +func (d *Yun139) personalPost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + return d.personalRequest(pathname, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, resp) +} + +func getPersonalTime(t string) time.Time { + stamp, err := time.ParseInLocation("2006-01-02T15:04:05.999-07:00", t, utils.CNLoc) + if err != nil { + panic(err) + } + return stamp +} + +func (d *Yun139) personalGetFiles(fileId string) ([]model.Obj, error) { + files := make([]model.Obj, 0) + nextPageCursor := "" + for { + data := base.Json{ + "imageThumbnailStyleList": []string{"Small", "Large"}, + "orderBy": "updated_at", + "orderDirection": "DESC", + "pageInfo": base.Json{ + "pageCursor": nextPageCursor, + "pageSize": 100, + }, + "parentFileId": fileId, + } + var resp PersonalListResp + _, err := d.personalPost("/hcy/file/list", data, &resp) + if err != nil { + return nil, err + } + nextPageCursor = resp.Data.NextPageCursor + for _, item := range resp.Data.Items { + var isFolder = (item.Type == "folder") + var f model.Obj + if isFolder { + f = &model.Object{ + ID: item.FileId, + Name: item.Name, + Size: 0, + Modified: getPersonalTime(item.UpdatedAt), + Ctime: getPersonalTime(item.CreatedAt), + IsFolder: isFolder, + } + } else { + var Thumbnails = item.Thumbnails + var ThumbnailUrl string + if len(Thumbnails) > 0 { + ThumbnailUrl = Thumbnails[len(Thumbnails)-1].Url + } + f = &model.ObjThumb{ + Object: model.Object{ + ID: item.FileId, + Name: item.Name, + Size: item.Size, + Modified: getPersonalTime(item.UpdatedAt), + Ctime: getPersonalTime(item.CreatedAt), + IsFolder: isFolder, + }, + Thumbnail: model.Thumbnail{Thumbnail: ThumbnailUrl}, + } + } + files = append(files, f) + } + if len(nextPageCursor) == 0 { + break + } + } + return files, nil +} + +func (d *Yun139) personalGetLink(fileId string) (string, error) { + data := base.Json{ + "fileId": fileId, + } + res, err := d.personalPost("/hcy/file/getDownloadUrl", + data, nil) + if err != nil { + return "", err + } + var cdnUrl = jsoniter.Get(res, "data", "cdnUrl").ToString() + if cdnUrl != "" { + return cdnUrl, nil + } else { + return jsoniter.Get(res, "data", "url").ToString(), nil + } +} From 54f7b21a73fba63625b43644bb62e2cc5fd4ff1c Mon Sep 17 00:00:00 2001 From: tonsr Date: Sun, 17 Dec 2023 15:21:32 +0800 Subject: [PATCH 056/659] fix(123): api sign error (#5689 close #5083) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix：123 driver connect error * feat: calculate sign with pure go --------- Co-authored-by: tangminghao Co-authored-by: Andy Hsu --- drivers/123/util.go | 141 +++++++++++++++++++++++++++++++++++++++----- go.mod | 12 ++-- go.sum | 26 ++++---- 3 files changed, 143 insertions(+), 36 deletions(-) diff --git a/drivers/123/util.go b/drivers/123/util.go index e9eb63375d1..1a86e1bdef2 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -3,12 +3,18 @@ package _123 import ( "errors" "fmt" + "hash/crc32" + "math" + "math/rand" "net/http" + "net/url" "strconv" + "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/go-resty/resty/v2" + resty "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" ) @@ -18,7 +24,7 @@ const ( Api = "https://www.123pan.com/api" AApi = "https://www.123pan.com/a/api" BApi = "https://www.123pan.com/b/api" - MainApi = Api + MainApi = BApi SignIn = MainApi + "/user/sign_in" Logout = MainApi + "/user/logout" UserInfo = MainApi + "/user/info" @@ -37,6 +43,104 @@ const ( //AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) +func signPath(path string, os string, version string) (k string, v string) { + table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'} + random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64())) + now := time.Now().In(time.FixedZone("CST", 8*3600)) + timestamp := fmt.Sprint(now.Unix()) + nowStr := []byte(now.Format("200601021504")) + for i := 0; i < len(nowStr); i++ { + nowStr[i] = table[nowStr[i]-48] + } + timeSign := fmt.Sprint(crc32.ChecksumIEEE(nowStr)) + data := strings.Join([]string{timestamp, random, path, os, version, timeSign}, "|") + dataSign := fmt.Sprint(crc32.ChecksumIEEE([]byte(data))) + return timeSign, strings.Join([]string{timestamp, random, dataSign}, "-") +} + +func GetApi(rawUrl string) string { + u, _ := url.Parse(rawUrl) + query := u.Query() + query.Add(signPath(u.Path, "web", "3")) + u.RawQuery = query.Encode() + return u.String() +} + +//func GetApi(url string) string { +// vm := js.New() +// vm.Set("url", url[22:]) +// r, err := vm.RunString(` +// (function(e){ +// function A(t, e) { +// e = 1 < arguments.length && void 0 !== e ? e : 10; +// for (var n = function() { +// for (var t = [], e = 0; e < 256; e++) { +// for (var n = e, r = 0; r < 8; r++) +// n = 1 & n ? 3988292384 ^ n >>> 1 : n >>> 1; +// t[e] = n +// } +// return t +// }(), r = function(t) { +// t = t.replace(/\\r\\n/g, "\\n"); +// for (var e = "", n = 0; n < t.length; n++) { +// var r = t.charCodeAt(n); +// r < 128 ? e += String.fromCharCode(r) : e = 127 < r && r < 2048 ? (e += String.fromCharCode(r >> 6 | 192)) + String.fromCharCode(63 & r | 128) : (e = (e += String.fromCharCode(r >> 12 | 224)) + String.fromCharCode(r >> 6 & 63 | 128)) + String.fromCharCode(63 & r | 128) +// } +// return e +// }(t), a = -1, i = 0; i < r.length; i++) +// a = a >>> 8 ^ n[255 & (a ^ r.charCodeAt(i))]; +// return (a = (-1 ^ a) >>> 0).toString(e) +// } +// +// function v(t) { +// return (v = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function(t) { +// return typeof t +// } +// : function(t) { +// return t && "function" == typeof Symbol && t.constructor === Symbol && t !== Symbol.prototype ? "symbol" : typeof t +// } +// )(t) +// } +// +// for (p in a = Math.round(1e7 * Math.random()), +// o = Math.round(((new Date).getTime() + 60 * (new Date).getTimezoneOffset() * 1e3 + 288e5) / 1e3).toString(), +// m = ["a", "d", "e", "f", "g", "h", "l", "m", "y", "i", "j", "n", "o", "p", "k", "q", "r", "s", "t", "u", "b", "c", "v", "w", "s", "z"], +// u = function(t, e, n) { +// var r; +// n = 2 < arguments.length && void 0 !== n ? n : 8; +// return 0 === arguments.length ? null : (r = "object" === v(t) ? t : (10 === "".concat(t).length && (t = 1e3 * Number.parseInt(t)), +// new Date(t)), +// t += 6e4 * new Date(t).getTimezoneOffset(), +// { +// y: (r = new Date(t + 36e5 * n)).getFullYear(), +// m: r.getMonth() + 1 < 10 ? "0".concat(r.getMonth() + 1) : r.getMonth() + 1, +// d: r.getDate() < 10 ? "0".concat(r.getDate()) : r.getDate(), +// h: r.getHours() < 10 ? "0".concat(r.getHours()) : r.getHours(), +// f: r.getMinutes() < 10 ? "0".concat(r.getMinutes()) : r.getMinutes() +// }) +// }(o), +// h = u.y, +// g = u.m, +// l = u.d, +// c = u.h, +// u = u.f, +// d = [h, g, l, c, u].join(""), +// f = [], +// d) +// f.push(m[Number(d[p])]); +// return h = A(f.join("")), +// g = A("".concat(o, "|").concat(a, "|").concat(e, "|").concat("web", "|").concat("3", "|").concat(h)), +// "".concat(h, "=").concat(o, "-").concat(a, "-").concat(g); +// })(url) +// `) +// if err != nil { +// fmt.Println(err) +// return url +// } +// v, _ := r.Export().(string) +// return url + "?" + v +//} + func (d *Pan123) login() error { var body base.Json if utils.IsEmailFormat(d.Username) { @@ -57,8 +161,8 @@ func (d *Pan123) login() error { "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "user-agent": "Dart/2.19(dart:io)", - "platform": "android", - "app-version": "36", + "platform": "web", + "app-version": "3", //"user-agent": base.UserAgent, }). SetBody(body).Post(SignIn) @@ -93,9 +197,9 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, - "user-agent": "Dart/2.19(dart:io)", - "platform": "android", - "app-version": "36", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", + "platform": "web", + "app-version": "3", //"user-agent": base.UserAgent, }) if callback != nil { @@ -109,7 +213,7 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r // return nil, err //} //req.SetQueryParam("auth-key", *authKey) - res, err := req.Execute(method, url) + res, err := req.Execute(method, GetApi(url)) if err != nil { return nil, err } @@ -134,14 +238,19 @@ func (d *Pan123) getFiles(parentId string) ([]File, error) { for { var resp Files query := map[string]string{ - "driveId": "0", - "limit": "100", - "next": "0", - "orderBy": d.OrderBy, - "orderDirection": d.OrderDirection, - "parentFileId": parentId, - "trashed": "false", - "Page": strconv.Itoa(page), + "driveId": "0", + "limit": "100", + "next": "0", + "orderBy": d.OrderBy, + "orderDirection": d.OrderDirection, + "parentFileId": parentId, + "trashed": "false", + "SearchData": "", + "Page": strconv.Itoa(page), + "OnlyLookAbnormalFile": "0", + "event": "homeListFile", + "operateType": "4", + "inDirectSpace": "false", } _, err := d.request(FileList, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) diff --git a/go.mod b/go.mod index e6ced69b695..e66a5eb0d10 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 - github.com/go-resty/resty/v2 v2.9.1 + github.com/go-resty/resty/v2 v2.10.0 github.com/go-webauthn/webauthn v0.8.6 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.3.1 @@ -48,10 +48,10 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.16.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/image v0.11.0 - golang.org/x/net v0.17.0 + golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.12.0 golang.org/x/time v0.3.0 google.golang.org/appengine v1.6.7 @@ -191,9 +191,9 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.3.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/api v0.134.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/grpc v1.57.0 // indirect diff --git a/go.sum b/go.sum index 36c2b49a252..d79017efbff 100644 --- a/go.sum +++ b/go.sum @@ -160,8 +160,8 @@ github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4 github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= -github.com/go-resty/resty/v2 v2.9.1 h1:PIgGx4VrHvag0juCJ4dDv3MiFRlDmP0vicBucwf+gLM= -github.com/go-resty/resty/v2 v2.9.1/go.mod h1:4/GYJVjh9nhkhGR6AUNW3XhpDYNUr+Uvy9gV/VGZIy4= +github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= +github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.8.6 h1:bKMtL1qzd2WTFkf1mFTVbreYrwn7dsYmEPjTq6QN90E= @@ -449,8 +449,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= -github.com/xhofe/tache v0.1.0 h1:W0uoyLWCmUEQudXwB93owdlGSlN8gwZmiiDlKFCerKA= -github.com/xhofe/tache v0.1.0/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/tache v0.1.1 h1:O5QY4cVjIGELx3UGh6LbVAc18MWGXgRNQjMt72x6w/8= github.com/xhofe/tache v0.1.1/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -474,9 +472,9 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -495,9 +493,9 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -529,17 +527,17 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -549,10 +547,10 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From e91c42c9dc11990c11b24f07d82dafca16e48522 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 17 Dec 2023 15:45:27 +0800 Subject: [PATCH 057/659] fix(alist_v3): timeout on upload (close #5465) --- drivers/alist_v3/driver.go | 5 +++-- drivers/alist_v3/util.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index e2deabacccb..f46e68d07a4 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -8,6 +8,7 @@ import ( "path" "strconv" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" @@ -174,13 +175,13 @@ func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { } func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - _, err := d.request("/fs/put", http.MethodPut, func(req *resty.Request) { + _, err := d.requestWithTimeout("/fs/put", http.MethodPut, func(req *resty.Request) { req.SetHeader("File-Path", path.Join(dstDir.GetPath(), stream.GetName())). SetHeader("Password", d.MetaPassword). SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). SetContentLength(true). SetBody(io.ReadCloser(stream)) - }) + }, time.Hour*6) return err } diff --git a/drivers/alist_v3/util.go b/drivers/alist_v3/util.go index bf47c61289a..978f3ac0568 100644 --- a/drivers/alist_v3/util.go +++ b/drivers/alist_v3/util.go @@ -3,6 +3,7 @@ package alist_v3 import ( "fmt" "net/http" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/op" @@ -56,3 +57,33 @@ func (d *AListV3) request(api, method string, callback base.ReqCallback, retry . } return res.Body(), nil } + +func (d *AListV3) requestWithTimeout(api, method string, callback base.ReqCallback, timeout time.Duration, retry ...bool) ([]byte, error) { + url := d.Address + "/api" + api + client := base.NewRestyClient().SetTimeout(timeout) + req := client.R() + req.SetHeader("Authorization", d.Token) + if callback != nil { + callback(req) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + log.Debugf("[alist_v3] response body: %s", res.String()) + if res.StatusCode() >= 400 { + return nil, fmt.Errorf("request failed, status: %s", res.Status()) + } + code := utils.Json.Get(res.Body(), "code").ToInt() + if code != 200 { + if (code == 401 || code == 403) && !utils.IsBool(retry...) { + err = d.login() + if err != nil { + return nil, err + } + return d.requestWithTimeout(api, method, callback, timeout, true) + } + return nil, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) + } + return res.Body(), nil +} From ab216ed170878c608b3c3d3ed1a35a5b248e3912 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 17 Dec 2023 22:58:26 +0800 Subject: [PATCH 058/659] fix(onedrive): rename object in root folder (close #5468) --- drivers/onedrive/driver.go | 39 ++++++++++++++++++++++++++++ drivers/onedrive_app/driver.go | 39 ++++++++++++++++++++++++++++ internal/op/fs.go | 46 ++++++++++++++++++---------------- 3 files changed, 102 insertions(+), 22 deletions(-) diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index 50e129d99c5..319fd906b7d 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "path" + "sync" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -19,6 +20,8 @@ type Onedrive struct { model.Storage Addition AccessToken string + root *Object + mutex sync.Mutex } func (d *Onedrive) Config() driver.Config { @@ -40,6 +43,42 @@ func (d *Onedrive) Drop(ctx context.Context) error { return nil } +func (d *Onedrive) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root != nil { + return d.root, nil + } + d.mutex.Lock() + defer d.mutex.Unlock() + root := &Object{ + ObjThumb: model.ObjThumb{ + Object: model.Object{ + ID: "root", + Path: d.RootFolderPath, + Name: "root", + Size: 0, + Modified: d.Modified, + Ctime: d.Modified, + IsFolder: true, + }, + }, + ParentID: "", + } + if !utils.PathEqual(d.RootFolderPath, "/") { + // get root folder id + url := d.GetMetaUrl(false, d.RootFolderPath) + var resp struct { + Id string `json:"id"` + } + _, err := d.Request(url, http.MethodGet, nil, &resp) + if err != nil { + return nil, err + } + root.ID = resp.Id + } + d.root = root + return d.root, nil +} + func (d *Onedrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetPath()) if err != nil { diff --git a/drivers/onedrive_app/driver.go b/drivers/onedrive_app/driver.go index 84ff878a4d2..8a924341bb7 100644 --- a/drivers/onedrive_app/driver.go +++ b/drivers/onedrive_app/driver.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "path" + "sync" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -19,6 +20,8 @@ type OnedriveAPP struct { model.Storage Addition AccessToken string + root *Object + mutex sync.Mutex } func (d *OnedriveAPP) Config() driver.Config { @@ -40,6 +43,42 @@ func (d *OnedriveAPP) Drop(ctx context.Context) error { return nil } +func (d *OnedriveAPP) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root != nil { + return d.root, nil + } + d.mutex.Lock() + defer d.mutex.Unlock() + root := &Object{ + ObjThumb: model.ObjThumb{ + Object: model.Object{ + ID: "root", + Path: d.RootFolderPath, + Name: "root", + Size: 0, + Modified: d.Modified, + Ctime: d.Modified, + IsFolder: true, + }, + }, + ParentID: "", + } + if !utils.PathEqual(d.RootFolderPath, "/") { + // get root folder id + url := d.GetMetaUrl(false, d.RootFolderPath) + var resp struct { + Id string `json:"id"` + } + _, err := d.Request(url, http.MethodGet, nil, &resp) + if err != nil { + return nil, err + } + root.ID = resp.Id + } + d.root = root + return d.root, nil +} + func (d *OnedriveAPP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetPath()) if err != nil { diff --git a/internal/op/fs.go b/internal/op/fs.go index 9fe7d5e6a3f..4f0cbbdd3ae 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -177,30 +177,32 @@ func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, er // is root folder if utils.PathEqual(path, "/") { var rootObj model.Obj - switch r := storage.GetAddition().(type) { - case driver.IRootId: - rootObj = &model.Object{ - ID: r.GetRootId(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, - } - case driver.IRootPath: - rootObj = &model.Object{ - Path: r.GetRootPath(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, + if getRooter, ok := storage.(driver.GetRooter); ok { + obj, err := getRooter.GetRoot(ctx) + if err != nil { + return nil, errors.WithMessage(err, "failed get root obj") } - default: - if storage, ok := storage.(driver.GetRooter); ok { - obj, err := storage.GetRoot(ctx) - if err != nil { - return nil, errors.WithMessage(err, "failed get root obj") + rootObj = obj + } else { + switch r := storage.GetAddition().(type) { + case driver.IRootId: + rootObj = &model.Object{ + ID: r.GetRootId(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + } + case driver.IRootPath: + rootObj = &model.Object{ + Path: r.GetRootPath(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, } - rootObj = obj + default: + return nil, errors.Errorf("please implement IRootPath or IRootId or GetRooter method") } } if rootObj == nil { From 3eca38e5993046a3823c08b34b9358330018a6e7 Mon Sep 17 00:00:00 2001 From: "Feng.YJ" <32027253+huiyifyj@users.noreply.github.com> Date: Sun, 24 Dec 2023 15:21:17 +0800 Subject: [PATCH 059/659] feat: add support for client-side discoverable WebAuthn login (#5722) * Add support for client-side discoverable in begin login Use `(*webauthn.WebAuthn).BeginDiscoverableLogin()` to handle client-side discoverable login. * Upgrade github.com/go-webauthn/webauthn to v0.10.0 Upgrade [go-webauthn/webauthn](github.com/go-webauthn/webauthn) library to latest. The convenient finish login function (as FinishDiscoverableLogin) for discoverable functions has been added in the v0.9.0. [^1] --- [^1]: https://github.com/go-webauthn/webauthn/releases/tag/v0.9.0 * Add support for client-side discoverable in validating login Use `(*webauthn.WebAuthn).FinishDiscoverableLogin()` to handle client-side discoverable login. > **NOTE**: - The first param `rawID` in this callback function is unnecessary to check, it's handled by the third-party webauthn library later. - `userHandle` param is equal to the ID returned by (User).WebAuthnID() function. --- go.mod | 10 +++---- go.sum | 20 ++++++------- server/handles/webauthn.go | 61 ++++++++++++++++++++++++-------------- 3 files changed, 54 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index e66a5eb0d10..2e1cdb50354 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,9 @@ require ( github.com/gin-contrib/cors v1.4.0 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.10.0 - github.com/go-webauthn/webauthn v0.8.6 + github.com/go-webauthn/webauthn v0.9.4 github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/google/uuid v1.3.1 + github.com/google/uuid v1.4.0 github.com/gorilla/websocket v1.5.0 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/go-ipfs-api v0.7.0 @@ -99,7 +99,7 @@ require ( github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/fxamacker/cbor/v2 v2.4.0 // indirect + github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gaoyb7/115drive-webdav v0.1.8 // indirect github.com/geoffgarside/ber v1.1.0 // indirect @@ -110,9 +110,9 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/go-webauthn/x v0.1.4 // indirect + github.com/go-webauthn/x v0.1.5 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index d79017efbff..e6327f0609b 100644 --- a/go.sum +++ b/go.sum @@ -126,8 +126,8 @@ github.com/foxxorcat/mopan-sdk-go v0.1.4/go.mod h1:iWHA2JFhzmKR28ySp1ON0g6DjLaYt github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= -github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE= +github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw= @@ -164,18 +164,18 @@ github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+Ck github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-webauthn/webauthn v0.8.6 h1:bKMtL1qzd2WTFkf1mFTVbreYrwn7dsYmEPjTq6QN90E= -github.com/go-webauthn/webauthn v0.8.6/go.mod h1:emwVLMCI5yx9evTTvr0r+aOZCdWJqMfbRhF0MufyUog= -github.com/go-webauthn/x v0.1.4 h1:sGmIFhcY70l6k7JIDfnjVBiAAFEssga5lXIUXe0GtAs= -github.com/go-webauthn/x v0.1.4/go.mod h1:75Ug0oK6KYpANh5hDOanfDI+dvPWHk788naJVG/37H8= +github.com/go-webauthn/webauthn v0.9.4 h1:YxvHSqgUyc5AK2pZbqkWWR55qKeDPhP8zLDr6lpIc2g= +github.com/go-webauthn/webauthn v0.9.4/go.mod h1:LqupCtzSef38FcxzaklmOn7AykGKhAhr9xlRbdbgnTw= +github.com/go-webauthn/x v0.1.5 h1:V2TCzDU2TGLd0kSZOXdrqDVV5JB9ILnKxA9S53CSBw0= +github.com/go-webauthn/x v0.1.5/go.mod h1:qbzWwcFcv4rTwtCLOZd+icnr6B7oSsAGZJqlt8cukqY= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= +github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -197,8 +197,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index 28a89522cfc..1bd1884ef11 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -2,6 +2,7 @@ package handles import ( "encoding/base64" + "encoding/binary" "encoding/json" "fmt" @@ -13,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "github.com/go-webauthn/webauthn/protocol" "github.com/go-webauthn/webauthn/webauthn" ) @@ -22,28 +24,30 @@ func BeginAuthnLogin(c *gin.Context) { common.ErrorStrResp(c, "WebAuthn is not enabled", 403) return } - username := c.Query("username") - if username == "" { - common.ErrorStrResp(c, "empty or no username provided", 400) - return - } - user, err := db.GetUserByName(username) - if err != nil { - common.ErrorResp(c, err, 400) - return - } authnInstance, err := authn.NewAuthnInstance(c.Request) if err != nil { common.ErrorResp(c, err, 400) return } - options, sessionData, err := authnInstance.BeginLogin(user) - + var ( + options *protocol.CredentialAssertion + sessionData *webauthn.SessionData + ) + if username := c.Query("username"); username != "" { + var user *model.User + user, err = db.GetUserByName(username) + if err == nil { + options, sessionData, err = authnInstance.BeginLogin(user) + } + } else { // client-side discoverable login + options, sessionData, err = authnInstance.BeginDiscoverableLogin() + } if err != nil { common.ErrorResp(c, err, 400) return } + val, err := json.Marshal(sessionData) if err != nil { common.ErrorResp(c, err, 400) @@ -61,20 +65,13 @@ func FinishAuthnLogin(c *gin.Context) { common.ErrorStrResp(c, "WebAuthn is not enabled", 403) return } - username := c.Query("username") - user, err := db.GetUserByName(username) + authnInstance, err := authn.NewAuthnInstance(c.Request) if err != nil { common.ErrorResp(c, err, 400) return } sessionDataString := c.GetHeader("session") - - authnInstance, err := authn.NewAuthnInstance(c.Request) - if err != nil { - common.ErrorResp(c, err, 400) - return - } sessionDataBytes, err := base64.StdEncoding.DecodeString(sessionDataString) if err != nil { common.ErrorResp(c, err, 400) @@ -87,8 +84,28 @@ func FinishAuthnLogin(c *gin.Context) { return } - _, err = authnInstance.FinishLogin(user, sessionData, c.Request) - + var user *model.User + if username := c.Query("username"); username != "" { + user, err = db.GetUserByName(username) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = authnInstance.FinishLogin(user, sessionData, c.Request) + } else { // client-side discoverable login + _, err = authnInstance.FinishDiscoverableLogin(func(_, userHandle []byte) (webauthn.User, error) { + // first param `rawID` in this callback function is equal to ID in webauthn.Credential, + // but it's unnnecessary to check it. + // userHandle param is equal to (User).WebAuthnID(). + userID := uint(binary.LittleEndian.Uint64(userHandle)) + user, err = db.GetUserById(userID) + if err != nil { + return nil, err + } + + return user, nil + }, sessionData, c.Request) + } if err != nil { common.ErrorResp(c, err, 400) return From 299bfb4d7b2edfbb0c02b3979fb370fd355a0e6b Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Mon, 25 Dec 2023 11:28:57 +0800 Subject: [PATCH 060/659] feat(115): support 302 redirect (#5733) --- drivers/115/driver.go | 7 ++++++- drivers/115/meta.go | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 15f6b4087b4..96bf9d45837 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -63,8 +63,13 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err := d.WaitLimit(ctx); err != nil { return nil, err } + var userAgent = args.Header.Get("User-Agent") + if userAgent == "" { + userAgent = driver115.UA115Browser + } + downloadInfo, err := d.client. - DownloadWithUA(file.(*FileObj).PickCode, driver115.UA115Browser) + DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } diff --git a/drivers/115/meta.go b/drivers/115/meta.go index 16ec22cdab8..d3e937bf8e7 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -16,7 +16,7 @@ type Addition struct { var config = driver.Config{ Name: "115 Cloud", DefaultRoot: "0", - OnlyProxy: true, + //OnlyProxy: true, //OnlyLocal: true, NoOverwriteUpload: true, } From 697a0ed2d3864e532f17d7cf0027269eaf0ab8cf Mon Sep 17 00:00:00 2001 From: Guobao <8208908+JeremieCHN@users.noreply.github.com> Date: Sun, 31 Dec 2023 13:46:13 +0800 Subject: [PATCH 061/659] feat: add ldap login support (#5706) * feat: add ldap login support * fix: ldap permission config group --- go.mod | 2 + go.sum | 4 + internal/bootstrap/data/setting.go | 11 ++ internal/conf/const.go | 11 ++ internal/model/setting.go | 1 + server/handles/ldap_login.go | 159 +++++++++++++++++++++++++++++ server/router.go | 1 + 7 files changed, 189 insertions(+) create mode 100644 server/handles/ldap_login.go diff --git a/go.mod b/go.mod index 2e1cdb50354..30ff10b3fbb 100644 --- a/go.mod +++ b/go.mod @@ -198,6 +198,8 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/grpc v1.57.0 // indirect google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect + gopkg.in/ldap.v3 v3.1.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index e6327f0609b..7d7048f8377 100644 --- a/go.sum +++ b/go.sum @@ -574,11 +574,15 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= +gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 09bf79b0418..0aee410aab5 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -165,6 +165,17 @@ func InitialSettings() []model.SettingItem { {Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC}, + + // ldap settings + {Key: conf.LdapLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PUBLIC}, + {Key: conf.LdapServer, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapManagerDN, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapManagerPassword, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapUserSearchBase, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapUserSearchFilter, Value: "(uid=%s)", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapDefaultDir, Value: "/", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapDefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapLoginTips, Value: "login with ldap", Type: conf.TypeString, Group: model.LDAP, Flag: model.PUBLIC}, } initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { diff --git a/internal/conf/const.go b/internal/conf/const.go index eb70602ad3f..5ffdef2b577 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -73,6 +73,17 @@ const ( SSODefaultPermission = "sso_default_permission" SSOCompatibilityMode = "sso_compatibility_mode" + //ldap + LdapLoginEnabled = "ldap_login_enabled" + LdapServer = "ldap_server" + LdapManagerDN = "ldap_manager_dn" + LdapManagerPassword = "ldap_manager_password" + LdapUserSearchBase = "ldap_user_search_base" + LdapUserSearchFilter = "ldap_user_search_filter" + LdapDefaultPermission = "ldap_default_permission" + LdapDefaultDir = "ldap_default_dir" + LdapLoginTips = "ldap_login_tips" + // qbittorrent QbittorrentUrl = "qbittorrent_url" QbittorrentSeedtime = "qbittorrent_seedtime" diff --git a/internal/model/setting.go b/internal/model/setting.go index 3b2c30f1361..b561ad6b221 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -9,6 +9,7 @@ const ( OFFLINE_DOWNLOAD INDEX SSO + LDAP ) const ( diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go new file mode 100644 index 00000000000..b52e108249c --- /dev/null +++ b/server/handles/ldap_login.go @@ -0,0 +1,159 @@ +package handles + +import ( + "crypto/tls" + "errors" + "fmt" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "gopkg.in/ldap.v3" +) + +func LoginLdap(c *gin.Context) { + var req LoginReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + loginLdap(c, &req) +} + +func loginLdap(c *gin.Context, req *LoginReq) { + enabled := setting.GetBool(conf.LdapLoginEnabled) + if !enabled { + common.ErrorStrResp(c, "ldap is not enabled", 403) + return + } + + // check count of login + ip := c.ClientIP() + count, ok := loginCache.Get(ip) + if ok && count >= defaultTimes { + common.ErrorStrResp(c, "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.", 429) + loginCache.Expire(ip, defaultDuration) + return + } + + // Auth start + ldapServer := setting.GetStr(conf.LdapServer) + ldapManagerDN := setting.GetStr(conf.LdapManagerDN) + ldapManagerPassword := setting.GetStr(conf.LdapManagerPassword) + ldapUserSearchBase := setting.GetStr(conf.LdapUserSearchBase) + ldapUserSearchFilter := setting.GetStr(conf.LdapUserSearchFilter) // (uid=%s) + + var tlsEnabled bool = false + if strings.HasPrefix(ldapServer, "ldaps://") { + tlsEnabled = true + ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") + } else if strings.HasPrefix(ldapServer, "ldap://") { + ldapServer = strings.TrimPrefix(ldapServer, "ldap://") + } + + l, err := ldap.Dial("tcp", ldapServer) + if err != nil { + utils.Log.Errorf("failed to connect to LDAP: %v", err) + common.ErrorResp(c, err, 500) + return + } + defer l.Close() + + if tlsEnabled { + // Reconnect with TLS + err = l.StartTLS(&tls.Config{InsecureSkipVerify: true}) + if err != nil { + utils.Log.Errorf("failed to start tls: %v", err) + common.ErrorResp(c, err, 500) + return + } + } + + // First bind with a read only user + if ldapManagerDN != "" && ldapManagerPassword != "" { + err = l.Bind(ldapManagerDN, ldapManagerPassword) + if err != nil { + utils.Log.Errorf("Failed to bind to LDAP: %v", err) + common.ErrorResp(c, err, 500) + return + } + } + + // Search for the given username + searchRequest := ldap.NewSearchRequest( + ldapUserSearchBase, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + fmt.Sprintf(ldapUserSearchFilter, req.Username), + []string{"dn"}, + nil, + ) + sr, err := l.Search(searchRequest) + if err != nil { + utils.Log.Errorf("LDAP search failed: %v", err) + common.ErrorResp(c, err, 500) + return + } + if len(sr.Entries) != 1 { + utils.Log.Errorf("User does not exist or too many entries returned") + common.ErrorResp(c, err, 500) + return + } + userDN := sr.Entries[0].DN + + // Bind as the user to verify their password + err = l.Bind(userDN, req.Password) + if err != nil { + utils.Log.Errorf("Failed to auth. %v", err) + common.ErrorResp(c, err, 400) + loginCache.Set(ip, count+1) + return + } else { + utils.Log.Infof("Auth successful username:%s", req.Username) + } + // Auth finished + + user, err := op.GetUserByName(req.Username) + if err != nil { + user, err = ladpRegister(req.Username) + if err != nil { + common.ErrorResp(c, err, 400) + loginCache.Set(ip, count+1) + return + } + } + + // generate token + token, err := common.GenerateToken(user) + if err != nil { + common.ErrorResp(c, err, 400, true) + return + } + common.SuccessResp(c, gin.H{"token": token}) + loginCache.Del(ip) +} + +func ladpRegister(username string) (*model.User, error) { + if username == "" { + return nil, errors.New("cannot get username from ldap provider") + } + user := &model.User{ + ID: 0, + Username: username, + Password: random.String(16), + Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), + BasePath: setting.GetStr(conf.LdapDefaultDir), + Role: 0, + Disabled: false, + } + if err := db.CreateUser(user); err != nil { + return nil, err + } + return user, nil +} diff --git a/server/router.go b/server/router.go index 588cc071bfb..1421f66595d 100644 --- a/server/router.go +++ b/server/router.go @@ -48,6 +48,7 @@ func Init(e *gin.Engine) { api.POST("/auth/login", handles.Login) api.POST("/auth/login/hash", handles.LoginHash) + api.POST("/auth/login/ldap", handles.LoginLdap) auth.GET("/me", handles.CurrentUser) auth.POST("/me/update", handles.UpdateCurrent) auth.POST("/auth/2fa/generate", handles.Generate2FA) From 6b8f35e7fad463ae65883d768c1748fa66616a9c Mon Sep 17 00:00:00 2001 From: xiaofei <122727418+xiaozhou26@users.noreply.github.com> Date: Sun, 31 Dec 2023 14:29:14 +0800 Subject: [PATCH 062/659] feat(alipan): replace domain (#5751 close #5747) --- README.md | 4 ++-- README_cn.md | 4 ++-- README_ja.md | 4 ++-- drivers/aliyundrive/driver.go | 22 +++++++++++----------- drivers/aliyundrive/util.go | 16 ++++++++-------- drivers/aliyundrive_open/meta.go | 2 +- drivers/aliyundrive_share/driver.go | 8 ++++---- drivers/aliyundrive_share/util.go | 6 +++--- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 5f4d8ef4270..ef68e01656b 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] Multiple storages - [x] Local storage - - [x] [Aliyundrive](https://www.aliyundrive.com/) + - [x] [Aliyundrive](https://www.alipan.com/) - [x] OneDrive / Sharepoint ([global](https://www.office.com/), [cn](https://portal.partner.microsoftonline.cn),de,us) - [x] [189cloud](https://cloud.189.cn) (Personal, Family) - [x] [GoogleDrive](https://drive.google.com/) @@ -66,7 +66,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com/) - - [x] [Aliyundrive share](https://www.aliyundrive.com/) + - [x] [Aliyundrive share](https://www.alipan.com/) - [x] [Google photo](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) - [x] [Baidu photo](https://photo.baidu.com/) diff --git a/README_cn.md b/README_cn.md index 6af8aeaf1af..a5dfab47f45 100644 --- a/README_cn.md +++ b/README_cn.md @@ -45,7 +45,7 @@ - [x] 多种存储 - [x] 本地存储 - - [x] [阿里云盘](https://www.aliyundrive.com/) + - [x] [阿里云盘](https://www.alipan.com/) - [x] OneDrive / Sharepoint（[国际版](https://www.office.com/), [世纪互联](https://portal.partner.microsoftonline.cn),de,us） - [x] [天翼云盘](https://cloud.189.cn) (个人云, 家庭云) - [x] [GoogleDrive](https://drive.google.com/) @@ -65,7 +65,7 @@ - [x] [夸克网盘](https://pan.quark.cn) - [x] [迅雷网盘](https://pan.xunlei.com) - [x] [蓝奏云](https://www.lanzou.com/) - - [x] [阿里云盘分享](https://www.aliyundrive.com/) + - [x] [阿里云盘分享](https://www.alipan.com/) - [x] [谷歌相册](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) - [x] [一刻相册](https://photo.baidu.com/) diff --git a/README_ja.md b/README_ja.md index b873947fc86..3bcdd8de3d5 100644 --- a/README_ja.md +++ b/README_ja.md @@ -45,7 +45,7 @@ - [x] マルチストレージ - [x] ローカルストレージ - - [x] [Aliyundrive](https://www.aliyundrive.com/) + - [x] [Aliyundrive](https://www.alipan.com/) - [x] OneDrive / Sharepoint ([グローバル](https://www.office.com/), [cn](https://portal.partner.microsoftonline.cn),de,us) - [x] [189cloud](https://cloud.189.cn) (Personal, Family) - [x] [GoogleDrive](https://drive.google.com/) @@ -66,7 +66,7 @@ - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com/) - - [x] [Aliyundrive share](https://www.aliyundrive.com/) + - [x] [Aliyundrive share](https://www.alipan.com/) - [x] [Google photo](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) - [x] [Baidu photo](https://photo.baidu.com/) diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index 83c3f522452..eab38f58e1c 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -52,7 +52,7 @@ func (d *AliDrive) Init(ctx context.Context) error { return err } // get driver id - res, err, _ := d.request("https://api.aliyundrive.com/v2/user/get", http.MethodPost, nil, nil) + res, err, _ := d.request("https://api.alipan.com/v2/user/get", http.MethodPost, nil, nil) if err != nil { return err } @@ -106,7 +106,7 @@ func (d *AliDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs "file_id": file.GetID(), "expire_sec": 14400, } - res, err, _ := d.request("https://api.aliyundrive.com/v2/file/get_download_url", http.MethodPost, func(req *resty.Request) { + res, err, _ := d.request("https://api.alipan.com/v2/file/get_download_url", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) if err != nil { @@ -114,14 +114,14 @@ func (d *AliDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs } return &model.Link{ Header: http.Header{ - "Referer": []string{"https://www.aliyundrive.com/"}, + "Referer": []string{"https://www.alipan.com/"}, }, URL: utils.Json.Get(res, "url").ToString(), }, nil } func (d *AliDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err, _ := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "check_name_mode": "refuse", "drive_id": d.DriveId, @@ -139,7 +139,7 @@ func (d *AliDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *AliDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err, _ := d.request("https://api.aliyundrive.com/v3/file/update", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/v3/file/update", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "check_name_mode": "refuse", "drive_id": d.DriveId, @@ -156,7 +156,7 @@ func (d *AliDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *AliDrive) Remove(ctx context.Context, obj model.Obj) error { - _, err, _ := d.request("https://api.aliyundrive.com/v2/recyclebin/trash", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/v2/recyclebin/trash", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": obj.GetID(), @@ -216,7 +216,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil } var resp UploadResp - _, err, e := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + _, err, e := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(reqBody) }, &resp) @@ -270,7 +270,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil n, _ := io.NewSectionReader(localFile, o.Int64(), 8).Read(buf[:8]) reqBody["proof_code"] = base64.StdEncoding.EncodeToString(buf[:n]) - _, err, e := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + _, err, e := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(reqBody) }, &resp) if err != nil && e.Code != "PreHashMatched" { @@ -309,7 +309,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil } } var resp2 base.Json - _, err, e = d.request("https://api.aliyundrive.com/v2/file/complete", http.MethodPost, func(req *resty.Request) { + _, err, e = d.request("https://api.alipan.com/v2/file/complete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": resp.FileId, @@ -334,10 +334,10 @@ func (d *AliDrive) Other(ctx context.Context, args model.OtherArgs) (interface{} } switch args.Method { case "doc_preview": - url = "https://api.aliyundrive.com/v2/file/get_office_preview_url" + url = "https://api.alipan.com/v2/file/get_office_preview_url" data["access_token"] = d.AccessToken case "video_preview": - url = "https://api.aliyundrive.com/v2/file/get_video_preview_play_info" + url = "https://api.alipan.com/v2/file/get_video_preview_play_info" data["category"] = "live_transcoding" data["url_expire_sec"] = 14400 default: diff --git a/drivers/aliyundrive/util.go b/drivers/aliyundrive/util.go index b36900fb964..0e81b082bb9 100644 --- a/drivers/aliyundrive/util.go +++ b/drivers/aliyundrive/util.go @@ -26,7 +26,7 @@ func (d *AliDrive) createSession() error { state.retry = 0 return fmt.Errorf("createSession failed after three retries") } - _, err, _ := d.request("https://api.aliyundrive.com/users/v1/users/device/create_session", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/users/v1/users/device/create_session", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "deviceName": "samsung", "modelName": "SM-G9810", @@ -42,7 +42,7 @@ func (d *AliDrive) createSession() error { } // func (d *AliDrive) renewSession() error { -// _, err, _ := d.request("https://api.aliyundrive.com/users/v1/users/device/renew_session", http.MethodPost, nil, nil) +// _, err, _ := d.request("https://api.alipan.com/users/v1/users/device/renew_session", http.MethodPost, nil, nil) // return err // } @@ -58,7 +58,7 @@ func (d *AliDrive) sign() { // do others that not defined in Driver interface func (d *AliDrive) refreshToken() error { - url := "https://auth.aliyundrive.com/v2/account/token" + url := "https://auth.alipan.com/v2/account/token" var resp base.TokenResp var e RespErr _, err := base.RestyClient.R(). @@ -85,7 +85,7 @@ func (d *AliDrive) request(url, method string, callback base.ReqCallback, resp i req := base.RestyClient.R() state, ok := global.Load(d.UserID) if !ok { - if url == "https://api.aliyundrive.com/v2/user/get" { + if url == "https://api.alipan.com/v2/user/get" { state = &State{} } else { return nil, fmt.Errorf("can't load user state, user_id: %s", d.UserID), RespErr{} @@ -94,8 +94,8 @@ func (d *AliDrive) request(url, method string, callback base.ReqCallback, resp i req.SetHeaders(map[string]string{ "Authorization": "Bearer\t" + d.AccessToken, "content-type": "application/json", - "origin": "https://www.aliyundrive.com", - "Referer": "https://aliyundrive.com/", + "origin": "https://www.alipan.com", + "Referer": "https://alipan.com/", "X-Signature": state.signature, "x-request-id": uuid.NewString(), "X-Canary": "client=Android,app=adrive,version=v4.1.0", @@ -158,7 +158,7 @@ func (d *AliDrive) getFiles(fileId string) ([]File, error) { "video_thumbnail_process": "video/snapshot,t_0,f_jpg,ar_auto,w_300", "url_expire_sec": 14400, } - _, err, _ := d.request("https://api.aliyundrive.com/v2/file/list", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/v2/file/list", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) @@ -172,7 +172,7 @@ func (d *AliDrive) getFiles(fileId string) ([]File, error) { } func (d *AliDrive) batch(srcId, dstId string, url string) error { - res, err, _ := d.request("https://api.aliyundrive.com/v3/batch", http.MethodPost, func(req *resty.Request) { + res, err, _ := d.request("https://api.alipan.com/v3/batch", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "requests": []base.Json{ { diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index bd69211c77e..de9b45e01d6 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -36,7 +36,7 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { return &AliyundriveOpen{ - base: "https://openapi.aliyundrive.com", + base: "https://openapi.alipan.com", } }) } diff --git a/drivers/aliyundrive_share/driver.go b/drivers/aliyundrive_share/driver.go index 2e042ceef35..db2ff9ace16 100644 --- a/drivers/aliyundrive_share/driver.go +++ b/drivers/aliyundrive_share/driver.go @@ -105,7 +105,7 @@ func (d *AliyundriveShare) link(ctx context.Context, file model.Obj) (*model.Lin "share_id": d.ShareId, } var resp ShareLinkResp - _, err := d.request("https://api.aliyundrive.com/v2/file/get_share_link_download_url", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api.alipan.com/v2/file/get_share_link_download_url", http.MethodPost, func(req *resty.Request) { req.SetHeader(CanaryHeaderKey, CanaryHeaderValue).SetBody(data).SetResult(&resp) }) if err != nil { @@ -113,7 +113,7 @@ func (d *AliyundriveShare) link(ctx context.Context, file model.Obj) (*model.Lin } return &model.Link{ Header: http.Header{ - "Referer": []string{"https://www.aliyundrive.com/"}, + "Referer": []string{"https://www.alipan.com/"}, }, URL: resp.DownloadUrl, }, nil @@ -128,9 +128,9 @@ func (d *AliyundriveShare) Other(ctx context.Context, args model.OtherArgs) (int } switch args.Method { case "doc_preview": - url = "https://api.aliyundrive.com/v2/file/get_office_preview_url" + url = "https://api.alipan.com/v2/file/get_office_preview_url" case "video_preview": - url = "https://api.aliyundrive.com/v2/file/get_video_preview_play_info" + url = "https://api.alipan.com/v2/file/get_video_preview_play_info" data["category"] = "live_transcoding" default: return nil, errs.NotSupport diff --git a/drivers/aliyundrive_share/util.go b/drivers/aliyundrive_share/util.go index a29f86f5522..899e15cec1b 100644 --- a/drivers/aliyundrive_share/util.go +++ b/drivers/aliyundrive_share/util.go @@ -16,7 +16,7 @@ const ( ) func (d *AliyundriveShare) refreshToken() error { - url := "https://auth.aliyundrive.com/v2/account/token" + url := "https://auth.alipan.com/v2/account/token" var resp base.TokenResp var e ErrorResp _, err := base.RestyClient.R(). @@ -47,7 +47,7 @@ func (d *AliyundriveShare) getShareToken() error { var resp ShareTokenResp _, err := base.RestyClient.R(). SetResult(&resp).SetError(&e).SetBody(data). - Post("https://api.aliyundrive.com/v2/share_link/get_share_token") + Post("https://api.alipan.com/v2/share_link/get_share_token") if err != nil { return err } @@ -116,7 +116,7 @@ func (d *AliyundriveShare) getFiles(fileId string) ([]File, error) { SetHeader("x-share-token", d.ShareToken). SetHeader(CanaryHeaderKey, CanaryHeaderValue). SetResult(&resp).SetError(&e).SetBody(data). - Post("https://api.aliyundrive.com/adrive/v3/file/list") + Post("https://api.alipan.com/adrive/v3/file/list") if err != nil { return nil, err } From 478470f609b0435ab5179d7a9e0e73719d9c9c09 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 31 Dec 2023 15:03:25 +0800 Subject: [PATCH 063/659] feat!: replace regex package (close #5755) --- go.mod | 1 + go.sum | 2 ++ internal/model/obj.go | 10 +++++----- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 30ff10b3fbb..e6e8bc60130 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/deckarep/golang-set/v2 v2.3.1 github.com/disintegration/imaging v1.6.2 github.com/djherbis/times v1.5.0 + github.com/dlclark/regexp2 v1.10.0 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.4 github.com/foxxorcat/weiyun-sdk-go v0.1.3 diff --git a/go.sum b/go.sum index 7d7048f8377..a763a1ace13 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU= github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0= +github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= +github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/foxxorcat/mopan-sdk-go v0.1.4 h1:6utvPiBv8KDRDVKB7A4FERdrVxcHKZd2fBFCNuKcXzU= diff --git a/internal/model/obj.go b/internal/model/obj.go index 77c0700a35c..dbcdcaf12a3 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -2,13 +2,13 @@ package model import ( "io" - "regexp" "sort" "strings" "time" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/dlclark/regexp2" mapset "github.com/deckarep/golang-set/v2" @@ -169,7 +169,7 @@ func NewObjMerge() *ObjMerge { } type ObjMerge struct { - regs []*regexp.Regexp + regs []*regexp2.Regexp set mapset.Set[string] } @@ -190,7 +190,7 @@ func (om *ObjMerge) insertObjs(objs []Obj, objs_ ...Obj) []Obj { func (om *ObjMerge) clickObj(obj Obj) bool { for _, reg := range om.regs { - if reg.MatchString(obj.GetName()) { + if isMatch, _ := reg.MatchString(obj.GetName()); isMatch { return false } } @@ -199,9 +199,9 @@ func (om *ObjMerge) clickObj(obj Obj) bool { func (om *ObjMerge) InitHideReg(hides string) { rs := strings.Split(hides, "\n") - om.regs = make([]*regexp.Regexp, 0, len(rs)) + om.regs = make([]*regexp2.Regexp, 0, len(rs)) for _, r := range rs { - om.regs = append(om.regs, regexp.MustCompile(r)) + om.regs = append(om.regs, regexp2.MustCompile(r, regexp2.None)) } } From 57bac9e0d2a674a78f45234460ecd4fcbcbd9f78 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 1 Jan 2024 17:16:07 +0800 Subject: [PATCH 064/659] fix: some missing regexp lib modified --- server/common/check.go | 6 +++--- server/handles/meta.go | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/common/check.go b/server/common/check.go index 1f4227b000d..78051f4ee1e 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -2,7 +2,6 @@ package common import ( "path" - "regexp" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -10,6 +9,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/dlclark/regexp2" ) func IsStorageSignEnabled(rawPath string) bool { @@ -36,8 +36,8 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri if meta != nil && !user.CanSeeHides() && meta.Hide != "" && IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path for _, hide := range strings.Split(meta.Hide, "\n") { - re := regexp.MustCompile(hide) - if re.MatchString(path.Base(reqPath)) { + re := regexp2.MustCompile(hide, regexp2.None) + if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { return false } } diff --git a/server/handles/meta.go b/server/handles/meta.go index e7454d9df5a..00aa3137192 100644 --- a/server/handles/meta.go +++ b/server/handles/meta.go @@ -2,13 +2,13 @@ package handles import ( "fmt" - "regexp" "strconv" "strings" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" + "github.com/dlclark/regexp2" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) @@ -71,7 +71,7 @@ func UpdateMeta(c *gin.Context) { func validHide(hide string) (string, error) { rs := strings.Split(hide, "\n") for _, r := range rs { - _, err := regexp.Compile(r) + _, err := regexp2.Compile(r, regexp2.None) if err != nil { return r, err } From 182aacd309475ce35f3e3e335da59808e575f105 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 19:55:31 +0800 Subject: [PATCH 065/659] fix(deps): update module github.com/gorilla/websocket to v1.5.1 [skip ci] (#5770) * fix: missing modified in validate regexp * fix(deps): update module github.com/gorilla/websocket to v1.5.1 --------- Co-authored-by: Andy Hsu Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e6e8bc60130..c2693ab4d8e 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/go-webauthn/webauthn v0.9.4 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.4.0 - github.com/gorilla/websocket v1.5.0 + github.com/gorilla/websocket v1.5.1 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 @@ -56,6 +56,7 @@ require ( golang.org/x/oauth2 v0.12.0 golang.org/x/time v0.3.0 google.golang.org/appengine v1.6.7 + gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 gorm.io/driver/sqlite v1.4.4 @@ -200,7 +201,6 @@ require ( google.golang.org/grpc v1.57.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect - gopkg.in/ldap.v3 v3.1.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index a763a1ace13..6a9b8128b02 100644 --- a/go.sum +++ b/go.sum @@ -207,6 +207,8 @@ github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= +github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= From 977b3cf9ab8bf4d1d7c43bdf212ea5238b34dea7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 19:57:04 +0800 Subject: [PATCH 066/659] fix(deps): update golang.org/x/exp digest to 02704c9 [skip ci] (#5769) * fix: missing modified in validate regexp * fix(deps): update golang.org/x/exp digest to 02704c9 --------- Co-authored-by: Andy Hsu Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c2693ab4d8e..5c032a0ab11 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 golang.org/x/crypto v0.16.0 - golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/exp v0.0.0-20231226003508-02704c960a9b golang.org/x/image v0.11.0 golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.12.0 diff --git a/go.sum b/go.sum index 6a9b8128b02..4a678ac6403 100644 --- a/go.sum +++ b/go.sum @@ -481,6 +481,8 @@ golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= +golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= From be5d94cd11e860b56217ba240637a84028b6d60c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 21:42:03 +0800 Subject: [PATCH 067/659] fix(deps): update module golang.org/x/crypto to v0.17.0 [security] (#5768) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5c032a0ab11..4356acd8419 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 - golang.org/x/crypto v0.16.0 + golang.org/x/crypto v0.17.0 golang.org/x/exp v0.0.0-20231226003508-02704c960a9b golang.org/x/image v0.11.0 golang.org/x/net v0.19.0 diff --git a/go.sum b/go.sum index 4a678ac6403..923b8b029b7 100644 --- a/go.sum +++ b/go.sum @@ -479,6 +479,8 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= From a006f57637ee0375793a53986efe90b39cabb6d4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 21:42:43 +0800 Subject: [PATCH 068/659] fix(deps): update module google.golang.org/appengine to v1.6.8 [skip ci] (#5772) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4356acd8419..7632f451c7d 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.12.0 golang.org/x/time v0.3.0 - google.golang.org/appengine v1.6.7 + google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 diff --git a/go.sum b/go.sum index 923b8b029b7..f65e56924a6 100644 --- a/go.sum +++ b/go.sum @@ -185,6 +185,7 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= @@ -551,6 +552,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -573,6 +575,8 @@ google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= From 2c8d003c2e1d7bcb62a7bbdb25209c4fd7701e6c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 22:17:44 +0800 Subject: [PATCH 069/659] fix(deps): update module github.com/djherbis/times to v1.6.0 [skip ci] (#5422) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7632f451c7d..4a8d0889167 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.3.1 github.com/disintegration/imaging v1.6.2 - github.com/djherbis/times v1.5.0 + github.com/djherbis/times v1.6.0 github.com/dlclark/regexp2 v1.10.0 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.4 diff --git a/go.sum b/go.sum index f65e56924a6..946b47e840f 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU= github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= @@ -526,6 +528,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From bdf7abe717e1f402e92e7cb93135da53f0d0388a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 22:18:13 +0800 Subject: [PATCH 070/659] fix(deps): update module github.com/aws/aws-sdk-go to v1.49.13 [skip ci] (#5774) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4a8d0889167..a8316788059 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Xhofe/wopan-sdk-go v0.1.2 github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.46.7 + github.com/aws/aws-sdk-go v1.49.13 github.com/blevesearch/bleve/v2 v2.3.10 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.16.1 diff --git a/go.sum b/go.sum index 946b47e840f..701614b3ad0 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.46.7 h1:IjvAWeiJZlbETOemOwvheN5L17CvKvKW0T1xOC6d3Sc= github.com/aws/aws-sdk-go v1.46.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y= +github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= From e2434029f98ae4cf979ab96cacf7b0f009b07b8d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 22:18:51 +0800 Subject: [PATCH 071/659] fix(deps): update module github.com/maruel/natural to v1.1.1 [skip ci] (#5771) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a8316788059..2b69f8b5673 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 - github.com/maruel/natural v1.1.0 + github.com/maruel/natural v1.1.1 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 701614b3ad0..06389ace465 100644 --- a/go.sum +++ b/go.sum @@ -288,6 +288,8 @@ github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0g github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= From 5afd65b65c1f220dae0c718a9b7f127934ad592d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 22:19:25 +0800 Subject: [PATCH 072/659] fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v2.2.10+incompatible (#5447) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2b69f8b5673..7b3d1c44957 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/Xhofe/wopan-sdk-go v0.1.2 - github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible + github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.49.13 github.com/blevesearch/bleve/v2 v2.3.10 diff --git a/go.sum b/go.sum index 06389ace465..b4234fbc04d 100644 --- a/go.sum +++ b/go.sum @@ -25,6 +25,8 @@ github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible h1:ROMcuN61gI8SfQ+AEMh4d7GZ3gwTZLIhPjtd05TQCG4= +github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/Hcta4l4VrIXrI= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= From 2c15349ce4a68f0f0f1b04e84b39c092f4f23d59 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:12:13 +0800 Subject: [PATCH 073/659] fix(deps): update module github.com/gin-contrib/cors to v1.5.0 [skip ci] (#5779) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 13 +++++++------ go.sum | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 7b3d1c44957..beb448a3b3c 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.4 github.com/foxxorcat/weiyun-sdk-go v0.1.3 - github.com/gin-contrib/cors v1.4.0 + github.com/gin-contrib/cors v1.5.0 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.10.0 github.com/go-webauthn/webauthn v0.9.4 @@ -93,9 +93,10 @@ require ( github.com/blevesearch/zapx/v15 v15.3.13 // indirect github.com/bluele/gcache v0.0.2 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect - github.com/bytedance/sonic v1.9.1 // indirect + github.com/bytedance/sonic v1.10.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect + github.com/chenzhuoyu/iasm v0.9.0 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect @@ -110,7 +111,7 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/go-playground/validator/v10 v10.15.5 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/go-webauthn/x v0.1.5 // indirect github.com/goccy/go-json v0.10.2 // indirect @@ -166,7 +167,7 @@ require ( github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/ncw/swift/v2 v2.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect @@ -191,7 +192,7 @@ require ( github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/arch v0.3.0 // indirect + golang.org/x/arch v0.5.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect diff --git a/go.sum b/go.sum index b4234fbc04d..3dfabc02e75 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,9 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBW github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= +github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc= +github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -100,6 +103,10 @@ github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgk github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= +github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= +github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= +github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= @@ -144,6 +151,8 @@ github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9 github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= +github.com/gin-contrib/cors v1.5.0 h1:DgGKV7DDoOn36DFkNtbHrjoRiT5ExCe+PC9/xp7aKvk= +github.com/gin-contrib/cors v1.5.0/go.mod h1:TvU7MAZ3EwrPLI2ztzTt3tqgvBCq+wn8WpZmfADjupI= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= @@ -167,6 +176,8 @@ github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXS github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24= +github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= @@ -263,6 +274,7 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -358,6 +370,8 @@ github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OI github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -475,6 +489,8 @@ gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= +golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -630,5 +646,6 @@ gorm.io/gorm v1.24.5 h1:g6OPREKqqlWq4kh/3MCQbZKImeB9e6Xgc4zD+JgNZGE= gorm.io/gorm v1.24.5/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= From 8531b23382850ddf6804790f1893713bab5070a4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:12:33 +0800 Subject: [PATCH 074/659] fix(deps): update module github.com/deckarep/golang-set/v2 to v2.6.0 [skip ci] (#5778) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index beb448a3b3c..36eea8cce09 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/charmbracelet/bubbletea v0.24.2 github.com/charmbracelet/lipgloss v0.9.1 github.com/coreos/go-oidc v2.2.1+incompatible - github.com/deckarep/golang-set/v2 v2.3.1 + github.com/deckarep/golang-set/v2 v2.6.0 github.com/disintegration/imaging v1.6.2 github.com/djherbis/times v1.6.0 github.com/dlclark/regexp2 v1.10.0 diff --git a/go.sum b/go.sum index 3dfabc02e75..cef2b6b5274 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= +github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= From a4a967561613c9707198729336e1cee8298e133f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:12:54 +0800 Subject: [PATCH 075/659] fix(deps): update module github.com/charmbracelet/bubbletea to v0.25.0 [skip ci] (#5776) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 36eea8cce09..41f8c0fc461 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/blevesearch/bleve/v2 v2.3.10 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.16.1 - github.com/charmbracelet/bubbletea v0.24.2 + github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/lipgloss v0.9.1 github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.6.0 diff --git a/go.sum b/go.sum index cef2b6b5274..b466e55c0db 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,8 @@ github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5 github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= +github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= +github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= From 45b1ff4a24577526d9c838417ce87c9f4c164999 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:24:56 +0800 Subject: [PATCH 076/659] fix(deps): update module github.com/charmbracelet/bubbles to v0.17.1 [skip ci] (#5775) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 41f8c0fc461..b87965fc5ae 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/aws/aws-sdk-go v1.49.13 github.com/blevesearch/bleve/v2 v2.3.10 github.com/caarlos0/env/v9 v9.0.0 - github.com/charmbracelet/bubbles v0.16.1 + github.com/charmbracelet/bubbles v0.17.1 github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/lipgloss v0.9.1 github.com/coreos/go-oidc v2.2.1+incompatible diff --git a/go.sum b/go.sum index b466e55c0db..e7eac9a80cc 100644 --- a/go.sum +++ b/go.sum @@ -94,6 +94,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= +github.com/charmbracelet/bubbles v0.17.1 h1:0SIyjOnkrsfDo88YvPgAWvZMwXe26TP6drRvmkjyUu4= +github.com/charmbracelet/bubbles v0.17.1/go.mod h1:9HxZWlkCqz2PRwsCbYl7a3KXvGzFaDHpYbSYMJ+nE3o= github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= From c36644a17271317ecd8965797f37fad3b5b36761 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:27:08 +0800 Subject: [PATCH 077/659] fix(deps): update module github.com/go-resty/resty/v2 to v2.11.0 (#5781) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b87965fc5ae..e311433fb30 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gin-contrib/cors v1.5.0 github.com/gin-gonic/gin v1.9.1 - github.com/go-resty/resty/v2 v2.10.0 + github.com/go-resty/resty/v2 v2.11.0 github.com/go-webauthn/webauthn v0.9.4 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.4.0 diff --git a/go.sum b/go.sum index e7eac9a80cc..affcbe33f1f 100644 --- a/go.sum +++ b/go.sum @@ -187,6 +187,8 @@ github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QX github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= +github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= +github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.9.4 h1:YxvHSqgUyc5AK2pZbqkWWR55qKeDPhP8zLDr6lpIc2g= From 8a427ddc493c2db8540af3ef574c7885ace2f67a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:54:38 +0800 Subject: [PATCH 078/659] fix(deps): update module github.com/go-webauthn/webauthn to v0.10.0 [skip ci] (#5782) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e311433fb30..ab12966c22d 100644 --- a/go.mod +++ b/go.mod @@ -26,9 +26,9 @@ require ( github.com/gin-contrib/cors v1.5.0 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.11.0 - github.com/go-webauthn/webauthn v0.9.4 + github.com/go-webauthn/webauthn v0.10.0 github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/google/uuid v1.4.0 + github.com/google/uuid v1.5.0 github.com/gorilla/websocket v1.5.1 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/go-ipfs-api v0.7.0 @@ -113,7 +113,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.15.5 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/go-webauthn/x v0.1.5 // indirect + github.com/go-webauthn/x v0.1.6 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect diff --git a/go.sum b/go.sum index affcbe33f1f..58c33493e6d 100644 --- a/go.sum +++ b/go.sum @@ -193,8 +193,12 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.9.4 h1:YxvHSqgUyc5AK2pZbqkWWR55qKeDPhP8zLDr6lpIc2g= github.com/go-webauthn/webauthn v0.9.4/go.mod h1:LqupCtzSef38FcxzaklmOn7AykGKhAhr9xlRbdbgnTw= +github.com/go-webauthn/webauthn v0.10.0 h1:yuW2e1tXnRAwAvKrR4q4LQmc6XtCMH639/ypZGhZCwk= +github.com/go-webauthn/webauthn v0.10.0/go.mod h1:l0NiauXhL6usIKqNLCUM3Qir43GK7ORg8ggold0Uv/Y= github.com/go-webauthn/x v0.1.5 h1:V2TCzDU2TGLd0kSZOXdrqDVV5JB9ILnKxA9S53CSBw0= github.com/go-webauthn/x v0.1.5/go.mod h1:qbzWwcFcv4rTwtCLOZd+icnr6B7oSsAGZJqlt8cukqY= +github.com/go-webauthn/x v0.1.6 h1:QNAX+AWeqRt9loE8mULeWJCqhVG5D/jvdmJ47fIWCkQ= +github.com/go-webauthn/x v0.1.6/go.mod h1:W8dFVZ79o4f+nY1eOUICy/uq5dhrRl7mxQkYhXTo0FA= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -227,6 +231,8 @@ github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkj github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= From b5cc90cb5a667e2d6b37bf4ed070941f84c3dd5a Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:41:32 +0800 Subject: [PATCH 079/659] fix(115): support null `UserAgent` (#5787) --- drivers/115/driver.go | 6 +--- drivers/115/util.go | 70 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 96bf9d45837..57b6c45f9e7 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -64,11 +64,7 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, err } var userAgent = args.Header.Get("User-Agent") - if userAgent == "" { - userAgent = driver115.UA115Browser - } - - downloadInfo, err := d.client. + downloadInfo, err := d. DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err diff --git a/drivers/115/util.go b/drivers/115/util.go index 8e638d79a6a..fb425c8eef7 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -5,12 +5,8 @@ import ( "crypto/tls" "encoding/json" "fmt" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - "github.com/orzogc/fake115uploader/cipher" "io" + "net/http" "net/url" "path/filepath" "strconv" @@ -18,8 +14,15 @@ import ( "sync" "time" - driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + + driver115 "github.com/SheltonZhu/115driver/pkg/driver" + crypto "github.com/gaoyb7/115drive-webdav/115" + "github.com/orzogc/fake115uploader/cipher" "github.com/pkg/errors" ) @@ -74,6 +77,61 @@ const ( appVer = "2.0.3.6" ) +func (c *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { + key := crypto.GenerateKey() + result := driver115.DownloadResp{} + params, err := utils.Json.Marshal(map[string]string{"pickcode": pickCode}) + if err != nil { + return nil, err + } + + data := crypto.Encode(params, key) + + bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) + reqUrl := fmt.Sprintf("%s?t=%s", driver115.ApiDownloadGetUrl, driver115.Now().String()) + req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.Header.Set("Cookie", c.Cookie) + req.Header.Set("User-Agent", ua) + + resp, err := c.client.Client.GetClient().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if err := utils.Json.Unmarshal(body, &result); err != nil { + return nil, err + } + + if err = result.Err(string(body)); err != nil { + return nil, err + } + + bytes, err := crypto.Decode(string(result.EncodedData), key) + if err != nil { + return nil, err + } + + downloadInfo := driver115.DownloadData{} + if err := utils.Json.Unmarshal(bytes, &downloadInfo); err != nil { + return nil, err + } + + for _, info := range downloadInfo { + if info.FileSize < 0 { + return nil, driver115.ErrDownloadEmpty + } + info.Header = resp.Request.Header + return info, nil + } + return nil, driver115.ErrUnexpected +} + func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID string, stream model.FileStreamer) (*driver115.UploadInitResp, error) { var ( ecdhCipher *cipher.EcdhCipher From 7db27e6da8790616145cf1d34c2db1605568f9da Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:41:52 +0800 Subject: [PATCH 080/659] fix(deps): update module golang.org/x/time to v0.5.0 [skip ci] (#5786) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ab12966c22d..8051c899df2 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( golang.org/x/image v0.11.0 golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.12.0 - golang.org/x/time v0.3.0 + golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.4.7 diff --git a/go.sum b/go.sum index 58c33493e6d..217f566b215 100644 --- a/go.sum +++ b/go.sum @@ -603,6 +603,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 207c7e05feadc1fa8aa239b9f5054e5f4f8918c3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:42:17 +0800 Subject: [PATCH 081/659] fix(deps): update module github.com/spf13/cobra to v1.8.0 [skip ci] (#5783) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8051c899df2..f9e9beb08fc 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/pquerna/otp v1.4.0 github.com/rclone/rclone v1.63.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca github.com/u2takey/ffmpeg-go v0.5.0 diff --git a/go.sum b/go.sum index 217f566b215..18f156ba2a4 100644 --- a/go.sum +++ b/go.sum @@ -118,6 +118,7 @@ github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHo github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -444,6 +445,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From b97c9173af4dfb5a723aea6a7711f80eb43c4c94 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:42:38 +0800 Subject: [PATCH 082/659] fix(deps): update module golang.org/x/image to v0.14.0 [skip ci] (#5784) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index f9e9beb08fc..b70d4d10e8c 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/xhofe/tache v0.1.1 golang.org/x/crypto v0.17.0 golang.org/x/exp v0.0.0-20231226003508-02704c960a9b - golang.org/x/image v0.11.0 + golang.org/x/image v0.14.0 golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.12.0 golang.org/x/time v0.5.0 diff --git a/go.sum b/go.sum index 18f156ba2a4..d78156a4f53 100644 --- a/go.sum +++ b/go.sum @@ -530,6 +530,8 @@ golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGb golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= +golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= +golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= From 88831b5d5ace965eeb50e5c3f2284b08607b8d3d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 15:44:45 +0800 Subject: [PATCH 083/659] fix(deps): update module golang.org/x/oauth2 to v0.15.0 [skip ci] (#5785) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index b70d4d10e8c..78489c12ff0 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.4 github.com/foxxorcat/weiyun-sdk-go v0.1.3 + github.com/gaoyb7/115drive-webdav v0.1.8 github.com/gin-contrib/cors v1.5.0 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.11.0 @@ -53,7 +54,7 @@ require ( golang.org/x/exp v0.0.0-20231226003508-02704c960a9b golang.org/x/image v0.14.0 golang.org/x/net v0.19.0 - golang.org/x/oauth2 v0.12.0 + golang.org/x/oauth2 v0.15.0 golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 @@ -104,7 +105,6 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gaoyb7/115drive-webdav v0.1.8 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-chi/chi/v5 v5.0.10 // indirect diff --git a/go.sum b/go.sum index d78156a4f53..41ebd49835e 100644 --- a/go.sum +++ b/go.sum @@ -550,6 +550,8 @@ golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From ff25e51f808ec069a4b46d0c97a1e77335563da4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 17:43:08 +0800 Subject: [PATCH 084/659] chore(deps): update actions/setup-go action to v5 (#5789) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/auto_lang.yml | 2 +- .github/workflows/build.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/release_linux_musl.yml | 2 +- .github/workflows/release_linux_musl_arm.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/auto_lang.yml b/.github/workflows/auto_lang.yml index d1221f763f8..b93f2ad6d90 100644 --- a/.github/workflows/auto_lang.yml +++ b/.github/workflows/auto_lang.yml @@ -25,7 +25,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - name: Setup go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eeff969f581..03c48bc8a62 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 50a4719993f..1422dfaa5f9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,7 +21,7 @@ jobs: prerelease: true - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} diff --git a/.github/workflows/release_linux_musl.yml b/.github/workflows/release_linux_musl.yml index 0288427524b..4ef04f454f9 100644 --- a/.github/workflows/release_linux_musl.yml +++ b/.github/workflows/release_linux_musl.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} diff --git a/.github/workflows/release_linux_musl_arm.yml b/.github/workflows/release_linux_musl_arm.yml index abd96987c7f..db650c89e3c 100644 --- a/.github/workflows/release_linux_musl_arm.yml +++ b/.github/workflows/release_linux_musl_arm.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} From 8c432d3339c5445ae6c16f56522883d7663586fd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 17:43:22 +0800 Subject: [PATCH 085/659] chore(deps): update actions/checkout action to v4 (#5788) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/auto_lang.yml | 4 ++-- .github/workflows/build.yml | 2 +- .github/workflows/build_docker.yml | 4 ++-- .github/workflows/changelog.yml | 2 +- .github/workflows/release.yml | 4 ++-- .github/workflows/release_docker.yml | 4 ++-- .github/workflows/release_linux_musl.yml | 2 +- .github/workflows/release_linux_musl_arm.yml | 2 +- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/auto_lang.yml b/.github/workflows/auto_lang.yml index b93f2ad6d90..9108dd1bf2b 100644 --- a/.github/workflows/auto_lang.yml +++ b/.github/workflows/auto_lang.yml @@ -30,12 +30,12 @@ jobs: go-version: ${{ matrix.go-version }} - name: Checkout alist - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: alist - name: Checkout alist-web - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'alist-org/alist-web' ref: main diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 03c48bc8a62..bec800a8ddd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,7 +25,7 @@ jobs: go-version: ${{ matrix.go-version }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: benjlevesque/short-sha@v2.2 id: short-sha diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 48311e8f72c..cd9291fbd71 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Docker meta id: meta uses: docker/metadata-action@v4 @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: alist-org/with_aria2 ref: main diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index b0cfeaa8a63..8c1bfb67d03 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - run: npx changelogithub # or changelogithub@0.12 if ensure the stable result diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1422dfaa5f9..d43d39d0c3b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,7 @@ jobs: go-version: ${{ matrix.go-version }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -53,7 +53,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: alist-org/desktop-release ref: main diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index a16cf49d2d5..9a2c39f21b8 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Docker meta id: meta @@ -47,7 +47,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: alist-org/with_aria2 ref: main diff --git a/.github/workflows/release_linux_musl.yml b/.github/workflows/release_linux_musl.yml index 4ef04f454f9..9ec79af6fdf 100644 --- a/.github/workflows/release_linux_musl.yml +++ b/.github/workflows/release_linux_musl.yml @@ -20,7 +20,7 @@ jobs: go-version: ${{ matrix.go-version }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/release_linux_musl_arm.yml b/.github/workflows/release_linux_musl_arm.yml index db650c89e3c..8ddbc4f42cc 100644 --- a/.github/workflows/release_linux_musl_arm.yml +++ b/.github/workflows/release_linux_musl_arm.yml @@ -20,7 +20,7 @@ jobs: go-version: ${{ matrix.go-version }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 From 97a4b8321d89c8d1f0be05bc71412522dd381985 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:44:59 +0800 Subject: [PATCH 086/659] chore(deps): update actions/upload-artifact action to v4 (#5792) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bec800a8ddd..b75010ef8e7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -42,7 +42,7 @@ jobs: bash build.sh dev - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: alist_${{ env.SHA }} path: dist \ No newline at end of file From 6f742a68cfce61636a431b8e654a0a9a275d102d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:45:32 +0800 Subject: [PATCH 087/659] chore(deps): update docker/build-push-action action to v5 [skip ci] (#5793) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build_docker.yml | 2 +- .github/workflows/release_docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index cd9291fbd71..201998cba8e 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -34,7 +34,7 @@ jobs: password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push id: docker_build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: true diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 9a2c39f21b8..99f349aa5a6 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -33,7 +33,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: . push: true From be537aa49b8a2b9ffec7d066f95a2e22472fe6c7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:45:53 +0800 Subject: [PATCH 088/659] chore(deps): update docker/login-action action to v3 [skip ci] (#5794) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build_docker.yml | 2 +- .github/workflows/release_docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 201998cba8e..2152baf8408 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -28,7 +28,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: xhofe password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 99f349aa5a6..9ff4afba361 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -26,7 +26,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: xhofe password: ${{ secrets.DOCKERHUB_TOKEN }} From 2683621ed7076b7de2c3fd68ca7f053a189cf453 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:46:09 +0800 Subject: [PATCH 089/659] chore(deps): update docker/metadata-action action to v5 [skip ci] (#5795) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build_docker.yml | 2 +- .github/workflows/release_docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 2152baf8408..5c7c11282c0 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v4 - name: Docker meta id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: xhofe/alist - name: Replace release with dev diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 9ff4afba361..fcfd7720a78 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -15,7 +15,7 @@ jobs: - name: Docker meta id: meta - uses: docker/metadata-action@v4 + uses: docker/metadata-action@v5 with: images: xhofe/alist From 03dbdfc0dd4e124e953233dba6e7a394baebcf63 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:46:37 +0800 Subject: [PATCH 090/659] chore(deps): update docker/setup-buildx-action action to v3 (#5797) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build_docker.yml | 2 +- .github/workflows/release_docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 5c7c11282c0..8d8a836c78e 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -26,7 +26,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 with: diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index fcfd7720a78..a28b9f301f9 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -23,7 +23,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub uses: docker/login-action@v3 From 03b9b9a1191067e665ff69a37c080b0964ce6ec1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 14:51:54 +0800 Subject: [PATCH 091/659] chore(deps): update docker/setup-qemu-action action to v3 (#5798) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build_docker.yml | 2 +- .github/workflows/release_docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 8d8a836c78e..396981192ef 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -24,7 +24,7 @@ jobs: run: | sed -i 's/release/dev/g' Dockerfile - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to DockerHub diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index a28b9f301f9..51ef40cc6a7 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -20,7 +20,7 @@ jobs: images: xhofe/alist - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From fd96a7ccf47ec699df671cc6f865d7bdee3e6bee Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 4 Jan 2024 21:02:22 +0800 Subject: [PATCH 092/659] fix(deps): update golang.org/x/exp digest to be819d1 [skip ci] (#5807) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 78489c12ff0..86ed7b82791 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 golang.org/x/crypto v0.17.0 - golang.org/x/exp v0.0.0-20231226003508-02704c960a9b + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc golang.org/x/image v0.14.0 golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.15.0 diff --git a/go.sum b/go.sum index 41ebd49835e..855d1c6033a 100644 --- a/go.sum +++ b/go.sum @@ -527,6 +527,8 @@ golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= From 126cfe9f93d52feda66a25c7d9bd31597857cecb Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 4 Jan 2024 21:54:39 +0800 Subject: [PATCH 093/659] fix(vtencent): only show 50 files (close #5805) --- drivers/vtencent/util.go | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/drivers/vtencent/util.go b/drivers/vtencent/util.go index ad69793e694..bf260415e0c 100644 --- a/drivers/vtencent/util.go +++ b/drivers/vtencent/util.go @@ -100,27 +100,35 @@ func (d *Vtencent) LoadUser() (string, error) { } func (d *Vtencent) GetFiles(dirId string) ([]File, error) { - api := "https://api.vs.tencent.com/PaaS/Material/SearchResource" - form := fmt.Sprintf(`{ + var res []File + //offset := 0 + for { + api := "https://api.vs.tencent.com/PaaS/Material/SearchResource" + form := fmt.Sprintf(`{ "Text":"", "Text":"", - "Offset":0, - "Limit":20000, + "Offset":%d, + "Limit":50, "Sort":{"Field":"%s","Order":"%s"}, "CreateTimeRanges":[], "MaterialTypes":[], "ReviewStatuses":[], "Tags":[], "SearchScopes":[{"Owner":{"Type":"PERSON","Id":"%s"},"ClassId":%s,"SearchOneDepth":true}] - }`, d.Addition.OrderBy, d.Addition.OrderDirection, d.TfUid, dirId) - var resps RspFiles - _, err := d.request(api, http.MethodPost, func(req *resty.Request) { - req.SetBody(form).ForceContentType("application/json") - }, &resps) - if err != nil { - return []File{}, err + }`, len(res), d.Addition.OrderBy, d.Addition.OrderDirection, d.TfUid, dirId) + var resp RspFiles + _, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resp) + if err != nil { + return nil, err + } + res = append(res, resp.Data.ResourceInfoSet...) + if len(resp.Data.ResourceInfoSet) <= 0 || len(res) >= resp.Data.TotalCount { + break + } } - return resps.Data.ResourceInfoSet, nil + return res, nil } func (d *Vtencent) CreateUploadMaterial(classId int, fileName string, UploadSummaryKey string) (RspCreatrMaterial, error) { From 9d5fb7f595336abf567c9221f0c1e982eaf15f55 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 4 Jan 2024 22:03:15 +0800 Subject: [PATCH 094/659] feat: add `ILanzou` driver (#5810 close #5715) * wip: basic request and login * feat: impl list * feat: impl link * feat: impl mkdir, move, rename, delete * feat: impl upload * docs: add iLanzou to readme --- README.md | 1 + README_cn.md | 1 + README_ja.md | 1 + drivers/all.go | 1 + drivers/ilanzou/driver.go | 365 ++++++++++++++++++++++++++++++++++++++ drivers/ilanzou/meta.go | 35 ++++ drivers/ilanzou/types.go | 57 ++++++ drivers/ilanzou/util.go | 105 +++++++++++ 8 files changed, 566 insertions(+) create mode 100644 drivers/ilanzou/driver.go create mode 100644 drivers/ilanzou/meta.go create mode 100644 drivers/ilanzou/types.go create mode 100644 drivers/ilanzou/util.go diff --git a/README.md b/README.md index ef68e01656b..74cd291d7df 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com/) + - [x] [ILanzou](https://www.ilanzou.com/) - [x] [Aliyundrive share](https://www.alipan.com/) - [x] [Google photo](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) diff --git a/README_cn.md b/README_cn.md index a5dfab47f45..9d3603fa529 100644 --- a/README_cn.md +++ b/README_cn.md @@ -65,6 +65,7 @@ - [x] [夸克网盘](https://pan.quark.cn) - [x] [迅雷网盘](https://pan.xunlei.com) - [x] [蓝奏云](https://www.lanzou.com/) + - [x] [蓝奏云优享版](https://www.ilanzou.com/) - [x] [阿里云盘分享](https://www.alipan.com/) - [x] [谷歌相册](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) diff --git a/README_ja.md b/README_ja.md index 3bcdd8de3d5..a50425455b3 100644 --- a/README_ja.md +++ b/README_ja.md @@ -66,6 +66,7 @@ - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com/) + - [x] [ILanzou](https://www.ilanzou.com/) - [x] [Aliyundrive share](https://www.alipan.com/) - [x] [Google photo](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) diff --git a/drivers/all.go b/drivers/all.go index 40666028f11..599820c296c 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -25,6 +25,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" + _ "github.com/alist-org/alist/v3/drivers/ilanzou" _ "github.com/alist-org/alist/v3/drivers/ipfs_api" _ "github.com/alist-org/alist/v3/drivers/lanzou" _ "github.com/alist-org/alist/v3/drivers/local" diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go new file mode 100644 index 00000000000..85ba27e2e23 --- /dev/null +++ b/drivers/ilanzou/driver.go @@ -0,0 +1,365 @@ +package template + +import ( + "context" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/foxxorcat/mopan-sdk-go" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +type ILanZou struct { + model.Storage + Addition + + userID string + account string + upClient *resty.Client +} + +func (d *ILanZou) Config() driver.Config { + return config +} + +func (d *ILanZou) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ILanZou) Init(ctx context.Context) error { + d.upClient = base.NewRestyClient().SetTimeout(time.Minute * 10) + if d.UUID == "" { + res, err := d.unproved("/getUuid", http.MethodGet, nil) + if err != nil { + return err + } + d.UUID = utils.Json.Get(res, "uuid").ToString() + } + res, err := d.proved("/user/account/map", http.MethodGet, nil) + if err != nil { + return err + } + d.userID = utils.Json.Get(res, "map", "userId").ToString() + d.account = utils.Json.Get(res, "map", "account").ToString() + log.Debugf("[ilanzou] init response: %s", res) + return nil +} + +func (d *ILanZou) Drop(ctx context.Context) error { + return nil +} + +func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + offset := 1 + limit := 60 + var res []ListItem + for { + var resp ListResp + _, err := d.proved("/record/file/list", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "type": "0", + "folderId": dir.GetID(), + "offset": strconv.Itoa(offset), + "limit": strconv.Itoa(limit), + }).SetResult(&resp) + }) + if err != nil { + return nil, err + } + res = append(res, resp.List...) + if resp.TotalPage <= resp.Offset { + break + } + offset++ + } + return utils.SliceConvert(res, func(f ListItem) (model.Obj, error) { + updTime, err := time.ParseInLocation("2006-01-02 15:04:05", f.UpdTime, time.Local) + if err != nil { + return nil, err + } + obj := model.Object{ + ID: strconv.FormatInt(f.FileId, 10), + //Path: "", + Name: f.FileName, + Size: f.FileSize * 1024, + Modified: updTime, + Ctime: updTime, + IsFolder: false, + //HashInfo: utils.HashInfo{}, + } + if f.FileType == 2 { + obj.IsFolder = true + obj.Size = 0 + obj.ID = strconv.FormatInt(f.FolderId, 10) + obj.Name = f.FolderName + } + return &obj, nil + }) +} + +func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + u, err := url.Parse("https://api.ilanzou.com/unproved/file/redirect") + if err != nil { + return nil, err + } + query := u.Query() + query.Set("uuid", d.UUID) + query.Set("devType", "6") + query.Set("devCode", d.UUID) + query.Set("devModel", "chrome") + query.Set("devVersion", "120") + query.Set("appVersion", "") + ts, err := getTimestamp() + if err != nil { + return nil, err + } + query.Set("timestamp", ts) + //query.Set("appToken", d.Token) + query.Set("enable", "1") + downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), AesSecret) + if err != nil { + return nil, err + } + query.Set("downloadId", hex.EncodeToString(downloadId)) + auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), time.Now().UnixMilli())), AesSecret) + if err != nil { + return nil, err + } + query.Set("auth", hex.EncodeToString(auth)) + u.RawQuery = query.Encode() + link := model.Link{URL: u.String()} + return &link, nil +} + +func (d *ILanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + res, err := d.proved("/file/folder/save", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "folderDesc": "", + "folderId": parentDir.GetID(), + "folderName": dirName, + }) + }) + if err != nil { + return nil, err + } + return &model.Object{ + ID: utils.Json.Get(res, "list", "0", "id").ToString(), + //Path: "", + Name: dirName, + Size: 0, + Modified: time.Now(), + Ctime: time.Now(), + IsFolder: true, + //HashInfo: utils.HashInfo{}, + }, nil +} + +func (d *ILanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var fileIds, folderIds []string + if srcObj.IsDir() { + folderIds = []string{srcObj.GetID()} + } else { + fileIds = []string{srcObj.GetID()} + } + _, err := d.proved("/file/folder/move", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "folderIds": strings.Join(folderIds, ","), + "fileIds": strings.Join(fileIds, ","), + "targetId": dstDir.GetID(), + }) + }) + if err != nil { + return nil, err + } + return srcObj, nil +} + +func (d *ILanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var err error + if srcObj.IsDir() { + _, err = d.proved("/file/folder/edit", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "folderDesc": "", + "folderId": srcObj.GetID(), + "folderName": newName, + }) + }) + } else { + _, err = d.proved("/file/edit", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "fileDesc": "", + "fileId": srcObj.GetID(), + "fileName": newName, + }) + }) + } + if err != nil { + return nil, err + } + return &model.Object{ + ID: srcObj.GetID(), + //Path: "", + Name: newName, + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ILanZou) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *ILanZou) Remove(ctx context.Context, obj model.Obj) error { + var fileIds, folderIds []string + if obj.IsDir() { + folderIds = []string{obj.GetID()} + } else { + fileIds = []string{obj.GetID()} + } + _, err := d.proved("/file/delete", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "folderIds": strings.Join(folderIds, ","), + "fileIds": strings.Join(fileIds, ","), + "status": 0, + }) + }) + return err +} + +const DefaultPartSize = 1024 * 1024 * 8 + +func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + h := md5.New() + // need to calculate md5 of the full content + tempFile, err := stream.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer func() { + _ = tempFile.Close() + }() + if _, err = io.Copy(h, tempFile); err != nil { + return nil, err + } + _, err = tempFile.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + etag := hex.EncodeToString(h.Sum(nil)) + // get upToken + res, err := d.proved("/7n/getUpToken", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "fileId": "", + "fileName": stream.GetName(), + "fileSize": stream.GetSize() / 1024, + "folderId": dstDir.GetID(), + "md5": etag, + "type": 1, + }) + }) + if err != nil { + return nil, err + } + upToken := utils.Json.Get(res, "upToken").ToString() + now := time.Now() + key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) + var token string + if stream.GetSize() > DefaultPartSize { + res, err := d.upClient.R().SetMultipartFormData(map[string]string{ + "token": upToken, + "key": key, + "fname": stream.GetName(), + }).SetMultipartField("file", stream.GetName(), stream.GetMimetype(), tempFile). + Post("https://upload.qiniup.com/") + if err != nil { + return nil, err + } + token = utils.Json.Get(res.Body(), "token").ToString() + } else { + keyBase64 := base64.URLEncoding.EncodeToString([]byte(key)) + res, err := d.upClient.R().Post(fmt.Sprintf("https://upload.qiniup.com/buckets/wpanstore-lanzou/objects/%s/uploads", keyBase64)) + if err != nil { + return nil, err + } + uploadId := utils.Json.Get(res.Body(), "uploadId").ToString() + parts := make([]Part, 0) + partNum := (stream.GetSize() + DefaultPartSize - 1) / DefaultPartSize + for i := 1; i <= int(partNum); i++ { + u := fmt.Sprintf("https://upload.qiniup.com/buckets/wpanstore-lanzou/objects/%s/uploads/%s/%d", keyBase64, uploadId, i) + res, err = d.upClient.R().SetBody(io.LimitReader(tempFile, DefaultPartSize)).Put(u) + if err != nil { + return nil, err + } + etag := utils.Json.Get(res.Body(), "etag").ToString() + parts = append(parts, Part{ + PartNumber: i, + ETag: etag, + }) + } + res, err = d.upClient.R().SetBody(base.Json{ + "fnmae": stream.GetName(), + "parts": parts, + }).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/wpanstore-lanzou/objects/%s/uploads/%s", keyBase64, uploadId)) + if err != nil { + return nil, err + } + token = utils.Json.Get(res.Body(), "token").ToString() + } + // commit upload + var resp UploadResultResp + for i := 0; i < 10; i++ { + _, err = d.unproved("/7n/results", http.MethodPost, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "tokenList": token, + "tokenTime": time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)"), + }).SetResult(&resp) + }) + if err != nil { + return nil, err + } + if len(resp.List) == 0 { + return nil, fmt.Errorf("upload failed, empty response") + } + if resp.List[0].Status == 1 { + break + } + time.Sleep(time.Second * 1) + } + file := resp.List[0] + if file.Status != 1 { + return nil, fmt.Errorf("upload failed, status: %d", resp.List[0].Status) + } + return &model.Object{ + ID: strconv.FormatInt(file.FileId, 10), + //Path: , + Name: file.FileName, + Size: stream.GetSize(), + Modified: stream.ModTime(), + Ctime: stream.CreateTime(), + IsFolder: false, + HashInfo: utils.NewHashInfo(utils.MD5, etag), + }, nil +} + +//func (d *ILanZou) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*ILanZou)(nil) diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go new file mode 100644 index 00000000000..44adbf0a6f7 --- /dev/null +++ b/drivers/ilanzou/meta.go @@ -0,0 +1,35 @@ +package template + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Username string `json:"username" type:"string" required:"true"` + Password string `json:"password" type:"string" required:"true"` + + Token string + UUID string +} + +var config = driver.Config{ + Name: "ILanZou", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ILanZou{} + }) +} diff --git a/drivers/ilanzou/types.go b/drivers/ilanzou/types.go new file mode 100644 index 00000000000..135724c749c --- /dev/null +++ b/drivers/ilanzou/types.go @@ -0,0 +1,57 @@ +package template + +type ListResp struct { + Msg string `json:"msg"` + Total int `json:"total"` + Code int `json:"code"` + Offset int `json:"offset"` + TotalPage int `json:"totalPage"` + Limit int `json:"limit"` + List []ListItem `json:"list"` +} + +type ListItem struct { + IconId int `json:"iconId"` + IsAmt int `json:"isAmt"` + FolderDesc string `json:"folderDesc,omitempty"` + AddTime string `json:"addTime"` + FolderId int64 `json:"folderId"` + ParentId int64 `json:"parentId"` + ParentName string `json:"parentName"` + NoteType int `json:"noteType,omitempty"` + UpdTime string `json:"updTime"` + IsShare int `json:"isShare"` + FolderIcon string `json:"folderIcon,omitempty"` + FolderName string `json:"folderName,omitempty"` + FileType int `json:"fileType"` + Status int `json:"status"` + IsFileShare int `json:"isFileShare,omitempty"` + FileName string `json:"fileName,omitempty"` + FileStars float64 `json:"fileStars,omitempty"` + IsFileDownload int `json:"isFileDownload,omitempty"` + FileComments int `json:"fileComments,omitempty"` + FileSize int64 `json:"fileSize,omitempty"` + FileIcon string `json:"fileIcon,omitempty"` + FileDownloads int `json:"fileDownloads,omitempty"` + FileUrl interface{} `json:"fileUrl"` + FileLikes int `json:"fileLikes,omitempty"` + FileId int64 `json:"fileId,omitempty"` +} + +type Part struct { + PartNumber int `json:"partNumber"` + ETag string `json:"etag"` +} + +type UploadResultResp struct { + Msg string `json:"msg"` + Code int `json:"code"` + List []struct { + FileIconId int `json:"fileIconId"` + FileName string `json:"fileName"` + FileIcon string `json:"fileIcon"` + FileId int64 `json:"fileId"` + Status int `json:"status"` + Token string `json:"token"` + } `json:"list"` +} diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go new file mode 100644 index 00000000000..2ccaf52e165 --- /dev/null +++ b/drivers/ilanzou/util.go @@ -0,0 +1,105 @@ +package template + +import ( + "encoding/hex" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/foxxorcat/mopan-sdk-go" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +const ( + Base = "https://api.ilanzou.com" +) + +var ( + AesSecret = []byte("lanZouY-disk-app") +) + +func (d *ILanZou) login() error { + res, err := d.unproved("/login", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "loginName": d.Username, + "loginPwd": d.Password, + }) + }) + if err != nil { + return err + } + d.Token = utils.Json.Get(res, "data", "appToken").ToString() + if d.Token == "" { + return fmt.Errorf("failed to login: token is empty, resp: %s", res) + } + return nil +} + +func getTimestamp() (string, error) { + ts := time.Now().UnixMilli() + tsStr := strconv.FormatInt(ts, 10) + res, err := mopan.AesEncrypt([]byte(tsStr), AesSecret) + if err != nil { + return "", err + } + return hex.EncodeToString(res), nil +} + +func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, proved bool, retry ...bool) ([]byte, error) { + req := base.RestyClient.R() + ts, err := getTimestamp() + if err != nil { + return nil, err + } + req.SetQueryParams(map[string]string{ + "uuid": d.UUID, + "devType": "6", + "devCode": d.UUID, + "devModel": "chrome", + "devVersion": "120", + "appVersion": "", + "timestamp": ts, + //"appToken": d.Token, + "extra": "2", + }) + if proved { + req.SetQueryParam("appToken", d.Token) + } + if callback != nil { + callback(req) + } + res, err := req.Execute(method, Base+pathname) + if err != nil { + if res != nil { + log.Errorf("[iLanZou] request error: %s", res.String()) + } + return nil, err + } + isRetry := len(retry) > 0 && retry[0] + body := res.Body() + code := utils.Json.Get(body, "code").ToInt() + msg := utils.Json.Get(body, "msg").ToString() + if code != 200 { + if !isRetry && proved && (utils.SliceContains([]int{-1, -2}, code) || d.Token == "") { + err = d.login() + if err != nil { + return nil, err + } + return d.request(pathname, method, callback, proved, true) + } + return nil, fmt.Errorf("%d: %s", code, msg) + } + return body, nil +} + +func (d *ILanZou) unproved(pathname, method string, callback base.ReqCallback) ([]byte, error) { + return d.request("/unproved"+pathname, method, callback, false) +} + +func (d *ILanZou) proved(pathname, method string, callback base.ReqCallback) ([]byte, error) { + return d.request("/proved"+pathname, method, callback, true) +} From 8020d42b1047e74f6f96efc59bb47c1271220432 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 5 Jan 2024 11:40:44 +0800 Subject: [PATCH 095/659] fix: panic due to send on closed channel (close #5729) --- internal/net/request.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/internal/net/request.go b/internal/net/request.go index b450ede5a02..78aa3832843 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -450,15 +450,19 @@ type Buf struct { ctx context.Context off int rw sync.RWMutex - notify chan struct{} + //notify chan struct{} } // NewBuf is a buffer that can have 1 read & 1 write at the same time. // when read is faster write, immediately feed data to read after written func NewBuf(ctx context.Context, maxSize int, id int) *Buf { d := make([]byte, 0, maxSize) - return &Buf{ctx: ctx, buffer: bytes.NewBuffer(d), size: maxSize, notify: make(chan struct{})} - + return &Buf{ + ctx: ctx, + buffer: bytes.NewBuffer(d), + size: maxSize, + //notify: make(chan struct{}), + } } func (br *Buf) Reset(size int) { br.buffer.Reset() @@ -495,8 +499,8 @@ func (br *Buf) Read(p []byte) (n int, err error) { select { case <-br.ctx.Done(): return 0, br.ctx.Err() - case <-br.notify: - return 0, nil + //case <-br.notify: + // return 0, nil case <-time.After(time.Millisecond * 200): return 0, nil } @@ -510,12 +514,12 @@ func (br *Buf) Write(p []byte) (n int, err error) { defer br.rw.Unlock() n, err = br.buffer.Write(p) select { - case br.notify <- struct{}{}: + //case br.notify <- struct{}{}: default: } return } func (br *Buf) Close() { - close(br.notify) + //close(br.notify) } From 4448e08f5beb1ee44453c1844775332063b1454b Mon Sep 17 00:00:00 2001 From: Rammiah Date: Fri, 5 Jan 2024 12:20:08 +0800 Subject: [PATCH 096/659] fix(net): Buf use Mutex (#5823) Co-authored-by: Andy Hsu --- internal/net/request.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/net/request.go b/internal/net/request.go index 78aa3832843..71f45aa7afc 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -449,7 +449,7 @@ type Buf struct { size int //expected size ctx context.Context off int - rw sync.RWMutex + rw sync.Mutex //notify chan struct{} } @@ -480,9 +480,9 @@ func (br *Buf) Read(p []byte) (n int, err error) { if br.off >= br.size { return 0, io.EOF } - br.rw.RLock() + br.rw.Lock() n, err = br.buffer.Read(p) - br.rw.RUnlock() + br.rw.Unlock() if err == nil { br.off += n return n, err From fb729c18461566b219ef8d8f564cc69118f21a4f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:34:16 +0800 Subject: [PATCH 097/659] fix(deps): update module github.com/aws/aws-sdk-go to v1.49.15 (#5816) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 86ed7b82791..5b0073d064f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Xhofe/wopan-sdk-go v0.1.2 github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.49.13 + github.com/aws/aws-sdk-go v1.49.15 github.com/blevesearch/bleve/v2 v2.3.10 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.17.1 diff --git a/go.sum b/go.sum index 855d1c6033a..3075311274f 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ github.com/aws/aws-sdk-go v1.46.7 h1:IjvAWeiJZlbETOemOwvheN5L17CvKvKW0T1xOC6d3Sc github.com/aws/aws-sdk-go v1.46.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y= github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k= +github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= From 28bb3f631057fcb784e40dcbdca71b52b4122d4b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:35:41 +0800 Subject: [PATCH 098/659] fix(deps): update module golang.org/x/image to v0.15.0 (#5825) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5b0073d064f..9ba70e5ae54 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( github.com/xhofe/tache v0.1.1 golang.org/x/crypto v0.17.0 golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc - golang.org/x/image v0.14.0 + golang.org/x/image v0.15.0 golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.15.0 golang.org/x/time v0.5.0 diff --git a/go.sum b/go.sum index 3075311274f..880783c03ff 100644 --- a/go.sum +++ b/go.sum @@ -536,6 +536,8 @@ golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= +golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= From e6e2d03ba175edea1c9c2c6bd42259e11ed764a0 Mon Sep 17 00:00:00 2001 From: Mmx <36563672+Mmx233@users.noreply.github.com> Date: Fri, 5 Jan 2024 15:52:30 +0800 Subject: [PATCH 099/659] perf: make docker release 10 times faster (#5803) * build: improve multistage docker build * build: add dockerfile for ci * build: add BuildDockerMultiplatform function in build.sh for ci * ci: change build method * build: add missing mod download command to the Dockerfile * build: revert changes made ffmpeg installed * build: use musl build for docker release * ci: apply to dev version * fix: don't login on pr * fix: don't build_docker_with_aria2 on pr --------- Co-authored-by: Andy Hsu --- .github/workflows/build_docker.yml | 26 ++++++++++--- .github/workflows/release_docker.yml | 8 ++++ Dockerfile | 15 +++++--- Dockerfile.ci | 16 ++++++++ build.sh | 55 +++++++++++++++++++++++++++- entrypoint.sh | 6 ++- 6 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 Dockerfile.ci diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 396981192ef..3b733b3ba2f 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -3,6 +3,8 @@ name: build_docker on: push: branches: [ main ] + pull_request: + branches: [ main ] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -10,42 +12,54 @@ concurrency: jobs: build_docker: - name: Build docker + name: Build Docker runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + + - uses: actions/setup-go@v4 + with: + go-version: 'stable' + + - name: Build go binary + run: bash build.sh dev docker-multiplatform + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: images: xhofe/alist - - name: Replace release with dev - run: | - sed -i 's/release/dev/g' Dockerfile + - name: Set up QEMU uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub + if: github.event_name == 'push' uses: docker/login-action@v3 with: username: xhofe password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push id: docker_build uses: docker/build-push-action@v5 with: context: . - push: true + file: Dockerfile.ci + push: ${{ github.event_name == 'push' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x build_docker_with_aria2: needs: build_docker name: Build docker with aria2 runs-on: ubuntu-latest + if: github.event_name == 'push' steps: - name: Checkout repo uses: actions/checkout@v4 diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 51ef40cc6a7..b029484e9bb 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -13,6 +13,13 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - uses: actions/setup-go@v4 + with: + go-version: 'stable' + + - name: Build go binary + run: bash build.sh release docker-multiplatform + - name: Docker meta id: meta uses: docker/metadata-action@v5 @@ -36,6 +43,7 @@ jobs: uses: docker/build-push-action@v5 with: context: . + file: Dockerfile.ci push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/Dockerfile b/Dockerfile index 542d502c53b..23ca42da201 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ FROM alpine:edge as builder LABEL stage=go-builder WORKDIR /app/ +RUN apk add --no-cache bash curl gcc git go musl-dev +COPY go.mod go.sum ./ +RUN go mod download COPY ./ ./ -RUN apk add --no-cache bash curl gcc git go musl-dev; \ - bash build.sh release docker +RUN bash build.sh release docker FROM alpine:edge LABEL MAINTAINER="i@nn.ci" @@ -11,8 +13,11 @@ VOLUME /opt/alist/data/ WORKDIR /opt/alist/ COPY --from=builder /app/bin/alist ./ COPY entrypoint.sh /entrypoint.sh -RUN apk add --no-cache bash ca-certificates su-exec tzdata; \ - chmod +x /entrypoint.sh +RUN apk update && \ + apk upgrade --no-cache && \ + apk add --no-cache bash ca-certificates su-exec tzdata; \ + chmod +x /entrypoint.sh && \ + rm -rf /var/cache/apk/* ENV PUID=0 PGID=0 UMASK=022 EXPOSE 5244 5245 -CMD [ "/entrypoint.sh" ] +CMD [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 00000000000..dc18d259f58 --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,16 @@ +FROM alpine:edge +ARG TARGETPLATFORM +LABEL MAINTAINER="i@nn.ci" +VOLUME /opt/alist/data/ +WORKDIR /opt/alist/ +COPY /${TARGETPLATFORM}/alist ./ +COPY entrypoint.sh /entrypoint.sh +RUN apk update && \ + apk upgrade --no-cache && \ + apk add --no-cache bash ca-certificates su-exec tzdata; \ + chmod +x /entrypoint.sh && \ + rm -rf /var/cache/apk/* && \ + /entrypoint.sh version +ENV PUID=0 PGID=0 UMASK=022 +EXPOSE 5244 5245 +CMD [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/build.sh b/build.sh index 4ea19ecc273..7f56e5b2e9a 100644 --- a/build.sh +++ b/build.sh @@ -85,12 +85,61 @@ BuildDev() { cat md5.txt } -BuildDocker() { +PrepareBuildDocker() { echo "replace github.com/mattn/go-sqlite3 => github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed" >>go.mod go get gorm.io/driver/sqlite@v1.4.4 + go mod download +} + +BuildDocker() { + PrepareBuildDocker go build -o ./bin/alist -ldflags="$ldflags" -tags=jsoniter . } +BuildDockerMultiplatform() { + PrepareBuildDocker + + BASE="https://musl.cc/" + FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross) + for i in "${FILES[@]}"; do + url="${BASE}${i}.tgz" + curl -L -o "${i}.tgz" "${url}" + sudo tar xf "${i}.tgz" --strip-components 1 -C /usr/local + rm -f "${i}.tgz" + done + + docker_lflags="--extldflags '-static -fpic' $ldflags" + export CGO_ENABLED=1 + + OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-s390x) + CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc s390x-linux-musl-gcc) + for i in "${!OS_ARCHES[@]}"; do + os_arch=${OS_ARCHES[$i]} + cgo_cc=${CGO_ARGS[$i]} + os=${os_arch%%-*} + arch=${os_arch##*-} + export GOOS=$os + export GOARCH=$arch + export CC=${cgo_cc} + echo "building for $os_arch" + go build -o ./$os/$arch/alist -ldflags="$docker_lflags" -tags=jsoniter . + done + + DOCKER_ARM_ARCHES=(linux-arm/v6 linux-arm/v7) + CGO_ARGS=(armv6-linux-musleabihf-gcc armv7l-linux-musleabihf-gcc) + GO_ARM=(6 7) + export GOOS=linux + export GOARCH=arm + for i in "${!DOCKER_ARM_ARCHES[@]}"; do + docker_arch=${DOCKER_ARM_ARCHES[$i]} + cgo_cc=${CGO_ARGS[$i]} + export GOARM=${GO_ARM[$i]} + export CC=${cgo_cc} + echo "building for $docker_arch" + go build -o ./${docker_arch%%-*}/${docker_arch##*-}/alist -ldflags="$docker_lflags" -tags=jsoniter . + done +} + BuildRelease() { rm -rf .git/ mkdir -p "build" @@ -190,6 +239,8 @@ if [ "$1" = "dev" ]; then FetchWebDev if [ "$2" = "docker" ]; then BuildDocker + elif [ "$2" = "docker-multiplatform" ]; then + BuildDockerMultiplatform else BuildDev fi @@ -197,6 +248,8 @@ elif [ "$1" = "release" ]; then FetchWebRelease if [ "$2" = "docker" ]; then BuildDocker + elif [ "$2" = "docker-multiplatform" ]; then + BuildDockerMultiplatform elif [ "$2" = "linux_musl_arm" ]; then BuildReleaseLinuxMuslArm MakeRelease "md5-linux-musl-arm.txt" diff --git a/entrypoint.sh b/entrypoint.sh index 05bbf8d3c83..a0d8083509e 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,4 +4,8 @@ chown -R ${PUID}:${PGID} /opt/alist/ umask ${UMASK} -exec su-exec ${PUID}:${PGID} ./alist server --no-prefix \ No newline at end of file +if [ "$1" = "version" ]; then + ./alist version +else + exec su-exec ${PUID}:${PGID} ./alist server --no-prefix +fi \ No newline at end of file From 434892f1356e5c966b8a04bfffeb95973830005a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 9 Jan 2024 17:14:31 +0800 Subject: [PATCH 100/659] fix(deps): update module github.com/aliyun/aliyun-oss-go-sdk to v3 (#5800) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9ba70e5ae54..e25c077e20a 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/Xhofe/wopan-sdk-go v0.1.2 - github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.49.15 github.com/blevesearch/bleve/v2 v2.3.10 diff --git a/go.sum b/go.sum index 880783c03ff..48b8519388a 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiw github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible h1:ROMcuN61gI8SfQ+AEMh4d7GZ3gwTZLIhPjtd05TQCG4= github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/Hcta4l4VrIXrI= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= From 34b73b94f728fc10033fd2e016542addaa3861e5 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 9 Jan 2024 18:51:21 +0800 Subject: [PATCH 101/659] feat(local): allow specifying the recycle bin path (close #5832) --- drivers/local/driver.go | 14 +++++++++++--- drivers/local/meta.go | 1 + 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 04604366f65..4efee6d6c6c 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -257,10 +257,18 @@ func (d *Local) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { func (d *Local) Remove(ctx context.Context, obj model.Obj) error { var err error - if obj.IsDir() { - err = os.RemoveAll(obj.GetPath()) + if utils.SliceContains([]string{"", "delete permanently"}, d.RecycleBinPath) { + if obj.IsDir() { + err = os.RemoveAll(obj.GetPath()) + } else { + err = os.Remove(obj.GetPath()) + } } else { - err = os.Remove(obj.GetPath()) + dstPath := filepath.Join(d.RecycleBinPath, obj.GetName()) + if utils.Exists(dstPath) { + dstPath = filepath.Join(d.RecycleBinPath, obj.GetName()+"_"+time.Now().Format("20060102150405")) + } + err = os.Rename(obj.GetPath(), dstPath) } if err != nil { return err diff --git a/drivers/local/meta.go b/drivers/local/meta.go index 00c8f5ce8b1..51b49e64ef4 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -11,6 +11,7 @@ type Addition struct { ThumbCacheFolder string `json:"thumb_cache_folder"` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` MkdirPerm string `json:"mkdir_perm" default:"777"` + RecycleBinPath string `json:"recycle_bin_path" default:"delete permanently" help:"path to recycle bin, delete permanently if empty or keep 'delete permanently'"` } var config = driver.Config{ From bff56ffd0fe9e7002d921dbf3b607308964efac0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A7=8B=E7=A7=8B?= Date: Tue, 9 Jan 2024 19:00:11 +0800 Subject: [PATCH 102/659] ci: add `android` target to release build (#5844) * build: build android Signed-off-by: lateautumn233 * ci: add `android` target to release build Signed-off-by: lateautumn233 --------- Signed-off-by: lateautumn233 --- .github/workflows/release_android.yml | 34 +++++++++++++++++++++++++++ build.sh | 29 +++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 .github/workflows/release_android.yml diff --git a/.github/workflows/release_android.yml b/.github/workflows/release_android.yml new file mode 100644 index 00000000000..c696ddb743a --- /dev/null +++ b/.github/workflows/release_android.yml @@ -0,0 +1,34 @@ +name: release_android + +on: + release: + types: [ published ] + +jobs: + release_android: + strategy: + matrix: + platform: [ ubuntu-latest ] + go-version: [ '1.21' ] + name: Release + runs-on: ${{ matrix.platform }} + steps: + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build + run: | + bash build.sh release android + + - name: Upload assets + uses: softprops/action-gh-release@v1 + with: + files: build/compress/* diff --git a/build.sh b/build.sh index 7f56e5b2e9a..f036d714efa 100644 --- a/build.sh +++ b/build.sh @@ -211,6 +211,27 @@ BuildReleaseLinuxMuslArm() { done } +BuildReleaseAndroid() { + rm -rf .git/ + mkdir -p "build" + wget https://dl.google.com/android/repository/android-ndk-r26b-linux.zip + unzip android-ndk-r26b-linux.zip + rm android-ndk-r26b-linux.zip + OS_ARCHES=(amd64 arm64 386 arm) + CGO_ARGS=(x86_64-linux-android24-clang aarch64-linux-android24-clang i686-linux-android24-clang armv7a-linux-androideabi24-clang) + for i in "${!OS_ARCHES[@]}"; do + os_arch=${OS_ARCHES[$i]} + cgo_cc=$(realpath android-ndk-r26b/toolchains/llvm/prebuilt/linux-x86_64/bin/${CGO_ARGS[$i]}) + echo building for android-${os_arch} + export GOOS=android + export GOARCH=${os_arch##*-} + export CC=${cgo_cc} + export CGO_ENABLED=1 + go build -o ./build/$appName-android-$os_arch -ldflags="$ldflags" -tags=jsoniter . + android-ndk-r26b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip ./build/$appName-android-$os_arch + done +} + MakeRelease() { cd build mkdir compress @@ -218,6 +239,11 @@ MakeRelease() { cp "$i" alist tar -czvf compress/"$i".tar.gz alist rm -f alist + done + for i in $(find . -type f -name "$appName-android-*"); do + cp "$i" alist + tar -czvf compress/"$i".tar.gz alist + rm -f alist done for i in $(find . -type f -name "$appName-darwin-*"); do cp "$i" alist @@ -256,6 +282,9 @@ elif [ "$1" = "release" ]; then elif [ "$2" = "linux_musl" ]; then BuildReleaseLinuxMusl MakeRelease "md5-linux-musl.txt" + elif [ "$2" = "android" ]; then + BuildReleaseAndroid + MakeRelease "md5-android.txt" else BuildRelease MakeRelease "md5.txt" From 555ef0eb1aad5e68f1b4e6fcbe04c04219f6f7f5 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 10 Jan 2024 16:58:10 +0800 Subject: [PATCH 103/659] feat: add `feijipan` driver (close #5856) --- drivers/ilanzou/driver.go | 22 +++++++------ drivers/ilanzou/meta.go | 65 +++++++++++++++++++++++++++++++-------- drivers/ilanzou/util.go | 20 ++++-------- 3 files changed, 70 insertions(+), 37 deletions(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 85ba27e2e23..434994d89f9 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -30,10 +30,12 @@ type ILanZou struct { userID string account string upClient *resty.Client + conf Conf + config driver.Config } func (d *ILanZou) Config() driver.Config { - return config + return d.config } func (d *ILanZou) GetAddition() driver.Additional { @@ -123,19 +125,19 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) query.Set("devModel", "chrome") query.Set("devVersion", "120") query.Set("appVersion", "") - ts, err := getTimestamp() + ts, err := getTimestamp(d.conf.secret) if err != nil { return nil, err } query.Set("timestamp", ts) //query.Set("appToken", d.Token) query.Set("enable", "1") - downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), AesSecret) + downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) if err != nil { return nil, err } query.Set("downloadId", hex.EncodeToString(downloadId)) - auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), time.Now().UnixMilli())), AesSecret) + auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), time.Now().UnixMilli())), d.conf.secret) if err != nil { return nil, err } @@ -281,7 +283,7 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt now := time.Now() key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) var token string - if stream.GetSize() > DefaultPartSize { + if stream.GetSize() <= DefaultPartSize { res, err := d.upClient.R().SetMultipartFormData(map[string]string{ "token": upToken, "key": key, @@ -294,7 +296,7 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt token = utils.Json.Get(res.Body(), "token").ToString() } else { keyBase64 := base64.URLEncoding.EncodeToString([]byte(key)) - res, err := d.upClient.R().Post(fmt.Sprintf("https://upload.qiniup.com/buckets/wpanstore-lanzou/objects/%s/uploads", keyBase64)) + res, err := d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads", d.conf.bucket, keyBase64)) if err != nil { return nil, err } @@ -302,8 +304,8 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt parts := make([]Part, 0) partNum := (stream.GetSize() + DefaultPartSize - 1) / DefaultPartSize for i := 1; i <= int(partNum); i++ { - u := fmt.Sprintf("https://upload.qiniup.com/buckets/wpanstore-lanzou/objects/%s/uploads/%s/%d", keyBase64, uploadId, i) - res, err = d.upClient.R().SetBody(io.LimitReader(tempFile, DefaultPartSize)).Put(u) + u := fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s/%d", d.conf.bucket, keyBase64, uploadId, i) + res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(io.LimitReader(tempFile, DefaultPartSize)).Put(u) if err != nil { return nil, err } @@ -313,10 +315,10 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt ETag: etag, }) } - res, err = d.upClient.R().SetBody(base.Json{ + res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(base.Json{ "fnmae": stream.GetName(), "parts": parts, - }).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/wpanstore-lanzou/objects/%s/uploads/%s", keyBase64, uploadId)) + }).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s", d.conf.bucket, keyBase64, uploadId)) if err != nil { return nil, err } diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go index 44adbf0a6f7..f7c61e5a7fe 100644 --- a/drivers/ilanzou/meta.go +++ b/drivers/ilanzou/meta.go @@ -14,22 +14,61 @@ type Addition struct { UUID string } -var config = driver.Config{ - Name: "ILanZou", - LocalSort: false, - OnlyLocal: false, - OnlyProxy: false, - NoCache: false, - NoUpload: false, - NeedMs: false, - DefaultRoot: "0", - CheckStatus: false, - Alert: "", - NoOverwriteUpload: false, +type Conf struct { + base string + secret []byte + bucket string + unproved string + proved string } func init() { op.RegisterDriver(func() driver.Driver { - return &ILanZou{} + return &ILanZou{ + config: driver.Config{ + Name: "ILanZou", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, + }, + conf: Conf{ + base: "https://api.ilanzou.com", + secret: []byte("lanZouY-disk-app"), + bucket: "wpanstore-lanzou", + unproved: "unproved", + proved: "proved", + }, + } + }) + op.RegisterDriver(func() driver.Driver { + return &ILanZou{ + config: driver.Config{ + Name: "FeijiPan", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, + }, + conf: Conf{ + base: "https://api.feijipan.com", + secret: []byte("dingHao-disk-app"), + bucket: "wpanstore", + unproved: "ws", + proved: "app", + }, + } }) } diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index 2ccaf52e165..d8995523ea0 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -14,14 +14,6 @@ import ( log "github.com/sirupsen/logrus" ) -const ( - Base = "https://api.ilanzou.com" -) - -var ( - AesSecret = []byte("lanZouY-disk-app") -) - func (d *ILanZou) login() error { res, err := d.unproved("/login", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ @@ -39,10 +31,10 @@ func (d *ILanZou) login() error { return nil } -func getTimestamp() (string, error) { +func getTimestamp(secret []byte) (string, error) { ts := time.Now().UnixMilli() tsStr := strconv.FormatInt(ts, 10) - res, err := mopan.AesEncrypt([]byte(tsStr), AesSecret) + res, err := mopan.AesEncrypt([]byte(tsStr), secret) if err != nil { return "", err } @@ -51,7 +43,7 @@ func getTimestamp() (string, error) { func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, proved bool, retry ...bool) ([]byte, error) { req := base.RestyClient.R() - ts, err := getTimestamp() + ts, err := getTimestamp(d.conf.secret) if err != nil { return nil, err } @@ -72,7 +64,7 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr if callback != nil { callback(req) } - res, err := req.Execute(method, Base+pathname) + res, err := req.Execute(method, d.conf.base+pathname) if err != nil { if res != nil { log.Errorf("[iLanZou] request error: %s", res.String()) @@ -97,9 +89,9 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr } func (d *ILanZou) unproved(pathname, method string, callback base.ReqCallback) ([]byte, error) { - return d.request("/unproved"+pathname, method, callback, false) + return d.request("/"+d.conf.unproved+pathname, method, callback, false) } func (d *ILanZou) proved(pathname, method string, callback base.ReqCallback) ([]byte, error) { - return d.request("/proved"+pathname, method, callback, true) + return d.request("/"+d.conf.proved+pathname, method, callback, true) } From bb6747de4e1654b37359c9de3cb83a95edfd1a48 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 11 Jan 2024 10:15:16 +0800 Subject: [PATCH 104/659] docs: add `feijipan` to Readme --- README.md | 1 + README_cn.md | 1 + README_ja.md | 1 + 3 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 74cd291d7df..431b32128a5 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] [115](https://115.com/) - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) + - [x] [FeijiPan](https://www.feijipan.com/) - [x] Easy to deploy and out-of-the-box - [x] File preview (PDF, markdown, code, plain text, ...) - [x] Image preview in gallery mode diff --git a/README_cn.md b/README_cn.md index 9d3603fa529..db8455e1eb8 100644 --- a/README_cn.md +++ b/README_cn.md @@ -74,6 +74,7 @@ - [x] [115](https://115.com/) - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) + - [x] [飞机盘](https://www.feijipan.com/) - [x] 部署方便，开箱即用 - [x] 文件预览（PDF、markdown、代码、纯文本……） - [x] 画廊模式下的图像预览 diff --git a/README_ja.md b/README_ja.md index a50425455b3..67b2840a586 100644 --- a/README_ja.md +++ b/README_ja.md @@ -75,6 +75,7 @@ - [x] [115](https://115.com/) - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) + - [x] [FeijiPan](https://www.feijipan.com/) - [x] デプロイが簡単で、すぐに使える - [x] ファイルプレビュー (PDF, マークダウン, コード, プレーンテキスト, ...) - [x] ギャラリーモードでの画像プレビュー From 292bbe94eed0d4f5965b042d8a585c1c34e08fec Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 11 Jan 2024 10:16:14 +0800 Subject: [PATCH 105/659] fix(feijipan): incorrect address of download link (close #5859) --- drivers/ilanzou/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 434994d89f9..080e3b55129 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -114,7 +114,7 @@ func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) } func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - u, err := url.Parse("https://api.ilanzou.com/unproved/file/redirect") + u, err := url.Parse(d.conf.base + "/" + d.conf.unproved + "/file/redirect") if err != nil { return nil, err } From 1381e8fb27b1993b62f0b2c40f7a3dd5fd3633fb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Jan 2024 20:24:20 +0800 Subject: [PATCH 106/659] fix(deps): update module github.com/aws/aws-sdk-go to v1.49.18 (#5848) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e25c077e20a..e9961278624 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Xhofe/wopan-sdk-go v0.1.2 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.49.15 + github.com/aws/aws-sdk-go v1.49.18 github.com/blevesearch/bleve/v2 v2.3.10 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.17.1 diff --git a/go.sum b/go.sum index 48b8519388a..bccf4577cbd 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,8 @@ github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1 github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k= github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.49.18 h1:g/iMXkfXeJQ7MvnLwroxWsTTNkHtdVJGxIgrAIEG62M= +github.com/aws/aws-sdk-go v1.49.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= From 85fe65951d6dfb637c912148b6df24c0b7723a62 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Jan 2024 20:39:16 +0800 Subject: [PATCH 107/659] fix(deps): update golang.org/x/exp digest to 0dcbfd6 (#5862) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e9961278624..a2ff7b3c294 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 golang.org/x/crypto v0.17.0 - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc + golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/image v0.15.0 golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.15.0 diff --git a/go.sum b/go.sum index bccf4577cbd..c504c30fd52 100644 --- a/go.sum +++ b/go.sum @@ -535,6 +535,8 @@ golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcs golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= +golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= From 4930f85b90d455add7631bbbc87dc4eed79e5697 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 11 Jan 2024 20:56:19 +0800 Subject: [PATCH 108/659] fix(deps): update module golang.org/x/crypto to v0.18.0 (#5863) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a2ff7b3c294..474d4ca61e5 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 - golang.org/x/crypto v0.17.0 + golang.org/x/crypto v0.18.0 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/image v0.15.0 golang.org/x/net v0.19.0 @@ -194,8 +194,8 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.5.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/api v0.134.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect diff --git a/go.sum b/go.sum index c504c30fd52..abd416bcf3d 100644 --- a/go.sum +++ b/go.sum @@ -529,6 +529,8 @@ golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= @@ -597,6 +599,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -605,6 +609,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 86b35ae5cfec400871072356fec4dea88303195d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 12 Jan 2024 11:54:35 +0800 Subject: [PATCH 109/659] fix(deps): update module golang.org/x/oauth2 to v0.16.0 (#5865) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 474d4ca61e5..389a1bac8f3 100644 --- a/go.mod +++ b/go.mod @@ -53,8 +53,8 @@ require ( golang.org/x/crypto v0.18.0 golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/image v0.15.0 - golang.org/x/net v0.19.0 - golang.org/x/oauth2 v0.15.0 + golang.org/x/net v0.20.0 + golang.org/x/oauth2 v0.16.0 golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 diff --git a/go.sum b/go.sum index abd416bcf3d..c226f7eed9b 100644 --- a/go.sum +++ b/go.sum @@ -562,10 +562,14 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From f0e8c0e886d3521763c0436b558d26a02d481985 Mon Sep 17 00:00:00 2001 From: SoY0ung <32095074+SoY0ung@users.noreply.github.com> Date: Sun, 14 Jan 2024 12:53:31 +0800 Subject: [PATCH 110/659] fix(chaoxing): JSON parsing error in `content` field (#5877) * fix(chaoxing):fix JSON parsing error in `content` field * fix(chaoxing): optimizing `UnmarshalJSON` implementation * fix(chaoxing): use `objectID` when is empty --- drivers/chaoxing/types.go | 86 +++++++++++++++++++++++---------------- drivers/chaoxing/util.go | 10 +++-- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go index a1ce13c3019..ba636ec1dd0 100644 --- a/drivers/chaoxing/types.go +++ b/drivers/chaoxing/types.go @@ -1,7 +1,9 @@ package chaoxing import ( + "bytes" "fmt" + "strconv" "time" "github.com/alist-org/alist/v3/internal/model" @@ -88,44 +90,59 @@ type UserAuth struct { } `json:"operationAuth"` } +// 手机端学习通上传的文件的json内容(content字段)与网页端上传的有所不同 +// 网页端json `"puid": 54321, "size": 12345` +// 手机端json `"puid": "54321". "size": "12345"` +type int_str int + +// json 字符串数字和纯数字解析 +func (ios *int_str) UnmarshalJSON(data []byte) error { + intValue, err := strconv.Atoi(string(bytes.Trim(data, "\""))) + if err != nil { + return err + } + *ios = int_str(intValue) + return nil +} + type File struct { Cataid int `json:"cataid"` Cfid int `json:"cfid"` Content struct { - Cfid int `json:"cfid"` - Pid int `json:"pid"` - FolderName string `json:"folderName"` - ShareType int `json:"shareType"` - Preview string `json:"preview"` - Filetype string `json:"filetype"` - PreviewURL string `json:"previewUrl"` - IsImg bool `json:"isImg"` - ParentPath string `json:"parentPath"` - Icon string `json:"icon"` - Suffix string `json:"suffix"` - Duration int `json:"duration"` - Pantype string `json:"pantype"` - Puid int `json:"puid"` - Filepath string `json:"filepath"` - Crc string `json:"crc"` - Isfile bool `json:"isfile"` - Residstr string `json:"residstr"` - ObjectID string `json:"objectId"` - Extinfo string `json:"extinfo"` - Thumbnail string `json:"thumbnail"` - Creator int `json:"creator"` - ResTypeValue int `json:"resTypeValue"` - UploadDateFormat string `json:"uploadDateFormat"` - DisableOpt bool `json:"disableOpt"` - DownPath string `json:"downPath"` - Sort int `json:"sort"` - Topsort int `json:"topsort"` - Restype string `json:"restype"` - Size int `json:"size"` - UploadDate string `json:"uploadDate"` - FileSize string `json:"fileSize"` - Name string `json:"name"` - FileID string `json:"fileId"` + Cfid int `json:"cfid"` + Pid int `json:"pid"` + FolderName string `json:"folderName"` + ShareType int `json:"shareType"` + Preview string `json:"preview"` + Filetype string `json:"filetype"` + PreviewURL string `json:"previewUrl"` + IsImg bool `json:"isImg"` + ParentPath string `json:"parentPath"` + Icon string `json:"icon"` + Suffix string `json:"suffix"` + Duration int `json:"duration"` + Pantype string `json:"pantype"` + Puid int_str `json:"puid"` + Filepath string `json:"filepath"` + Crc string `json:"crc"` + Isfile bool `json:"isfile"` + Residstr string `json:"residstr"` + ObjectID string `json:"objectId"` + Extinfo string `json:"extinfo"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + ResTypeValue int `json:"resTypeValue"` + UploadDateFormat string `json:"uploadDateFormat"` + DisableOpt bool `json:"disableOpt"` + DownPath string `json:"downPath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + Restype string `json:"restype"` + Size int_str `json:"size"` + UploadDate string `json:"uploadDate"` + FileSize string `json:"fileSize"` + Name string `json:"name"` + FileID string `json:"fileId"` } `json:"content"` CreatorID int `json:"creatorId"` DesID string `json:"des_id"` @@ -204,7 +221,6 @@ type UploadFileDataRsp struct { } `json:"data"` } - type UploadDoneParam struct { Cataid string `json:"cataid"` Key string `json:"key"` diff --git a/drivers/chaoxing/util.go b/drivers/chaoxing/util.go index 2e34994dd90..b6725804e6c 100644 --- a/drivers/chaoxing/util.go +++ b/drivers/chaoxing/util.go @@ -79,7 +79,7 @@ func (d *ChaoXing) GetFiles(parent string) ([]File, error) { return nil, err } if resp.Result != 1 { - msg:=fmt.Sprintf("error code is:%d", resp.Result) + msg := fmt.Sprintf("error code is:%d", resp.Result) return nil, errors.New(msg) } if len(resp.List) > 0 { @@ -97,8 +97,12 @@ func (d *ChaoXing) GetFiles(parent string) ([]File, error) { if err != nil { return nil, err } - if len(resps.List) > 0 { - files = append(files, resps.List...) + for _, file := range resps.List { + // 手机端超星上传的文件没有fileID字段，但ObjectID与fileID相同，可代替 + if file.Content.FileID == "" { + file.Content.FileID = file.Content.ObjectID + } + files = append(files, file) } return files, nil } From e3e790f4614fbb9865a7614bc698de00fe2302e3 Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Tue, 16 Jan 2024 15:59:44 +0800 Subject: [PATCH 111/659] feat(115): add QR code source selection (#5891) * feat(115): add QR code source selection closed #5386 * feat(115_share): add QR code source selection --- drivers/115/meta.go | 9 +++-- drivers/115/util.go | 2 +- drivers/115_share/meta.go | 13 ++++--- drivers/115_share/utils.go | 2 +- go.mod | 2 +- go.sum | 79 +------------------------------------- 6 files changed, 17 insertions(+), 90 deletions(-) diff --git a/drivers/115/meta.go b/drivers/115/meta.go index d3e937bf8e7..2afc57a78c9 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -6,10 +6,11 @@ import ( ) type Addition struct { - Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` - QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` - PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` - LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` + Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` + QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` + QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,linux,mac,windows,tv" default:"linux" help:"select the QR code device, default linux"` + PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` + LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` driver.RootID } diff --git a/drivers/115/util.go b/drivers/115/util.go index fb425c8eef7..cb28fff43fe 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -42,7 +42,7 @@ func (d *Pan115) login() error { s := &driver115.QRCodeSession{ UID: d.Addition.QRCodeToken, } - if cr, err = d.client.QRCodeLogin(s); err != nil { + if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { return errors.Wrap(err, "failed to login by qrcode") } d.Addition.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s", cr.UID, cr.CID, cr.SEID) diff --git a/drivers/115_share/meta.go b/drivers/115_share/meta.go index b7f060e3d9f..90dd7d8f170 100644 --- a/drivers/115_share/meta.go +++ b/drivers/115_share/meta.go @@ -6,12 +6,13 @@ import ( ) type Addition struct { - Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` - QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` - PageSize int64 `json:"page_size" type:"number" default:"20" help:"list api per page size of 115 driver"` - LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` - ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"` - ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"` + Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` + QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` + QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,linux,mac,windows,tv" default:"linux" help:"select the QR code device, default linux"` + PageSize int64 `json:"page_size" type:"number" default:"20" help:"list api per page size of 115 driver"` + LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` + ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"` + ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"` driver.RootID } diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 42567c0eede..812352ef42a 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -93,7 +93,7 @@ func (d *Pan115Share) login() error { s := &driver115.QRCodeSession{ UID: d.QRCodeToken, } - if cr, err = d.client.QRCodeLogin(s); err != nil { + if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { return errors.Wrap(err, "failed to login by qrcode") } d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s", cr.UID, cr.CID, cr.SEID) diff --git a/go.mod b/go.mod index 389a1bac8f3..e170902c776 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/alist-org/alist/v3 go 1.21 require ( - github.com/SheltonZhu/115driver v1.0.21 + github.com/SheltonZhu/115driver v1.0.22 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/Xhofe/wopan-sdk-go v0.1.2 diff --git a/go.sum b/go.sum index c226f7eed9b..0a30e441856 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= -github.com/SheltonZhu/115driver v1.0.21 h1:Pz6r14VwIiuSyHj+OmJe57FHhbmWB/6IfnXAFL2iXbU= -github.com/SheltonZhu/115driver v1.0.21/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= +github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U= +github.com/SheltonZhu/115driver v1.0.22/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= @@ -23,10 +23,6 @@ github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0E github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= -github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= -github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible h1:ROMcuN61gI8SfQ+AEMh4d7GZ3gwTZLIhPjtd05TQCG4= -github.com/aliyun/aliyun-oss-go-sdk v2.2.10+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= @@ -34,12 +30,6 @@ github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.46.7 h1:IjvAWeiJZlbETOemOwvheN5L17CvKvKW0T1xOC6d3Sc= -github.com/aws/aws-sdk-go v1.46.7/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y= -github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go v1.49.15 h1:aH9bSV4kL4ziH0AMtuYbukGIVebXddXBL0cKZ1zj15k= -github.com/aws/aws-sdk-go v1.49.15/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.49.18 h1:g/iMXkfXeJQ7MvnLwroxWsTTNkHtdVJGxIgrAIEG62M= github.com/aws/aws-sdk-go v1.49.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -89,8 +79,6 @@ github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc= github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= @@ -98,12 +86,8 @@ github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= -github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= github.com/charmbracelet/bubbles v0.17.1 h1:0SIyjOnkrsfDo88YvPgAWvZMwXe26TP6drRvmkjyUu4= github.com/charmbracelet/bubbles v0.17.1/go.mod h1:9HxZWlkCqz2PRwsCbYl7a3KXvGzFaDHpYbSYMJ+nE3o= -github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= -github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= @@ -111,7 +95,6 @@ github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= @@ -123,7 +106,6 @@ github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjs github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= @@ -131,8 +113,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= -github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= @@ -141,8 +121,6 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= -github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU= -github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= @@ -162,8 +140,6 @@ github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKe github.com/gaoyb7/115drive-webdav v0.1.8/go.mod h1:BKbeY6j8SKs3+rzBFFALznGxbPmefEm3vA+dGhqgOGU= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= -github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= -github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= github.com/gin-contrib/cors v1.5.0 h1:DgGKV7DDoOn36DFkNtbHrjoRiT5ExCe+PC9/xp7aKvk= github.com/gin-contrib/cors v1.5.0/go.mod h1:TvU7MAZ3EwrPLI2ztzTt3tqgvBCq+wn8WpZmfADjupI= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -187,23 +163,15 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24= github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= -github.com/go-resty/resty/v2 v2.10.0 h1:Qla4W/+TMmv0fOeeRqzEpXPLfTUnR5HZ1+lGs+CkiCo= -github.com/go-resty/resty/v2 v2.10.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-webauthn/webauthn v0.9.4 h1:YxvHSqgUyc5AK2pZbqkWWR55qKeDPhP8zLDr6lpIc2g= -github.com/go-webauthn/webauthn v0.9.4/go.mod h1:LqupCtzSef38FcxzaklmOn7AykGKhAhr9xlRbdbgnTw= github.com/go-webauthn/webauthn v0.10.0 h1:yuW2e1tXnRAwAvKrR4q4LQmc6XtCMH639/ypZGhZCwk= github.com/go-webauthn/webauthn v0.10.0/go.mod h1:l0NiauXhL6usIKqNLCUM3Qir43GK7ORg8ggold0Uv/Y= -github.com/go-webauthn/x v0.1.5 h1:V2TCzDU2TGLd0kSZOXdrqDVV5JB9ILnKxA9S53CSBw0= -github.com/go-webauthn/x v0.1.5/go.mod h1:qbzWwcFcv4rTwtCLOZd+icnr6B7oSsAGZJqlt8cukqY= github.com/go-webauthn/x v0.1.6 h1:QNAX+AWeqRt9loE8mULeWJCqhVG5D/jvdmJ47fIWCkQ= github.com/go-webauthn/x v0.1.6/go.mod h1:W8dFVZ79o4f+nY1eOUICy/uq5dhrRl7mxQkYhXTo0FA= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -219,7 +187,6 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgR github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= @@ -236,16 +203,12 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -321,8 +284,6 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= -github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= -github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -389,8 +350,6 @@ github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= @@ -449,8 +408,6 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:s github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -469,7 +426,6 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4UvMT7OmKhH9sOIvS8iXkxfPdnn9wQA= @@ -510,8 +466,6 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -525,31 +479,16 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/exp v0.0.0-20231226003508-02704c960a9b h1:kLiC65FbiHWFAOu+lxwNPujcsl8VYyTYYEZnsOO1WK4= -golang.org/x/exp v0.0.0-20231226003508-02704c960a9b/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= -golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= -golang.org/x/image v0.14.0 h1:tNgSxAFe3jC4uYqvZdTr84SZoM1KfwdC9SKIFrLjFn4= -golang.org/x/image v0.14.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -560,14 +499,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -601,8 +534,6 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -611,8 +542,6 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -624,13 +553,11 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -643,8 +570,6 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= From ce06f394f1da5af4aa7a51fa353d2b0e50a1b9ed Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 17 Jan 2024 14:15:34 +0800 Subject: [PATCH 112/659] fix: missing salt of guest user (close #5737) --- internal/bootstrap/data/user.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 9ac62fe841f..451c60a327f 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -48,6 +48,7 @@ func initUser() { guest = &model.User{ Username: "guest", PwdHash: model.TwoHashPwd("guest", salt), + Salt: salt, Role: model.GUEST, BasePath: "/", Permission: 0, From 442c2f77ea7607e23848d654be952049acf1825b Mon Sep 17 00:00:00 2001 From: Echo Response <32877980+EchoResponse@users.noreply.github.com> Date: Fri, 19 Jan 2024 10:59:56 +0800 Subject: [PATCH 113/659] feat: add `quqi` driver (#5899 close #5251) * feat: add `quqi` driver * change signature of request function * specific header for every storage * todo: real upload * fix upload method * fix incorrect parameters for some request function calls --------- Co-authored-by: Andy Hsu --- drivers/all.go | 1 + drivers/quqi/driver.go | 367 +++++++++++++++++++++++++++++++++++++++++ drivers/quqi/meta.go | 27 +++ drivers/quqi/types.go | 167 +++++++++++++++++++ drivers/quqi/util.go | 92 +++++++++++ go.mod | 4 + go.sum | 16 ++ 7 files changed, 674 insertions(+) create mode 100644 drivers/quqi/driver.go create mode 100644 drivers/quqi/meta.go create mode 100644 drivers/quqi/types.go create mode 100644 drivers/quqi/util.go diff --git a/drivers/all.go b/drivers/all.go index 599820c296c..08d8f1cbd42 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -37,6 +37,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" _ "github.com/alist-org/alist/v3/drivers/quark_uc" + _ "github.com/alist-org/alist/v3/drivers/quqi" _ "github.com/alist-org/alist/v3/drivers/s3" _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go new file mode 100644 index 00000000000..0a3d347aa04 --- /dev/null +++ b/drivers/quqi/driver.go @@ -0,0 +1,367 @@ +package quqi + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/tencentyun/cos-go-sdk-v5" +) + +type Quqi struct { + model.Storage + Addition + GroupID string +} + +func (d *Quqi) Config() driver.Config { + return config +} + +func (d *Quqi) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Quqi) Init(ctx context.Context) error { + // 登录 + if err := d.login(); err != nil { + return err + } + + // (暂时仅获取私人云) 获取私人云ID + groupResp := &GroupRes{} + if _, err := d.request("group.quqi.com", "/v1/group/list", resty.MethodGet, nil, groupResp); err != nil { + return err + } + for _, groupInfo := range groupResp.Data { + if groupInfo == nil { + continue + } + if groupInfo.Type == 2 { + d.GroupID = strconv.Itoa(groupInfo.ID) + break + } + } + if d.GroupID == "" { + return errs.StorageNotFound + } + + return nil +} + +func (d *Quqi) Drop(ctx context.Context) error { + return nil +} + +func (d *Quqi) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var ( + listResp = &ListRes{} + files []model.Obj + ) + + if _, err := d.request("", "/api/dir/ls", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "node_id": dir.GetID(), + }) + }, listResp); err != nil { + return nil, err + } + + if listResp.Data == nil { + return nil, nil + } + + // dirs + for _, dirInfo := range listResp.Data.Dir { + if dirInfo == nil { + continue + } + files = append(files, &model.Object{ + ID: strconv.FormatInt(dirInfo.NodeID, 10), + Name: dirInfo.Name, + Modified: time.Unix(dirInfo.UpdateTime, 0), + Ctime: time.Unix(dirInfo.AddTime, 0), + IsFolder: true, + }) + } + + // files + for _, fileInfo := range listResp.Data.File { + if fileInfo == nil { + continue + } + if fileInfo.EXT != "" { + fileInfo.Name = strings.Join([]string{fileInfo.Name, fileInfo.EXT}, ".") + } + + files = append(files, &model.Object{ + ID: strconv.FormatInt(fileInfo.NodeID, 10), + Name: fileInfo.Name, + Size: fileInfo.Size, + Modified: time.Unix(fileInfo.UpdateTime, 0), + Ctime: time.Unix(fileInfo.AddTime, 0), + }) + } + + return files, nil +} + +func (d *Quqi) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var getDocResp = &GetDocRes{} + + if _, err := d.request("", "/api/doc/getDoc", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "node_id": file.GetID(), + }) + }, getDocResp); err != nil { + return nil, err + } + + return &model.Link{ + URL: getDocResp.Data.OriginPath, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Quqi) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var ( + makeDirRes = &MakeDirRes{} + timeNow = time.Now() + ) + + if _, err := d.request("", "/api/dir/mkDir", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "parent_id": parentDir.GetID(), + "name": dirName, + }) + }, makeDirRes); err != nil { + return nil, err + } + + return &model.Object{ + ID: strconv.FormatInt(makeDirRes.Data.NodeID, 10), + Name: dirName, + Modified: timeNow, + Ctime: timeNow, + IsFolder: true, + }, nil +} + +func (d *Quqi) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var moveRes = &MoveRes{} + + if _, err := d.request("", "/api/dir/mvDir", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "node_id": dstDir.GetID(), + "source_quqi_id": d.GroupID, + "source_node_id": srcObj.GetID(), + }) + }, moveRes); err != nil { + return nil, err + } + + return &model.Object{ + ID: strconv.FormatInt(moveRes.Data.NodeID, 10), + Name: moveRes.Data.NodeName, + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Quqi) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var renameRes = &RenameRes{} + + if _, err := d.request("", "/api/dir/renameDir", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "node_id": srcObj.GetID(), + "rename": newName, + }) + }, renameRes); err != nil { + return nil, err + } + + return &model.Object{ + ID: strconv.FormatInt(renameRes.Data.NodeID, 10), + Name: renameRes.Data.Rename, + Size: srcObj.GetSize(), + Modified: time.Unix(renameRes.Data.UpdateTime, 0), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Quqi) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // 无法从曲奇接口响应中直接获取复制后的文件信息 + if _, err := d.request("", "/api/node/copy", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "node_id": dstDir.GetID(), + "source_quqi_id": d.GroupID, + "source_node_id": srcObj.GetID(), + }) + }, nil); err != nil { + return nil, err + } + + return nil, nil +} + +func (d *Quqi) Remove(ctx context.Context, obj model.Obj) error { + // 暂时不做直接删除，默认都放到回收站。直接删除方法：先调用删除接口放入回收站，在通过回收站接口删除文件 + if _, err := d.request("", "/api/node/del", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "node_id": obj.GetID(), + }) + }, nil); err != nil { + return err + } + + return nil +} + +func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // base info + sizeStr := strconv.FormatInt(stream.GetSize(), 10) + f, err := stream.CacheFullInTempFile() + if err != nil { + return nil, err + } + md5, err := utils.HashFile(utils.MD5, f) + if err != nil { + return nil, err + } + sha, err := utils.HashFile(utils.SHA256, f) + if err != nil { + return nil, err + } + // init upload + var uploadInitResp UploadInitResp + _, err = d.request("", "/api/upload/v1/file/init", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "parent_id": dstDir.GetID(), + "size": sizeStr, + "file_name": stream.GetName(), + "md5": md5, + "sha": sha, + "is_slice": "true", + "client_id": "quqipc_F8X2qOlSfF", + }) + }, &uploadInitResp) + if err != nil { + return nil, err + } + // listParts + _, err = d.request("upload.quqi.com:20807", "/upload/v1/listParts", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "token": uploadInitResp.Data.Token, + "task_id": uploadInitResp.Data.TaskID, + "client_id": "quqipc_F8X2qOlSfF", + }) + }, nil) + if err != nil { + return nil, err + } + // get temp key + var tempKeyResp TempKeyResp + _, err = d.request("upload.quqi.com:20807", "/upload/v1/tempKey", resty.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "token": uploadInitResp.Data.Token, + "task_id": uploadInitResp.Data.TaskID, + }) + }, &tempKeyResp) + if err != nil { + return nil, err + } + // upload + u, err := url.Parse(fmt.Sprintf("https://%s.cos.ap-shanghai.myqcloud.com", uploadInitResp.Data.Bucket)) + b := &cos.BaseURL{BucketURL: u} + client := cos.NewClient(b, &http.Client{ + Transport: &cos.CredentialTransport{ + Credential: cos.NewTokenCredential(tempKeyResp.Data.Credentials.TmpSecretID, tempKeyResp.Data.Credentials.TmpSecretKey, tempKeyResp.Data.Credentials.SessionToken), + }, + }) + partSize := int64(1024 * 1024 * 2) + partCount := (stream.GetSize() + partSize - 1) / partSize + for i := 1; i <= int(partCount); i++ { + length := partSize + if i == int(partCount) { + length = stream.GetSize() - (int64(i)-1)*partSize + } + _, err := client.Object.UploadPart( + context.Background(), uploadInitResp.Data.Key, uploadInitResp.Data.UploadID, i, io.LimitReader(f, partSize), &cos.ObjectUploadPartOptions{ + ContentLength: length, + }, + ) + if err != nil { + return nil, err + } + } + //cfg := &aws.Config{ + // Credentials: credentials.NewStaticCredentials(tempKeyResp.Data.Credentials.TmpSecretID, tempKeyResp.Data.Credentials.TmpSecretKey, tempKeyResp.Data.Credentials.SessionToken), + // Region: aws.String("shanghai"), + // Endpoint: aws.String("cos.ap-shanghai.myqcloud.com"), + // // S3ForcePathStyle: aws.Bool(true), + //} + //s, err := session.NewSession(cfg) + //if err != nil { + // return nil, err + //} + //uploader := s3manager.NewUploader(s) + //input := &s3manager.UploadInput{ + // Bucket: &uploadInitResp.Data.Bucket, + // Key: &uploadInitResp.Data.Key, + // Body: f, + //} + //_, err = uploader.UploadWithContext(ctx, input) + //if err != nil { + // return nil, err + //} + // finish upload + var uploadFinishResp UploadFinishResp + _, err = d.request("", "/api/upload/v1/file/finish", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "token": uploadInitResp.Data.Token, + "task_id": uploadInitResp.Data.TaskID, + "client_id": "quqipc_F8X2qOlSfF", + }) + }, &uploadFinishResp) + if err != nil { + return nil, err + } + return &model.Object{ + ID: strconv.FormatInt(uploadFinishResp.Data.NodeID, 10), + Name: uploadFinishResp.Data.NodeName, + Size: stream.GetSize(), + Modified: stream.ModTime(), + Ctime: stream.CreateTime(), + }, nil +} + +//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Quqi)(nil) diff --git a/drivers/quqi/meta.go b/drivers/quqi/meta.go new file mode 100644 index 00000000000..0820796e749 --- /dev/null +++ b/drivers/quqi/meta.go @@ -0,0 +1,27 @@ +package quqi + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Phone string `json:"phone"` + Password string `json:"password"` + Cookie string `json:"cookie" help:"Cookie can be used on multiple clients at the same time"` +} + +var config = driver.Config{ + Name: "Quqi", + OnlyLocal: true, + LocalSort: true, + //NoUpload: true, + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Quqi{} + }) +} diff --git a/drivers/quqi/types.go b/drivers/quqi/types.go new file mode 100644 index 00000000000..3f6c4ebe2dd --- /dev/null +++ b/drivers/quqi/types.go @@ -0,0 +1,167 @@ +package quqi + +type BaseReqQuery struct { + ID string `json:"quqiid"` +} + +type BaseReq struct { + GroupID string `json:"quqi_id"` +} + +type BaseRes struct { + //Data interface{} `json:"data"` + Code int `json:"err"` + Message string `json:"msg"` +} + +type GroupRes struct { + BaseRes + Data []*Group `json:"data"` +} + +type ListRes struct { + BaseRes + Data *List `json:"data"` +} + +type GetDocRes struct { + BaseRes + Data struct { + OriginPath string `json:"origin_path"` + } `json:"data"` +} + +type MakeDirRes struct { + BaseRes + Data struct { + IsRoot bool `json:"is_root"` + NodeID int64 `json:"node_id"` + ParentID int64 `json:"parent_id"` + } `json:"data"` +} + +type MoveRes struct { + BaseRes + Data struct { + NodeChildNum int64 `json:"node_child_num"` + NodeID int64 `json:"node_id"` + NodeName string `json:"node_name"` + ParentID int64 `json:"parent_id"` + GroupID int64 `json:"quqi_id"` + TreeID int64 `json:"tree_id"` + } `json:"data"` +} + +type RenameRes struct { + BaseRes + Data struct { + NodeID int64 `json:"node_id"` + GroupID int64 `json:"quqi_id"` + Rename string `json:"rename"` + TreeID int64 `json:"tree_id"` + UpdateTime int64 `json:"updatetime"` + } `json:"data"` +} + +type CopyRes struct { + BaseRes +} + +type RemoveRes struct { + BaseRes +} + +type Group struct { + ID int `json:"quqi_id"` + Type int `json:"type"` + Name string `json:"name"` + IsAdministrator int `json:"is_administrator"` + Role int `json:"role"` + Avatar string `json:"avatar_url"` + IsStick int `json:"is_stick"` + Nickname string `json:"nickname"` + Status int `json:"status"` +} + +type List struct { + ListDir + Dir []*ListDir `json:"dir"` + File []*ListFile `json:"file"` +} + +type ListItem struct { + AddTime int64 `json:"add_time"` + IsDir int `json:"is_dir"` + IsExpand int `json:"is_expand"` + IsFinalize int `json:"is_finalize"` + LastEditorName string `json:"last_editor_name"` + Name string `json:"name"` + NodeID int64 `json:"nid"` + ParentID int64 `json:"parent_id"` + Permission int `json:"permission"` + TreeID int64 `json:"tid"` + UpdateCNT int64 `json:"update_cnt"` + UpdateTime int64 `json:"update_time"` +} + +type ListDir struct { + ListItem + ChildDocNum int64 `json:"child_doc_num"` + DirDetail string `json:"dir_detail"` + DirType int `json:"dir_type"` +} + +type ListFile struct { + ListItem + BroadDocType string `json:"broad_doc_type"` + CanDisplay bool `json:"can_display"` + Detail string `json:"detail"` + EXT string `json:"ext"` + Filetype string `json:"filetype"` + HasMobileThumbnail bool `json:"has_mobile_thumbnail"` + HasThumbnail bool `json:"has_thumbnail"` + Size int64 `json:"size"` + Version int `json:"version"` +} + +type UploadInitResp struct { + Data struct { + Bucket string `json:"bucket"` + Exist bool `json:"exist"` + Key string `json:"key"` + TaskID string `json:"task_id"` + Token string `json:"token"` + UploadID string `json:"upload_id"` + URL string `json:"url"` + } `json:"data"` + Err int `json:"err"` + Msg string `json:"msg"` +} + +type TempKeyResp struct { + Err int `json:"err"` + Msg string `json:"msg"` + Data struct { + ExpiredTime int `json:"expiredTime"` + Expiration string `json:"expiration"` + Credentials struct { + SessionToken string `json:"sessionToken"` + TmpSecretID string `json:"tmpSecretId"` + TmpSecretKey string `json:"tmpSecretKey"` + } `json:"credentials"` + RequestID string `json:"requestId"` + StartTime int `json:"startTime"` + } `json:"data"` +} + +type UploadFinishResp struct { + Data struct { + NodeID int64 `json:"node_id"` + NodeName string `json:"node_name"` + ParentID int64 `json:"parent_id"` + QuqiID int64 `json:"quqi_id"` + TreeID int64 `json:"tree_id"` + } `json:"data"` + Err int `json:"err"` + Msg string `json:"msg"` +} diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go new file mode 100644 index 00000000000..d6a5642aa47 --- /dev/null +++ b/drivers/quqi/util.go @@ -0,0 +1,92 @@ +package quqi + +import ( + "encoding/base64" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +// do others that not defined in Driver interface +func (d *Quqi) request(host string, path string, method string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { + var ( + reqUrl = url.URL{ + Scheme: "https", + Host: "quqi.com", + Path: path, + } + req = base.RestyClient.R() + result BaseRes + ) + + if host != "" { + reqUrl.Host = host + } + req.SetHeaders(map[string]string{ + "Origin": "https://quqi.com", + "Cookie": d.Cookie, + }).SetResult(&result) + + if d.GroupID != "" { + req.SetQueryParam("quqiid", d.GroupID) + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, reqUrl.String()) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, errors.New(result.Message) + } + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + return nil, err + } + } + return res, nil +} + +func (d *Quqi) login() error { + if d.Cookie != "" && d.checkLogin() { + return nil + } + + if d.Phone == "" || d.Password == "" { + return errors.New("empty phone number or password") + } + + resp, err := d.request("", "/auth/person/v2/login/password", resty.MethodPost, func(req *resty.Request){ + req.SetFormData(map[string]string{ + "phone": d.Phone, + "password": base64.StdEncoding.EncodeToString([]byte(d.Password)), + }) + }, nil) + if err != nil { + return err + } + + var cookies []string + for _, cookie := range resp.RawResponse.Cookies() { + cookies = append(cookies, fmt.Sprintf("%s=%s", cookie.Name, cookie.Value)) + } + d.Cookie = strings.Join(cookies, ";") + + return nil +} + +func (d *Quqi) checkLogin() bool { + if _, err := d.request("", "/auth/account/baseInfo", resty.MethodGet, nil, nil); err != nil { + return false + } + return true +} diff --git a/go.mod b/go.mod index e170902c776..efe1cade1e8 100644 --- a/go.mod +++ b/go.mod @@ -98,6 +98,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect + github.com/clbanning/mxj v1.8.4 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect @@ -119,6 +120,7 @@ require ( github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-tpm v0.9.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -152,6 +154,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mozillazg/go-httpheader v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect @@ -183,6 +186,7 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tencentyun/cos-go-sdk-v5 v0.7.45 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 0a30e441856..02bb391dc10 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U= @@ -100,6 +101,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= @@ -119,6 +122,7 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= @@ -193,10 +197,14 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -305,6 +313,7 @@ github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -314,6 +323,9 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w= +github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= @@ -430,6 +442,10 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4UvMT7OmKhH9sOIvS8iXkxfPdnn9wQA= github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= +github.com/tencentyun/cos-go-sdk-v5 v0.7.45 h1:5/ZGOv846tP6+2X7w//8QjLgH2KcUK+HciFbfjWquFU= +github.com/tencentyun/cos-go-sdk-v5 v0.7.45/go.mod h1:DH9US8nB+AJXqwu/AMOrCFN1COv3dpytXuJWHgdg7kE= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= From 0f29a811bfa608cd99642c083538852ed7f923d1 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 19 Jan 2024 12:05:10 +0800 Subject: [PATCH 114/659] fix: s3 upload exceeded total allowed configured MaxUploadParts (close #5909) --- drivers/123/driver.go | 10 +++++++--- drivers/mediatrack/driver.go | 3 +++ drivers/pikpak/driver.go | 3 +++ drivers/quqi/driver.go | 2 +- drivers/teambition/util.go | 3 +++ drivers/thunder/driver.go | 6 +++++- drivers/vtencent/util.go | 3 +++ 7 files changed, 25 insertions(+), 5 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 6f7fec1bd43..1f1ae85886a 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,6 +6,10 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "io" + "net/http" + "net/url" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -17,9 +21,6 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" - "io" - "net/http" - "net/url" ) type Pan123 struct { @@ -232,6 +233,9 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } uploader := s3manager.NewUploader(s) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Key, diff --git a/drivers/mediatrack/driver.go b/drivers/mediatrack/driver.go index 90e66ae0e34..ef571832eb7 100644 --- a/drivers/mediatrack/driver.go +++ b/drivers/mediatrack/driver.go @@ -188,6 +188,9 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil _ = tempFile.Close() }() uploader := s3manager.NewUploader(s) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Object, diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 52ca15c798a..f3676b8291b 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -172,6 +172,9 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } uploader := s3manager.NewUploader(ss) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } input := &s3manager.UploadInput{ Bucket: ¶ms.Bucket, Key: ¶ms.Key, diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 0a3d347aa04..508c8b6b32e 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -311,7 +311,7 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea length = stream.GetSize() - (int64(i)-1)*partSize } _, err := client.Object.UploadPart( - context.Background(), uploadInitResp.Data.Key, uploadInitResp.Data.UploadID, i, io.LimitReader(f, partSize), &cos.ObjectUploadPartOptions{ + ctx, uploadInitResp.Data.Key, uploadInitResp.Data.UploadID, i, io.LimitReader(f, partSize), &cos.ObjectUploadPartOptions{ ContentLength: length, }, ) diff --git a/drivers/teambition/util.go b/drivers/teambition/util.go index 79de7007c78..181cc58f64c 100644 --- a/drivers/teambition/util.go +++ b/drivers/teambition/util.go @@ -244,6 +244,9 @@ func (d *Teambition) newUpload(ctx context.Context, dstDir model.Obj, stream mod return err } uploader := s3manager.NewUploader(ss) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } input := &s3manager.UploadInput{ Bucket: &uploadToken.Upload.Bucket, Key: &uploadToken.Upload.Key, diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index cac6733f01b..9ba5dd825f7 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -373,7 +373,11 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, stream model. if err != nil { return err } - _, err = s3manager.NewUploader(s).UploadWithContext(ctx, &s3manager.UploadInput{ + uploader := s3manager.NewUploader(s) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } + _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), diff --git a/drivers/vtencent/util.go b/drivers/vtencent/util.go index bf260415e0c..ba87f1abe51 100644 --- a/drivers/vtencent/util.go +++ b/drivers/vtencent/util.go @@ -272,6 +272,9 @@ func (d *Vtencent) FileUpload(ctx context.Context, dstDir model.Obj, stream mode return err } uploader := s3manager.NewUploader(ss) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } input := &s3manager.UploadInput{ Bucket: aws.String(fmt.Sprintf("%s-%d", params.StorageBucket, params.StorageAppID)), Key: ¶ms.Video.StoragePath, From 8bccb69e8d9ca8d602cc6bd11980fc1003751cac Mon Sep 17 00:00:00 2001 From: None <30817148+Xiefengshang@users.noreply.github.com> Date: Fri, 19 Jan 2024 13:02:05 +0800 Subject: [PATCH 115/659] fix(google_photo): add support for streaming video, range requests (#5905) * Update util.go Return mediaMetadata * Update driver.go Using width and height --- drivers/google_photo/driver.go | 30 +++++++++++++++++++++++++++--- drivers/google_photo/util.go | 2 +- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/drivers/google_photo/driver.go b/drivers/google_photo/driver.go index b54132ef9ed..85a0520b4a5 100644 --- a/drivers/google_photo/driver.go +++ b/drivers/google_photo/driver.go @@ -58,9 +58,33 @@ func (d *GooglePhoto) Link(ctx context.Context, file model.Obj, args model.LinkA URL: f.BaseURL + "=d", }, nil } else if strings.Contains(f.MimeType, "video/") { - return &model.Link{ - URL: f.BaseURL + "=dv", - }, nil + var width, height int + + fmt.Sscanf(f.MediaMetadata.Width, "%d", &width) + fmt.Sscanf(f.MediaMetadata.Height, "%d", &height) + + switch { + // 1080P + case width == 1920 && height == 1080: + return &model.Link{ + URL: f.BaseURL + "=m37", + }, nil + // 720P + case width == 1280 && height == 720: + return &model.Link{ + URL: f.BaseURL + "=m22", + }, nil + // 360P + case width == 640 && height == 360: + return &model.Link{ + URL: f.BaseURL + "=m18", + }, nil + default: + return &model.Link{ + URL: f.BaseURL + "=dv", + }, nil + } + } return &model.Link{}, nil } diff --git a/drivers/google_photo/util.go b/drivers/google_photo/util.go index fbbff9ab1cb..0fd271b9bb4 100644 --- a/drivers/google_photo/util.go +++ b/drivers/google_photo/util.go @@ -151,7 +151,7 @@ func (d *GooglePhoto) getMedia(id string) (MediaItem, error) { var resp MediaItem query := map[string]string{ - "fields": "baseUrl,mimeType", + "fields": "mediaMetadata,baseUrl,mimeType", } _, err := d.request(fmt.Sprintf("https://photoslibrary.googleapis.com/v1/mediaItems/%s", id), http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) From a8c900d09ed3df95f0474507106e179c85cdb308 Mon Sep 17 00:00:00 2001 From: Echo Response <32877980+EchoResponse@users.noreply.github.com> Date: Fri, 19 Jan 2024 13:57:31 +0800 Subject: [PATCH 116/659] fix(quqi): file extension duplication when rename and some missing form parameters (#5910) * feat: add `quqi` driver * change signature of request function * specific header for every storage * todo: real upload * fix upload method * fix incorrect parameters for some request function calls * refine some form parameters to avoid potential problems * fix file extension duplication in rename function * improve the error message in login function --------- Co-authored-by: Andy Hsu --- drivers/quqi/driver.go | 71 +++++++++++++++++++++++++++++++----------- drivers/quqi/util.go | 13 ++++++-- 2 files changed, 62 insertions(+), 22 deletions(-) diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 508c8b6b32e..9bc23f8b837 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -14,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" "github.com/go-resty/resty/v2" "github.com/tencentyun/cos-go-sdk-v5" ) @@ -21,7 +22,8 @@ import ( type Quqi struct { model.Storage Addition - GroupID string + GroupID string // 私人云群组ID + ClientID string // 随机生成客户端ID 经过测试，部分接口调用若不携带client id会出现错误 } func (d *Quqi) Config() driver.Config { @@ -38,7 +40,10 @@ func (d *Quqi) Init(ctx context.Context) error { return err } - // (暂时仅获取私人云) 获取私人云ID + // 生成随机client id (与网页端生成逻辑一致) + d.ClientID = "quqipc_" + random.String(10) + + // 获取私人云ID (暂时仅获取私人云) groupResp := &GroupRes{} if _, err := d.request("group.quqi.com", "/v1/group/list", resty.MethodGet, nil, groupResp); err != nil { return err @@ -71,8 +76,10 @@ func (d *Quqi) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] if _, err := d.request("", "/api/dir/ls", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ - "quqi_id": d.GroupID, - "node_id": dir.GetID(), + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": dir.GetID(), + "client_id": d.ClientID, }) }, listResp); err != nil { return nil, err @@ -122,8 +129,10 @@ func (d *Quqi) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* if _, err := d.request("", "/api/doc/getDoc", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ - "quqi_id": d.GroupID, - "node_id": file.GetID(), + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": file.GetID(), + "client_id": d.ClientID, }) }, getDocResp); err != nil { return nil, err @@ -147,8 +156,10 @@ func (d *Quqi) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) if _, err := d.request("", "/api/dir/mkDir", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ "quqi_id": d.GroupID, + "tree_id": "1", "parent_id": parentDir.GetID(), "name": dirName, + "client_id": d.ClientID, }) }, makeDirRes); err != nil { return nil, err @@ -169,9 +180,12 @@ func (d *Quqi) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e if _, err := d.request("", "/api/dir/mvDir", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ "quqi_id": d.GroupID, + "tree_id": "1", "node_id": dstDir.GetID(), "source_quqi_id": d.GroupID, + "source_tree_id": "1", "source_node_id": srcObj.GetID(), + "client_id": d.ClientID, }) }, moveRes); err != nil { return nil, err @@ -188,23 +202,37 @@ func (d *Quqi) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e } func (d *Quqi) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { - var renameRes = &RenameRes{} + var realName = newName + + if !srcObj.IsDir() { + srcExt, newExt := utils.Ext(srcObj.GetName()), utils.Ext(newName) + + // 曲奇网盘的文件名称由文件名和扩展名组成，若存在扩展名，则重命名时仅支持更改文件名，扩展名在曲奇服务端保留 + if srcExt != "" && srcExt == newExt { + parts := strings.Split(newName, ".") + if len(parts) > 1 { + realName = strings.Join(parts[:len(parts)-1], ".") + } + } + } if _, err := d.request("", "/api/dir/renameDir", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ - "quqi_id": d.GroupID, - "node_id": srcObj.GetID(), - "rename": newName, + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": srcObj.GetID(), + "rename": realName, + "client_id": d.ClientID, }) - }, renameRes); err != nil { + }, nil); err != nil { return nil, err } return &model.Object{ - ID: strconv.FormatInt(renameRes.Data.NodeID, 10), - Name: renameRes.Data.Rename, + ID: srcObj.GetID(), + Name: newName, Size: srcObj.GetSize(), - Modified: time.Unix(renameRes.Data.UpdateTime, 0), + Modified: time.Now(), Ctime: srcObj.CreateTime(), IsFolder: srcObj.IsDir(), }, nil @@ -215,9 +243,12 @@ func (d *Quqi) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e if _, err := d.request("", "/api/node/copy", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ "quqi_id": d.GroupID, + "tree_id": "1", "node_id": dstDir.GetID(), "source_quqi_id": d.GroupID, + "source_tree_id": "1", "source_node_id": srcObj.GetID(), + "client_id": d.ClientID, }) }, nil); err != nil { return nil, err @@ -230,8 +261,10 @@ func (d *Quqi) Remove(ctx context.Context, obj model.Obj) error { // 暂时不做直接删除，默认都放到回收站。直接删除方法：先调用删除接口放入回收站，在通过回收站接口删除文件 if _, err := d.request("", "/api/node/del", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ - "quqi_id": d.GroupID, - "node_id": obj.GetID(), + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": obj.GetID(), + "client_id": d.ClientID, }) }, nil); err != nil { return err @@ -267,7 +300,7 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea "md5": md5, "sha": sha, "is_slice": "true", - "client_id": "quqipc_F8X2qOlSfF", + "client_id": d.ClientID, }) }, &uploadInitResp) if err != nil { @@ -278,7 +311,7 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea req.SetFormData(map[string]string{ "token": uploadInitResp.Data.Token, "task_id": uploadInitResp.Data.TaskID, - "client_id": "quqipc_F8X2qOlSfF", + "client_id": d.ClientID, }) }, nil) if err != nil { @@ -345,7 +378,7 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea req.SetFormData(map[string]string{ "token": uploadInitResp.Data.Token, "task_id": uploadInitResp.Data.TaskID, - "client_id": "quqipc_F8X2qOlSfF", + "client_id": d.ClientID, }) }, &uploadFinishResp) if err != nil { diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go index d6a5642aa47..91fc0a02d7c 100644 --- a/drivers/quqi/util.go +++ b/drivers/quqi/util.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -61,11 +62,17 @@ func (d *Quqi) login() error { return nil } - if d.Phone == "" || d.Password == "" { - return errors.New("empty phone number or password") + if d.Cookie != "" { + return errors.New("cookie is invalid") + } + if d.Phone == "" { + return errors.New("phone number is empty") + } + if d.Password == "" { + return errs.EmptyPassword } - resp, err := d.request("", "/auth/person/v2/login/password", resty.MethodPost, func(req *resty.Request){ + resp, err := d.request("", "/auth/person/v2/login/password", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ "phone": d.Phone, "password": base64.StdEncoding.EncodeToString([]byte(d.Password)), From 4f7761fe2c104f20e7bf57a768f13963ba417d49 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 20 Jan 2024 13:06:46 +0800 Subject: [PATCH 117/659] fix: set progress to 100 when it's NaN (close #5906) --- internal/offline_download/aria2/aria2.go | 3 ++- internal/offline_download/tool/download.go | 3 ++- server/handles/task.go | 9 ++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index ea6404a6229..d22b32f9d55 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -3,10 +3,11 @@ package aria2 import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/errs" "strconv" "time" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 975530e7f6a..f0a5d5d4376 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -2,13 +2,14 @@ package tool import ( "fmt" + "time" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/setting" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" - "time" ) type DownloadTask struct { diff --git a/server/handles/task.go b/server/handles/task.go index 9c9486b9de2..a8b4d21b2b9 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -1,6 +1,8 @@ package handles import ( + "math" + "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/pkg/utils" @@ -23,12 +25,17 @@ func getTaskInfo[T tache.TaskWithInfo](task T) TaskInfo { if task.GetErr() != nil { errMsg = task.GetErr().Error() } + progress := task.GetProgress() + // if progress is NaN, set it to 100 + if math.IsNaN(progress) { + progress = 100 + } return TaskInfo{ ID: task.GetID(), Name: task.GetName(), State: task.GetState(), Status: task.GetStatus(), - Progress: task.GetProgress(), + Progress: progress, Error: errMsg, } } From 85a28d98223d3114670b0c0101f6ad2dffed8829 Mon Sep 17 00:00:00 2001 From: Echo Response <32877980+EchoResponse@users.noreply.github.com> Date: Sat, 20 Jan 2024 21:22:50 +0800 Subject: [PATCH 118/659] fix(quqi): error on uploading an existing file (#5920) --- drivers/quqi/driver.go | 23 ++++++++++++++++++++++- drivers/quqi/types.go | 3 +++ drivers/quqi/util.go | 11 +++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 9bc23f8b837..98c184bd49e 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -306,6 +306,22 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return nil, err } + // check exist + // if the file already exists in Quqi server, there is no need to actually upload it + if uploadInitResp.Data.Exist { + // the file name returned by Quqi does not include the extension name + nodeName, nodeExt := uploadInitResp.Data.NodeName, rawExt(stream.GetName()) + if nodeExt != "" { + nodeName = nodeName + "." + nodeExt + } + return &model.Object{ + ID: strconv.FormatInt(uploadInitResp.Data.NodeID, 10), + Name: nodeName, + Size: stream.GetSize(), + Modified: stream.ModTime(), + Ctime: stream.CreateTime(), + }, nil + } // listParts _, err = d.request("upload.quqi.com:20807", "/upload/v1/listParts", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ @@ -384,9 +400,14 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return nil, err } + // the file name returned by Quqi does not include the extension name + nodeName, nodeExt := uploadFinishResp.Data.NodeName, rawExt(stream.GetName()) + if nodeExt != "" { + nodeName = nodeName + "." + nodeExt + } return &model.Object{ ID: strconv.FormatInt(uploadFinishResp.Data.NodeID, 10), - Name: uploadFinishResp.Data.NodeName, + Name: nodeName, Size: stream.GetSize(), Modified: stream.ModTime(), Ctime: stream.CreateTime(), diff --git a/drivers/quqi/types.go b/drivers/quqi/types.go index 3f6c4ebe2dd..f64fb748e47 100644 --- a/drivers/quqi/types.go +++ b/drivers/quqi/types.go @@ -133,6 +133,9 @@ type UploadInitResp struct { Token string `json:"token"` UploadID string `json:"upload_id"` URL string `json:"url"` + NodeID int64 `json:"node_id"` + NodeName string `json:"node_name"` + ParentID int64 `json:"parent_id"` } `json:"data"` Err int `json:"err"` Msg string `json:"msg"` diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go index 91fc0a02d7c..23a6a966934 100644 --- a/drivers/quqi/util.go +++ b/drivers/quqi/util.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/url" + stdpath "path" "strings" "github.com/alist-org/alist/v3/drivers/base" @@ -97,3 +98,13 @@ func (d *Quqi) checkLogin() bool { } return true } + +// rawExt 保留扩展名大小写 +func rawExt(name string) string { + ext := stdpath.Ext(name) + if strings.HasPrefix(ext, ".") { + ext = ext[1:] + } + + return ext +} From d88b54d98a9d20a96e06440c80194b4c093c9b05 Mon Sep 17 00:00:00 2001 From: Echo Response <32877980+EchoResponse@users.noreply.github.com> Date: Sun, 21 Jan 2024 15:28:52 +0800 Subject: [PATCH 119/659] fix(quqi): empty file link for non vip user (#5926) * fix(quqi): error returned when uploading a file that existed * fix empty download link for no vip user * fix cannot parse request result --------- Co-authored-by: Andy Hsu --- drivers/quqi/driver.go | 27 ++++++++++++++++++++++++++- drivers/quqi/types.go | 7 +++++++ drivers/quqi/util.go | 7 ++++++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 98c184bd49e..43cdbf7780b 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -127,6 +127,7 @@ func (d *Quqi) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] func (d *Quqi) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var getDocResp = &GetDocRes{} + // 优先从getDoc接口获取文件预览链接，速度比实际下载链接更快 if _, err := d.request("", "/api/doc/getDoc", resty.MethodPost, func(req *resty.Request) { req.SetFormData(map[string]string{ "quqi_id": d.GroupID, @@ -137,9 +138,33 @@ func (d *Quqi) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* }, getDocResp); err != nil { return nil, err } + if getDocResp.Data.OriginPath != "" { + return &model.Link{ + URL: getDocResp.Data.OriginPath, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }, nil + } + // 对于非会员用户，无法从getDoc接口获取文件预览链接，只能获取下载链接 + var getDownloadResp GetDownloadResp + if _, err := d.request("", "/api/doc/getDownload", resty.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": file.GetID(), + "url_type": "undefined", + "entry_type": "undefined", + "client_id": d.ClientID, + "no_redirect": "1", + }) + }, &getDownloadResp); err != nil { + return nil, err + } return &model.Link{ - URL: getDocResp.Data.OriginPath, + URL: getDownloadResp.Data.Url, Header: http.Header{ "Origin": []string{"https://quqi.com"}, "Cookie": []string{d.Cookie}, diff --git a/drivers/quqi/types.go b/drivers/quqi/types.go index f64fb748e47..00ca0d9880c 100644 --- a/drivers/quqi/types.go +++ b/drivers/quqi/types.go @@ -31,6 +31,13 @@ type GetDocRes struct { } `json:"data"` } +type GetDownloadResp struct { + BaseRes + Data struct { + Url string `json:"url"` + } `json:"data"` +} + type MakeDirRes struct { BaseRes Data struct { diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go index 23a6a966934..943891f5863 100644 --- a/drivers/quqi/util.go +++ b/drivers/quqi/util.go @@ -32,7 +32,7 @@ func (d *Quqi) request(host string, path string, method string, callback base.Re req.SetHeaders(map[string]string{ "Origin": "https://quqi.com", "Cookie": d.Cookie, - }).SetResult(&result) + }) if d.GroupID != "" { req.SetQueryParam("quqiid", d.GroupID) @@ -46,6 +46,11 @@ func (d *Quqi) request(host string, path string, method string, callback base.Re if err != nil { return nil, err } + // resty.Request.SetResult cannot parse result correctly sometimes + err = utils.Json.Unmarshal(res.Body(), &result) + if err != nil { + return nil, err + } if result.Code != 0 { return nil, errors.New(result.Message) } From 9222510d8d7483a7532157060aa4c117e0570111 Mon Sep 17 00:00:00 2001 From: Echo Response <32877980+EchoResponse@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:47:49 +0800 Subject: [PATCH 120/659] feat(quqi): add download link with cdn (#5938) * feat(quqi): add download link by cdn * fix(quqi): cookie error when login with phone number --- drivers/quqi/driver.go | 58 ++++-------- drivers/quqi/meta.go | 1 + drivers/quqi/types.go | 20 ++++ drivers/quqi/util.go | 205 ++++++++++++++++++++++++++++++++++++++++- go.mod | 1 + go.sum | 3 + 6 files changed, 246 insertions(+), 42 deletions(-) diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 43cdbf7780b..89052a55f24 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -16,12 +16,14 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" "github.com/tencentyun/cos-go-sdk-v5" ) type Quqi struct { model.Storage Addition + Cookie string // Cookie GroupID string // 私人云群组ID ClientID string // 随机生成客户端ID 经过测试，部分接口调用若不携带client id会出现错误 } @@ -125,51 +127,27 @@ func (d *Quqi) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] } func (d *Quqi) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - var getDocResp = &GetDocRes{} - - // 优先从getDoc接口获取文件预览链接，速度比实际下载链接更快 - if _, err := d.request("", "/api/doc/getDoc", resty.MethodPost, func(req *resty.Request) { - req.SetFormData(map[string]string{ - "quqi_id": d.GroupID, - "tree_id": "1", - "node_id": file.GetID(), - "client_id": d.ClientID, - }) - }, getDocResp); err != nil { - return nil, err + if d.CDN { + link, err := d.linkFromCDN(file.GetID()) + if err != nil { + log.Warn(err) + } else { + return link, nil + } } - if getDocResp.Data.OriginPath != "" { - return &model.Link{ - URL: getDocResp.Data.OriginPath, - Header: http.Header{ - "Origin": []string{"https://quqi.com"}, - "Cookie": []string{d.Cookie}, - }, - }, nil + + link, err := d.linkFromPreview(file.GetID()) + if err != nil { + log.Warn(err) + } else { + return link, nil } - // 对于非会员用户，无法从getDoc接口获取文件预览链接，只能获取下载链接 - var getDownloadResp GetDownloadResp - if _, err := d.request("", "/api/doc/getDownload", resty.MethodGet, func(req *resty.Request) { - req.SetQueryParams(map[string]string{ - "quqi_id": d.GroupID, - "tree_id": "1", - "node_id": file.GetID(), - "url_type": "undefined", - "entry_type": "undefined", - "client_id": d.ClientID, - "no_redirect": "1", - }) - }, &getDownloadResp); err != nil { + link, err = d.linkFromDownload(file.GetID()) + if err != nil { return nil, err } - return &model.Link{ - URL: getDownloadResp.Data.Url, - Header: http.Header{ - "Origin": []string{"https://quqi.com"}, - "Cookie": []string{d.Cookie}, - }, - }, nil + return link, nil } func (d *Quqi) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { diff --git a/drivers/quqi/meta.go b/drivers/quqi/meta.go index 0820796e749..aaaa0a19444 100644 --- a/drivers/quqi/meta.go +++ b/drivers/quqi/meta.go @@ -10,6 +10,7 @@ type Addition struct { Phone string `json:"phone"` Password string `json:"password"` Cookie string `json:"cookie" help:"Cookie can be used on multiple clients at the same time"` + CDN bool `json:"cdn" help:"If you enable this option, the download speed can be increased, but there will be some performance loss"` } var config = driver.Config{ diff --git a/drivers/quqi/types.go b/drivers/quqi/types.go index 00ca0d9880c..32557361532 100644 --- a/drivers/quqi/types.go +++ b/drivers/quqi/types.go @@ -175,3 +175,23 @@ type UploadFinishResp struct { Err int `json:"err"` Msg string `json:"msg"` } + +type UrlExchangeResp struct { + BaseRes + Data struct { + Name string `json:"name"` + Mime string `json:"mime"` + Size int64 `json:"size"` + DownloadType int `json:"download_type"` + ChannelType int `json:"channel_type"` + ChannelID int `json:"channel_id"` + Url string `json:"url"` + ExpiredTime int64 `json:"expired_time"` + IsEncrypted bool `json:"is_encrypted"` + EncryptedSize int64 `json:"encrypted_size"` + EncryptedAlg string `json:"encrypted_alg"` + EncryptedKey string `json:"encrypted_key"` + PassportID int64 `json:"passport_id"` + RequestExpiredTime int64 `json:"request_expired_time"` + } `json:"data"` +} diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go index 943891f5863..c025f6ee8af 100644 --- a/drivers/quqi/util.go +++ b/drivers/quqi/util.go @@ -1,17 +1,26 @@ package quqi import ( + "bufio" + "context" "encoding/base64" "errors" "fmt" + "io" + "net/http" "net/url" stdpath "path" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + "github.com/minio/sio" ) // do others that not defined in Driver interface @@ -64,10 +73,12 @@ func (d *Quqi) request(host string, path string, method string, callback base.Re } func (d *Quqi) login() error { - if d.Cookie != "" && d.checkLogin() { + if d.Addition.Cookie != "" { + d.Cookie = d.Addition.Cookie + } + if d.checkLogin() { return nil } - if d.Cookie != "" { return errors.New("cookie is invalid") } @@ -113,3 +124,193 @@ func rawExt(name string) string { return ext } + +// decryptKey 获取密码 +func decryptKey(encodeKey string) []byte { + // 移除非法字符 + u := strings.ReplaceAll(encodeKey, "[^A-Za-z0-9+\\/]", "") + + // 计算输出字节数组的长度 + o := len(u) + a := 32 + + // 创建输出字节数组 + c := make([]byte, a) + + // 编码循环 + s := uint32(0) // 累加器 + f := 0 // 输出数组索引 + for l := 0; l < o; l++ { + r := l & 3 // 取模4，得到当前字符在四字节块中的位置 + i := u[l] // 当前字符的ASCII码 + + // 编码当前字符 + switch { + case i >= 65 && i < 91: // 大写字母 + s |= uint32(i-65) << uint32(6*(3-r)) + case i >= 97 && i < 123: // 小写字母 + s |= uint32(i-71) << uint32(6*(3-r)) + case i >= 48 && i < 58: // 数字 + s |= uint32(i+4) << uint32(6*(3-r)) + case i == 43: // 加号 + s |= uint32(62) << uint32(6*(3-r)) + case i == 47: // 斜杠 + s |= uint32(63) << uint32(6*(3-r)) + } + + // 如果累加器已经包含了四个字符，或者是最后一个字符，则写入输出数组 + if r == 3 || l == o-1 { + for e := 0; e < 3 && f < a; e, f = e+1, f+1 { + c[f] = byte(s >> (16 >> e & 24) & 255) + } + s = 0 + } + } + + return c +} + +func (d *Quqi) linkFromPreview(id string) (*model.Link, error) { + var getDocResp GetDocRes + if _, err := d.request("", "/api/doc/getDoc", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": id, + "client_id": d.ClientID, + }) + }, &getDocResp); err != nil { + return nil, err + } + if getDocResp.Data.OriginPath == "" { + return nil, errors.New("cannot get link from preview") + } + return &model.Link{ + URL: getDocResp.Data.OriginPath, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Quqi) linkFromDownload(id string) (*model.Link, error) { + var getDownloadResp GetDownloadResp + if _, err := d.request("", "/api/doc/getDownload", resty.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": id, + "url_type": "undefined", + "entry_type": "undefined", + "client_id": d.ClientID, + "no_redirect": "1", + }) + }, &getDownloadResp); err != nil { + return nil, err + } + if getDownloadResp.Data.Url == "" { + return nil, errors.New("cannot get link from download") + } + + return &model.Link{ + URL: getDownloadResp.Data.Url, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Quqi) linkFromCDN(id string) (*model.Link, error) { + downloadLink, err := d.linkFromDownload(id) + if err != nil { + return nil, err + } + + var urlExchangeResp UrlExchangeResp + if _, err = d.request("api.quqi.com", "/preview/downloadInfo/url/exchange", resty.MethodGet, func(req *resty.Request) { + req.SetQueryParam("url", downloadLink.URL) + }, &urlExchangeResp); err != nil { + return nil, err + } + if urlExchangeResp.Data.Url == "" { + return nil, errors.New("cannot get link from cdn") + } + + // 假设存在未加密的情况 + if !urlExchangeResp.Data.IsEncrypted { + return &model.Link{ + URL: urlExchangeResp.Data.Url, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }, nil + } + + // 根据sio(https://github.com/minio/sio/blob/master/DARE.md)描述及实际测试，得出以下结论： + // 1. 加密后大小(encrypted_size)-原始文件大小(size) = 加密包的头大小+身份验证标识 = (16+16) * N -> N为加密包的数量 + // 2. 原始文件大小(size)+64*1024-1 / (64*1024) = N -> 每个包的有效负载为64K + remoteClosers := utils.EmptyClosers() + payloadSize := int64(1 << 16) + expiration := time.Until(time.Unix(urlExchangeResp.Data.ExpiredTime, 0)) + resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + encryptedOffset := httpRange.Start / payloadSize * (payloadSize + 32) + decryptedOffset := httpRange.Start % payloadSize + encryptedLength := (httpRange.Length+httpRange.Start+payloadSize-1)/payloadSize*(payloadSize+32) - encryptedOffset + if httpRange.Length < 0 { + encryptedLength = httpRange.Length + } else { + if httpRange.Length+httpRange.Start >= urlExchangeResp.Data.Size || encryptedLength+encryptedOffset >= urlExchangeResp.Data.EncryptedSize { + encryptedLength = -1 + } + } + //log.Debugf("size: %d\tencrypted_size: %d", urlExchangeResp.Data.Size, urlExchangeResp.Data.EncryptedSize) + //log.Debugf("http range offset: %d, length: %d", httpRange.Start, httpRange.Length) + //log.Debugf("encrypted offset: %d, length: %d, decrypted offset: %d", encryptedOffset, encryptedLength, decryptedOffset) + + rrc, err := stream.GetRangeReadCloserFromLink(urlExchangeResp.Data.EncryptedSize, &model.Link{ + URL: urlExchangeResp.Data.Url, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }) + if err != nil { + return nil, err + } + + rc, err := rrc.RangeRead(ctx, http_range.Range{Start: encryptedOffset, Length: encryptedLength}) + remoteClosers.AddClosers(rrc.GetClosers()) + if err != nil { + return nil, err + } + + decryptReader, err := sio.DecryptReader(rc, sio.Config{ + MinVersion: sio.Version10, + MaxVersion: sio.Version20, + CipherSuites: []byte{sio.CHACHA20_POLY1305, sio.AES_256_GCM}, + Key: decryptKey(urlExchangeResp.Data.EncryptedKey), + SequenceNumber: uint32(httpRange.Start / payloadSize), + }) + if err != nil { + return nil, err + } + bufferReader := bufio.NewReader(decryptReader) + bufferReader.Discard(int(decryptedOffset)) + + return utils.NewReadCloser(bufferReader, func() error { + return nil + }), nil + } + + return &model.Link{ + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + RangeReadCloser: &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: remoteClosers}, + Expiration: &expiration, + }, nil +} diff --git a/go.mod b/go.mod index efe1cade1e8..604fdb12646 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 github.com/maruel/natural v1.1.1 + github.com/minio/sio v0.3.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 02bb391dc10..f77fff3c7ec 100644 --- a/go.sum +++ b/go.sum @@ -311,6 +311,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= +github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -486,6 +488,7 @@ golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= From aef952ae6815d6d52d74b8cb95f0ca414e00a8b9 Mon Sep 17 00:00:00 2001 From: shouko Date: Wed, 24 Jan 2024 18:03:50 +0900 Subject: [PATCH 121/659] feat(dropbox): add root_namespace_id to access teams folder (#5929) * feat(dropbox): add root_namespace_id to access teams folder * fix(dropbox): get_current_account API request * feat(dropbox): extract root_namespace_id properly * style: format code --- drivers/dropbox/driver.go | 20 +++++++++++++++++++- drivers/dropbox/meta.go | 3 ++- drivers/dropbox/types.go | 7 +++++++ drivers/dropbox/util.go | 14 ++++++++++++-- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/drivers/dropbox/driver.go b/drivers/dropbox/driver.go index 95148b94e96..9b1717b04d9 100644 --- a/drivers/dropbox/driver.go +++ b/drivers/dropbox/driver.go @@ -45,7 +45,25 @@ func (d *Dropbox) Init(ctx context.Context) error { if result != query { return fmt.Errorf("failed to check user: %s", string(res)) } - return nil + d.RootNamespaceId, err = d.GetRootNamespaceId(ctx) + + return err +} + +func (d *Dropbox) GetRootNamespaceId(ctx context.Context) (string, error) { + res, err := d.request("/2/users/get_current_account", http.MethodPost, func(req *resty.Request) { + req.SetBody(nil) + }) + if err != nil { + return "", err + } + var currentAccountResp CurrentAccountResp + err = utils.Json.Unmarshal(res, ¤tAccountResp) + if err != nil { + return "", err + } + rootNamespaceId := currentAccountResp.RootInfo.RootNamespaceId + return rootNamespaceId, nil } func (d *Dropbox) Drop(ctx context.Context) error { diff --git a/drivers/dropbox/meta.go b/drivers/dropbox/meta.go index 07ab44695c1..6e7bc014790 100644 --- a/drivers/dropbox/meta.go +++ b/drivers/dropbox/meta.go @@ -17,7 +17,8 @@ type Addition struct { ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` - AccessToken string + AccessToken string + RootNamespaceId string } var config = driver.Config{ diff --git a/drivers/dropbox/types.go b/drivers/dropbox/types.go index 7ddbb9683c1..f2ec4cb7e7d 100644 --- a/drivers/dropbox/types.go +++ b/drivers/dropbox/types.go @@ -23,6 +23,13 @@ type RefreshTokenErrorResp struct { ErrorDescription string `json:"error_description"` } +type CurrentAccountResp struct { + RootInfo struct { + RootNamespaceId string `json:"root_namespace_id"` + HomeNamespaceId string `json:"home_namespace_id"` + } `json:"root_info"` +} + type File struct { Tag string `json:".tag"` Name string `json:"name"` diff --git a/drivers/dropbox/util.go b/drivers/dropbox/util.go index 14a5c6c6fc6..5065f08d394 100644 --- a/drivers/dropbox/util.go +++ b/drivers/dropbox/util.go @@ -46,12 +46,22 @@ func (d *Dropbox) refreshToken() error { func (d *Dropbox) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) - if method == http.MethodPost { - req.SetHeader("Content-Type", "application/json") + if d.RootNamespaceId != "" { + apiPathRootJson, err := utils.Json.MarshalToString(map[string]interface{}{ + ".tag": "root", + "root": d.RootNamespaceId, + }) + if err != nil { + return nil, err + } + req.SetHeader("Dropbox-API-Path-Root", apiPathRootJson) } if callback != nil { callback(req) } + if method == http.MethodPost && req.Body != nil { + req.SetHeader("Content-Type", "application/json") + } var e ErrorResp req.SetError(&e) res, err := req.Execute(method, d.base+uri) From c82866975e96a8fe016c5a8e280201573724cbae Mon Sep 17 00:00:00 2001 From: Jing <42014615+jing332@users.noreply.github.com> Date: Tue, 30 Jan 2024 21:21:53 +0800 Subject: [PATCH 122/659] fix: error on repeated reading `static` (#5957) * Update static.go * rm initial value of static --------- Co-authored-by: Andy Hsu --- server/static/static.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/static/static.go b/server/static/static.go index 8e2054af7f9..cb5109952f4 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -16,11 +16,11 @@ import ( "github.com/gin-gonic/gin" ) -var static fs.FS = public.Public +var static fs.FS func initStatic() { if conf.Conf.DistDir == "" { - dist, err := fs.Sub(static, "dist") + dist, err := fs.Sub(public.Public, "dist") if err != nil { utils.Log.Fatalf("failed to read dist dir") } From 9bd3c87bccff4585313fdc5f9caf54ed1c534216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9D=BF=E9=9F=B3?= Date: Thu, 1 Feb 2024 10:43:08 +0800 Subject: [PATCH 123/659] fix(ldap): `exiting by peer` exception occurred during the TLS connection(#5977) --- server/handles/ldap_login.go | 38 +++++++++++++++++------------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go index b52e108249c..cf3148291b1 100644 --- a/server/handles/ldap_login.go +++ b/server/handles/ldap_login.go @@ -50,31 +50,13 @@ func loginLdap(c *gin.Context, req *LoginReq) { ldapUserSearchBase := setting.GetStr(conf.LdapUserSearchBase) ldapUserSearchFilter := setting.GetStr(conf.LdapUserSearchFilter) // (uid=%s) - var tlsEnabled bool = false - if strings.HasPrefix(ldapServer, "ldaps://") { - tlsEnabled = true - ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") - } else if strings.HasPrefix(ldapServer, "ldap://") { - ldapServer = strings.TrimPrefix(ldapServer, "ldap://") - } - - l, err := ldap.Dial("tcp", ldapServer) + // Connect to LdapServer + l, err := dial(ldapServer) if err != nil { utils.Log.Errorf("failed to connect to LDAP: %v", err) common.ErrorResp(c, err, 500) return } - defer l.Close() - - if tlsEnabled { - // Reconnect with TLS - err = l.StartTLS(&tls.Config{InsecureSkipVerify: true}) - if err != nil { - utils.Log.Errorf("failed to start tls: %v", err) - common.ErrorResp(c, err, 500) - return - } - } // First bind with a read only user if ldapManagerDN != "" && ldapManagerPassword != "" { @@ -157,3 +139,19 @@ func ladpRegister(username string) (*model.User, error) { } return user, nil } + +func dial(ldapServer string) (*ldap.Conn, error) { + var tlsEnabled bool = false + if strings.HasPrefix(ldapServer, "ldaps://") { + tlsEnabled = true + ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") + } else if strings.HasPrefix(ldapServer, "ldap://") { + ldapServer = strings.TrimPrefix(ldapServer, "ldap://") + } + + if tlsEnabled { + return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: true}) + } else { + return ldap.Dial("tcp", ldapServer) + } +} From 812f58ae6d2b3b6a86ebe5c0c2a93003ee7ae7cd Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Fri, 2 Feb 2024 21:04:43 +0800 Subject: [PATCH 124/659] fix(mopan): client version is too low (#5987) * fix(mopan): download err ` client version is too low` * feat(mopan):support sms login * refactor(quqi): upload use s3 --- drivers/mopan/driver.go | 34 ++++++++++++--- drivers/mopan/meta.go | 1 + drivers/quqi/driver.go | 97 +++++++++++++++++++++++------------------ go.mod | 6 +-- go.sum | 20 +-------- 5 files changed, 86 insertions(+), 72 deletions(-) diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index f3bb4e74928..b5441bff2ea 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -43,23 +43,31 @@ func (d *MoPan) Init(ctx context.Context) error { if d.uploadThread < 1 || d.uploadThread > 32 { d.uploadThread, d.UploadThread = 3, "3" } - login := func() error { - data, err := d.client.Login(d.Phone, d.Password) + + defer func() { d.SMSCode = "" }() + + login := func() (err error) { + var loginData *mopan.LoginResp + if d.SMSCode != "" { + loginData, err = d.client.LoginBySmsStep2(d.Phone, d.SMSCode) + } else { + loginData, err = d.client.Login(d.Phone, d.Password) + } if err != nil { return err } - d.client.SetAuthorization(data.Token) + d.client.SetAuthorization(loginData.Token) info, err := d.client.GetUserInfo() if err != nil { return err } d.userID = info.UserID - log.Debugf("[mopan] Phone: %s UserCloudStorageRelations: %+v", d.Phone, data.UserCloudStorageRelations) + log.Debugf("[mopan] Phone: %s UserCloudStorageRelations: %+v", d.Phone, loginData.UserCloudStorageRelations) cloudCircleApp, _ := d.client.QueryAllCloudCircleApp() log.Debugf("[mopan] Phone: %s CloudCircleApp: %+v", d.Phone, cloudCircleApp) if d.RootFolderID == "" { - for _, userCloudStorage := range data.UserCloudStorageRelations { + for _, userCloudStorage := range loginData.UserCloudStorageRelations { if userCloudStorage.Path == "/文件" { d.RootFolderID = userCloudStorage.FolderID } @@ -76,8 +84,20 @@ func (d *MoPan) Init(ctx context.Context) error { op.MustSaveDriverStorage(d) } return err - }).SetDeviceInfo(d.DeviceInfo) - d.DeviceInfo = d.client.GetDeviceInfo() + }) + + var deviceInfo mopan.DeviceInfo + if strings.TrimSpace(d.DeviceInfo) != "" && utils.Json.UnmarshalFromString(d.DeviceInfo, &deviceInfo) == nil { + d.client.SetDeviceInfo(&deviceInfo) + } + d.DeviceInfo, _ = utils.Json.MarshalToString(d.client.GetDeviceInfo()) + + if strings.Contains(d.SMSCode, "send") { + if _, err := d.client.LoginBySms(d.Phone); err != nil { + return err + } + return errors.New("please enter the SMS code") + } return login() } diff --git a/drivers/mopan/meta.go b/drivers/mopan/meta.go index e6583fc1b8c..c111fedc16a 100644 --- a/drivers/mopan/meta.go +++ b/drivers/mopan/meta.go @@ -8,6 +8,7 @@ import ( type Addition struct { Phone string `json:"phone" required:"true"` Password string `json:"password" required:"true"` + SMSCode string `json:"sms_code" help:"input 'send' send sms "` RootFolderID string `json:"root_folder_id" default:""` diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 89052a55f24..51e54981a18 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -1,11 +1,9 @@ package quqi import ( + "bytes" "context" - "fmt" "io" - "net/http" - "net/url" "strconv" "strings" "time" @@ -15,9 +13,13 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" - "github.com/tencentyun/cos-go-sdk-v5" ) type Quqi struct { @@ -348,49 +350,60 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea return nil, err } // upload - u, err := url.Parse(fmt.Sprintf("https://%s.cos.ap-shanghai.myqcloud.com", uploadInitResp.Data.Bucket)) - b := &cos.BaseURL{BucketURL: u} - client := cos.NewClient(b, &http.Client{ - Transport: &cos.CredentialTransport{ - Credential: cos.NewTokenCredential(tempKeyResp.Data.Credentials.TmpSecretID, tempKeyResp.Data.Credentials.TmpSecretKey, tempKeyResp.Data.Credentials.SessionToken), - }, - }) - partSize := int64(1024 * 1024 * 2) - partCount := (stream.GetSize() + partSize - 1) / partSize - for i := 1; i <= int(partCount); i++ { - length := partSize - if i == int(partCount) { - length = stream.GetSize() - (int64(i)-1)*partSize + // u, err := url.Parse(fmt.Sprintf("https://%s.cos.ap-shanghai.myqcloud.com", uploadInitResp.Data.Bucket)) + // b := &cos.BaseURL{BucketURL: u} + // client := cos.NewClient(b, &http.Client{ + // Transport: &cos.CredentialTransport{ + // Credential: cos.NewTokenCredential(tempKeyResp.Data.Credentials.TmpSecretID, tempKeyResp.Data.Credentials.TmpSecretKey, tempKeyResp.Data.Credentials.SessionToken), + // }, + // }) + // partSize := int64(1024 * 1024 * 2) + // partCount := (stream.GetSize() + partSize - 1) / partSize + // for i := 1; i <= int(partCount); i++ { + // length := partSize + // if i == int(partCount) { + // length = stream.GetSize() - (int64(i)-1)*partSize + // } + // _, err := client.Object.UploadPart( + // ctx, uploadInitResp.Data.Key, uploadInitResp.Data.UploadID, i, io.LimitReader(f, partSize), &cos.ObjectUploadPartOptions{ + // ContentLength: length, + // }, + // ) + // if err != nil { + // return nil, err + // } + // } + + cfg := &aws.Config{ + Credentials: credentials.NewStaticCredentials(tempKeyResp.Data.Credentials.TmpSecretID, tempKeyResp.Data.Credentials.TmpSecretKey, tempKeyResp.Data.Credentials.SessionToken), + Region: aws.String("ap-shanghai"), + Endpoint: aws.String("cos.ap-shanghai.myqcloud.com"), + } + s, err := session.NewSession(cfg) + if err != nil { + return nil, err + } + uploader := s3manager.NewUploader(s) + buf := make([]byte, 1024*1024*2) + for partNumber := int64(1); ; partNumber++ { + n, err := io.ReadFull(f, buf) + if err != nil && err != io.ErrUnexpectedEOF { + if err == io.EOF { + break + } + return nil, err } - _, err := client.Object.UploadPart( - ctx, uploadInitResp.Data.Key, uploadInitResp.Data.UploadID, i, io.LimitReader(f, partSize), &cos.ObjectUploadPartOptions{ - ContentLength: length, - }, - ) + _, err = uploader.S3.UploadPartWithContext(ctx, &s3.UploadPartInput{ + UploadId: &uploadInitResp.Data.UploadID, + Key: &uploadInitResp.Data.Key, + Bucket: &uploadInitResp.Data.Bucket, + PartNumber: aws.Int64(partNumber), + Body: bytes.NewReader(buf[:n]), + }) if err != nil { return nil, err } } - //cfg := &aws.Config{ - // Credentials: credentials.NewStaticCredentials(tempKeyResp.Data.Credentials.TmpSecretID, tempKeyResp.Data.Credentials.TmpSecretKey, tempKeyResp.Data.Credentials.SessionToken), - // Region: aws.String("shanghai"), - // Endpoint: aws.String("cos.ap-shanghai.myqcloud.com"), - // // S3ForcePathStyle: aws.Bool(true), - //} - //s, err := session.NewSession(cfg) - //if err != nil { - // return nil, err - //} - //uploader := s3manager.NewUploader(s) - //input := &s3manager.UploadInput{ - // Bucket: &uploadInitResp.Data.Bucket, - // Key: &uploadInitResp.Data.Key, - // Body: f, - //} - //_, err = uploader.UploadWithContext(ctx, input) - //if err != nil { - // return nil, err - //} // finish upload var uploadFinishResp UploadFinishResp _, err = d.request("", "/api/upload/v1/file/finish", resty.MethodPost, func(req *resty.Request) { diff --git a/go.mod b/go.mod index 604fdb12646..81f71d5291f 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/djherbis/times v1.6.0 github.com/dlclark/regexp2 v1.10.0 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 - github.com/foxxorcat/mopan-sdk-go v0.1.4 + github.com/foxxorcat/mopan-sdk-go v0.1.5 github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gaoyb7/115drive-webdav v0.1.8 github.com/gin-contrib/cors v1.5.0 @@ -99,7 +99,6 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.0 // indirect - github.com/clbanning/mxj v1.8.4 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect @@ -121,7 +120,6 @@ require ( github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-querystring v1.1.0 // indirect github.com/google/go-tpm v0.9.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect @@ -155,7 +153,6 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mozillazg/go-httpheader v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect @@ -187,7 +184,6 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tencentyun/cos-go-sdk-v5 v0.7.45 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index f77fff3c7ec..f3781212a81 100644 --- a/go.sum +++ b/go.sum @@ -7,7 +7,6 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U= @@ -101,8 +100,6 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= -github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= -github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= @@ -122,7 +119,6 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= @@ -131,8 +127,8 @@ github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= -github.com/foxxorcat/mopan-sdk-go v0.1.4 h1:6utvPiBv8KDRDVKB7A4FERdrVxcHKZd2fBFCNuKcXzU= -github.com/foxxorcat/mopan-sdk-go v0.1.4/go.mod h1:iWHA2JFhzmKR28ySp1ON0g6DjLaYtvb5jhTqPVTDW9A= +github.com/foxxorcat/mopan-sdk-go v0.1.5 h1:N3LqOvk2aWWxszsFIkArP5udIv74uTei/bH2jM3tfSc= +github.com/foxxorcat/mopan-sdk-go v0.1.5/go.mod h1:iWHA2JFhzmKR28ySp1ON0g6DjLaYtvb5jhTqPVTDW9A= github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -197,14 +193,10 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= -github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -315,7 +307,6 @@ github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -325,9 +316,6 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= -github.com/mozillazg/go-httpheader v0.4.0 h1:aBn6aRXtFzyDLZ4VIRLsZbbJloagQfMnCiYgOq6hK4w= -github.com/mozillazg/go-httpheader v0.4.0/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= @@ -444,10 +432,6 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4UvMT7OmKhH9sOIvS8iXkxfPdnn9wQA= github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= -github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= -github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= -github.com/tencentyun/cos-go-sdk-v5 v0.7.45 h1:5/ZGOv846tP6+2X7w//8QjLgH2KcUK+HciFbfjWquFU= -github.com/tencentyun/cos-go-sdk-v5 v0.7.45/go.mod h1:DH9US8nB+AJXqwu/AMOrCFN1COv3dpytXuJWHgdg7kE= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= From da5e35578aa6dbd6e0da5b09255525a6cf26b584 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 3 Feb 2024 19:44:50 +0800 Subject: [PATCH 125/659] fix: embed all files of dist --- public/public.go | 2 +- server/static/static.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/public.go b/public/public.go index a158f3e0226..e94146c3b08 100644 --- a/public/public.go +++ b/public/public.go @@ -2,5 +2,5 @@ package public import "embed" -//go:embed dist +//go:embed all:dist var Public embed.FS diff --git a/server/static/static.go b/server/static/static.go index cb5109952f4..ec16014c22b 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -3,7 +3,6 @@ package static import ( "errors" "fmt" - "github.com/alist-org/alist/v3/public" "io" "io/fs" "net/http" @@ -13,6 +12,7 @@ import ( "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/public" "github.com/gin-gonic/gin" ) From e49fda3e2ac884d8cad81891db4d53c1f559d93e Mon Sep 17 00:00:00 2001 From: ArcticLampyrid Date: Thu, 8 Feb 2024 19:22:29 +0800 Subject: [PATCH 126/659] fix: WebDAV's creation date should use `RFC3339` format (#6015 close #5878) --- server/webdav/prop.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/webdav/prop.go b/server/webdav/prop.go index 3c3b10d8227..466644a0eef 100644 --- a/server/webdav/prop.go +++ b/server/webdav/prop.go @@ -14,6 +14,7 @@ import ( "net/http" "path" "strconv" + "time" "github.com/alist-org/alist/v3/internal/model" ) @@ -384,7 +385,7 @@ func findLastModified(ctx context.Context, ls LockSystem, name string, fi model. return fi.ModTime().UTC().Format(http.TimeFormat), nil } func findCreationDate(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { - return fi.CreateTime().UTC().Format(http.TimeFormat), nil + return fi.CreateTime().UTC().Format(time.RFC3339), nil } // ErrNotImplemented should be returned by optional interfaces if they From 6d85f1b0c0f68b5ef3876de296bc62122a2b2970 Mon Sep 17 00:00:00 2001 From: 123pan <136693889+123pan-com@users.noreply.github.com> Date: Fri, 9 Feb 2024 14:45:44 +0800 Subject: [PATCH 127/659] fix(123): `User-Agent` and rate limit (#6012) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 修复标签 * 新增接口限流器。防止云盘云端把Alist当做攻击,封禁Alist客户端 --------- Co-authored-by: 风信子 --- drivers/123/driver.go | 11 +++++++++++ drivers/123/util.go | 9 +++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 1f1ae85886a..f5d981ef636 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,9 +6,12 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "golang.org/x/time/rate" "io" "net/http" "net/url" + "sync" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -26,6 +29,7 @@ import ( type Pan123 struct { model.Storage Addition + apiRateLimit sync.Map } func (d *Pan123) Config() driver.Config { @@ -254,4 +258,11 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } +func (d *Pan123) APIRateLimit(api string) bool { + limiter, _ := d.apiRateLimit.LoadOrStore(api, + rate.NewLimiter(rate.Every(time.Millisecond*700), 1)) + ins := limiter.(*rate.Limiter) + return ins.Allow() +} + var _ driver.Driver = (*Pan123)(nil) diff --git a/drivers/123/util.go b/drivers/123/util.go index 1a86e1bdef2..9d5d6780167 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -160,7 +160,7 @@ func (d *Pan123) login() error { SetHeaders(map[string]string{ "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", - "user-agent": "Dart/2.19(dart:io)", + "user-agent": "Dart/2.19(dart:io)-alist", "platform": "web", "app-version": "3", //"user-agent": base.UserAgent, @@ -197,7 +197,7 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, - "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.0.0", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) alist-client", "platform": "web", "app-version": "3", //"user-agent": base.UserAgent, @@ -235,7 +235,12 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r func (d *Pan123) getFiles(parentId string) ([]File, error) { page := 1 res := make([]File, 0) + // 2024-02-06 fix concurrency by 123pan for { + if !d.APIRateLimit(FileList) { + time.Sleep(time.Millisecond * 200) + continue + } var resp Files query := map[string]string{ "driveId": "0", From 47f4b05517677f4ca8d7d2782a66fc39909fbbca Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 15 Feb 2024 18:54:19 +0800 Subject: [PATCH 128/659] feat(sftp): allow ignore symlink error (close #6026) --- drivers/sftp/meta.go | 1 + drivers/sftp/types.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/drivers/sftp/meta.go b/drivers/sftp/meta.go index d77398f333f..bdc3d827ff2 100644 --- a/drivers/sftp/meta.go +++ b/drivers/sftp/meta.go @@ -11,6 +11,7 @@ type Addition struct { PrivateKey string `json:"private_key" type:"text"` Password string `json:"password"` driver.RootPath + IgnoreSymlinkError bool `json:"ignore_symlink_error" default:"false" info:"Ignore symlink error"` } var config = driver.Config{ diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 70a03b983ab..493e884c151 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -30,6 +30,14 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { } _f, err := d.client.Stat(target) if err != nil { + if d.IgnoreSymlinkError { + return &model.Object{ + Name: f.Name(), + Size: f.Size(), + Modified: f.ModTime(), + IsFolder: f.IsDir(), + }, nil + } return nil, err } // set basic info From 53926d5cd0a02b3ae26c627fa6550d3ace1af845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=8D=E6=8F=92=E7=94=B5?= <69096367+r27153733@users.noreply.github.com> Date: Tue, 20 Feb 2024 19:12:07 +0800 Subject: [PATCH 129/659] fix(search): duplicate folder on autoupdate (#6063 close #6062) * fix(search): the problem of not returning in time when index does not support auto update. * fix(search): the problem of duplicate indexing of folders. --- internal/search/build.go | 17 +++++++++-------- server/handles/index.go | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/internal/search/build.go b/internal/search/build.go index a806d08fadb..1d3bfb7cd5d 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -211,14 +211,15 @@ func Update(parent string, objs []model.Obj) { } for i := range objs { if toAdd.Contains(objs[i].GetName()) { - log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) - err = Index(ctx, parent, objs[i]) - if err != nil { - log.Errorf("update search index error while index new node: %+v", err) - return - } - // build index if it's a folder - if objs[i].IsDir() { + if !objs[i].IsDir() { + log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) + err = Index(ctx, parent, objs[i]) + if err != nil { + log.Errorf("update search index error while index new node: %+v", err) + return + } + } else { + // build index if it's a folder dir := path.Join(parent, objs[i].GetName()) err = BuildIndex(ctx, []string{dir}, diff --git a/server/handles/index.go b/server/handles/index.go index 4e8babd209f..0fa1fa0e9bf 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -51,6 +51,7 @@ func UpdateIndex(c *gin.Context) { } if !search.Config(c).AutoUpdate { common.ErrorStrResp(c, "update is not supported for current index", 400) + return } go func() { ctx := context.Background() From 0c7e47a76c3aae2f2268582abca7e5176d6f8ec1 Mon Sep 17 00:00:00 2001 From: Mmx Date: Wed, 21 Feb 2024 14:04:22 +0800 Subject: [PATCH 130/659] feat: add docker image with pre-installed ffmpeg (#6054) * build: add dockerfile for ffmpeg version * ci: add docker image with ffmpeg release * fix: donnot push on docker build test --- .github/workflows/build_docker.yml | 21 ++++++++++++++++++++- .github/workflows/release_docker.yml | 19 +++++++++++++++++++ Dockerfile.ffmpeg | 4 ++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.ffmpeg diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 3b733b3ba2f..3a63630dc63 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -55,6 +55,25 @@ jobs: labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + - name: Docker meta with ffmpeg + id: meta-ffmpeg + uses: docker/metadata-action@v5 + with: + images: xhofe/alist + flavor: | + latest=true + suffix=-ffmpeg,onlatest=true + + - name: Build and push with ffmpeg + id: docker_build_ffmpeg + uses: docker/build-push-action@v5 + with: + file: Dockerfile.ffmpeg + push: ${{ github.event_name == 'push' }} + tags: ${{ steps.meta-ffmpeg.outputs.tags }} + labels: ${{ steps.meta-ffmpeg.outputs.labels }} + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + build_docker_with_aria2: needs: build_docker name: Build docker with aria2 @@ -80,4 +99,4 @@ jobs: with: github_token: ${{ secrets.MY_TOKEN }} branch: main - repository: alist-org/with_aria2 \ No newline at end of file + repository: alist-org/with_aria2 diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index b029484e9bb..10f8554220b 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -49,6 +49,25 @@ jobs: labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + - name: Docker meta with ffmpeg + id: meta-ffmpeg + uses: docker/metadata-action@v5 + with: + images: xhofe/alist + flavor: | + latest=true + suffix=-ffmpeg,onlatest=true + + - name: Build and push with ffmpeg + id: docker_build_ffmpeg + uses: docker/build-push-action@v5 + with: + file: Dockerfile.ffmpeg + push: true + tags: ${{ steps.meta-ffmpeg.outputs.tags }} + labels: ${{ steps.meta-ffmpeg.outputs.labels }} + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + release_docker_with_aria2: needs: release_docker name: Release docker with aria2 diff --git a/Dockerfile.ffmpeg b/Dockerfile.ffmpeg new file mode 100644 index 00000000000..9799d777edf --- /dev/null +++ b/Dockerfile.ffmpeg @@ -0,0 +1,4 @@ +FROM xhofe/alist:latest +RUN apk update && \ + apk add --no-cache ffmpeg \ + rm -rf /var/cache/apk/* \ No newline at end of file From 858ba19670ac54db90349192cbd97746457177eb Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 21 Feb 2024 14:58:45 +0800 Subject: [PATCH 131/659] ci: also push docker to hub for pr --- .github/workflows/build_docker.yml | 5 ++--- .github/workflows/del-pr-docker.yml | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/del-pr-docker.yml diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 3a63630dc63..989fcd75069 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -38,7 +38,6 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - if: github.event_name == 'push' uses: docker/login-action@v3 with: username: xhofe @@ -50,7 +49,7 @@ jobs: with: context: . file: Dockerfile.ci - push: ${{ github.event_name == 'push' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x @@ -69,7 +68,7 @@ jobs: uses: docker/build-push-action@v5 with: file: Dockerfile.ffmpeg - push: ${{ github.event_name == 'push' }} + push: true tags: ${{ steps.meta-ffmpeg.outputs.tags }} labels: ${{ steps.meta-ffmpeg.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x diff --git a/.github/workflows/del-pr-docker.yml b/.github/workflows/del-pr-docker.yml new file mode 100644 index 00000000000..82ae69cf894 --- /dev/null +++ b/.github/workflows/del-pr-docker.yml @@ -0,0 +1,24 @@ +name: delete closed pr docker tag + +on: + pull_request: + types: [closed] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + del_docker_tag: + name: Build Docker + runs-on: ubuntu-latest + steps: + + - name: Delete docker tag + id: del_docker_tag + uses: xhofe/del-docker-tag@main + with: + username: xhofe + password: ${{ secrets.DOCKERHUB_TOKEN }} + # token: ${{ secrets.DOCKER_TOKEN }} + tags: xhofe/alist:pr-${{ github.event.pull_request.number }},xhofe/alist:pr-${{ github.event.pull_request.number }}-ffmpeg \ No newline at end of file From 424ab2d0c06a04ca1bb4b109a5ba0d6bc1f0cf3b Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 21 Feb 2024 15:50:05 +0800 Subject: [PATCH 132/659] ci: remove docker latest tag on dev --- .github/workflows/build_docker.yml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 989fcd75069..ec56d426ce1 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -18,6 +18,20 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: xhofe/alist + + - name: Docker meta with ffmpeg + id: meta-ffmpeg + uses: docker/metadata-action@v5 + with: + images: xhofe/alist + flavor: | + suffix=-ffmpeg,onlatest=true + - uses: actions/setup-go@v4 with: go-version: 'stable' @@ -25,12 +39,6 @@ jobs: - name: Build go binary run: bash build.sh dev docker-multiplatform - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: xhofe/alist - - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -54,15 +62,6 @@ jobs: labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x - - name: Docker meta with ffmpeg - id: meta-ffmpeg - uses: docker/metadata-action@v5 - with: - images: xhofe/alist - flavor: | - latest=true - suffix=-ffmpeg,onlatest=true - - name: Build and push with ffmpeg id: docker_build_ffmpeg uses: docker/build-push-action@v5 From 1f835502ba54bcf0cb635524e8cb6da825e38a13 Mon Sep 17 00:00:00 2001 From: Mars160 <74127225+Mars160@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:28:48 +0800 Subject: [PATCH 133/659] feat: support customize dsn for mysql and pg (#6031) * support for unixsocket to connect to mysql * feat: customize dsn for mysql and pg --------- Co-authored-by: Andy Hsu --- internal/bootstrap/db.go | 7 +++++++ internal/conf/config.go | 1 + 2 files changed, 8 insertions(+) diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index 4c4044f19e3..5dfa2820d18 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -56,14 +56,21 @@ func InitDB() { } case "mysql": { + //[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local&tls=%s", database.User, database.Password, database.Host, database.Port, database.Name, database.SSLMode) + if database.DSN != "" { + dsn = database.DSN + } dB, err = gorm.Open(mysql.Open(dsn), gormConfig) } case "postgres": { dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", database.Host, database.User, database.Password, database.Name, database.Port, database.SSLMode) + if database.DSN != "" { + dsn = database.DSN + } dB, err = gorm.Open(postgres.Open(dsn), gormConfig) } default: diff --git a/internal/conf/config.go b/internal/conf/config.go index 92761ea7e90..b4664562035 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -17,6 +17,7 @@ type Database struct { DBFile string `json:"db_file" env:"FILE"` TablePrefix string `json:"table_prefix" env:"TABLE_PREFIX"` SSLMode string `json:"ssl_mode" env:"SSL_MODE"` + DSN string `json:"dsn" env:"DSN"` } type Scheme struct { From f1979a8bbccb514589700db6d4a9791d56460bda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=8D=E6=8F=92=E7=94=B5?= <69096367+r27153733@users.noreply.github.com> Date: Fri, 23 Feb 2024 15:37:40 +0800 Subject: [PATCH 134/659] feat(search): search with `meilisearch` (#6060) * feat(search): search with meilisearch. * feat(search): meilisearch supports auto update. * chores: remove utils.Log. * fix(search): the null pointer caused by deleting non-existing file/folder indexes. --------- Co-authored-by: Andy Hsu --- go.mod | 7 + go.sum | 21 +++ internal/bootstrap/data/setting.go | 2 +- internal/conf/config.go | 13 +- internal/search/import.go | 1 + internal/search/meilisearch/init.go | 89 ++++++++++ internal/search/meilisearch/search.go | 227 ++++++++++++++++++++++++++ pkg/utils/slice.go | 20 +++ 8 files changed, 377 insertions(+), 3 deletions(-) create mode 100644 internal/search/meilisearch/init.go create mode 100644 internal/search/meilisearch/search.go diff --git a/go.mod b/go.mod index 81f71d5291f..a2953ad78cc 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 github.com/maruel/natural v1.1.1 + github.com/meilisearch/meilisearch-go v0.26.1 github.com/minio/sio v0.3.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 @@ -73,6 +74,7 @@ require ( github.com/abbot/go-http-auth v0.4.0 // indirect github.com/aead/ecdh v0.2.0 // indirect github.com/andreburgaud/crypt2go v1.2.0 // indirect + github.com/andybalholm/brotli v1.0.4 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -133,7 +135,9 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect @@ -142,6 +146,7 @@ require ( github.com/libp2p/go-libp2p v0.27.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -189,6 +194,8 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u2takey/go-utils v0.3.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect diff --git a/go.sum b/go.sum index f3781212a81..9ad983a75b7 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/Hcta4l4VrIXrI= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -247,6 +249,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= @@ -254,6 +258,10 @@ github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= @@ -284,6 +292,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= @@ -301,6 +311,8 @@ github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOj github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/meilisearch/meilisearch-go v0.26.1 h1:3bmo2uLijX7kvBmiZ9LupVfC95TFcRJDgrRTzbOoE4A= +github.com/meilisearch/meilisearch-go v0.26.1/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= @@ -449,8 +461,13 @@ github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4d github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -478,6 +495,7 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= @@ -497,6 +515,7 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -524,6 +543,8 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 0aee410aab5..4fd99ca2383 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -145,7 +145,7 @@ func InitialSettings() []model.SettingItem { // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX}, + {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,meilisearch,none", Group: model.INDEX}, {Key: conf.AutoUpdateIndex, Value: "false", Type: conf.TypeBool, Group: model.INDEX}, {Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.MaxIndexDepth, Value: "20", Type: conf.TypeNumber, Group: model.INDEX, Flag: model.PRIVATE, Help: `max depth of index`}, diff --git a/internal/conf/config.go b/internal/conf/config.go index b4664562035..0f1e0048c75 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -1,10 +1,9 @@ package conf import ( - "path/filepath" - "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/pkg/utils/random" + "path/filepath" ) type Database struct { @@ -20,6 +19,12 @@ type Database struct { DSN string `json:"dsn" env:"DSN"` } +type Meilisearch struct { + Host string `json:"host" env:"HOST"` + APIKey string `json:"api_key" env:"API_KEY"` + IndexPrefix string `json:"index_prefix" env:"INDEX_PREFIX"` +} + type Scheme struct { Address string `json:"address" env:"ADDR"` HttpPort int `json:"http_port" env:"HTTP_PORT"` @@ -65,6 +70,7 @@ type Config struct { JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` Database Database `json:"database" envPrefix:"DB_"` + Meilisearch Meilisearch `json:"meilisearch" env:"MEILISEARCH"` Scheme Scheme `json:"scheme"` TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` @@ -101,6 +107,9 @@ func DefaultConfig() *Config { TablePrefix: "x_", DBFile: dbPath, }, + Meilisearch: Meilisearch{ + Host: "http://localhost:7700", + }, BleveDir: indexDir, Log: LogConfig{ Enable: true, diff --git a/internal/search/import.go b/internal/search/import.go index 822ae3f793f..a34c36f9a34 100644 --- a/internal/search/import.go +++ b/internal/search/import.go @@ -4,4 +4,5 @@ import ( _ "github.com/alist-org/alist/v3/internal/search/bleve" _ "github.com/alist-org/alist/v3/internal/search/db" _ "github.com/alist-org/alist/v3/internal/search/db_non_full_text" + _ "github.com/alist-org/alist/v3/internal/search/meilisearch" ) diff --git a/internal/search/meilisearch/init.go b/internal/search/meilisearch/init.go new file mode 100644 index 00000000000..8f5f24733ee --- /dev/null +++ b/internal/search/meilisearch/init.go @@ -0,0 +1,89 @@ +package meilisearch + +import ( + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/meilisearch/meilisearch-go" +) + +var config = searcher.Config{ + Name: "meilisearch", + AutoUpdate: true, +} + +func init() { + searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { + m := Meilisearch{ + Client: meilisearch.NewClient(meilisearch.ClientConfig{ + Host: conf.Conf.Meilisearch.Host, + APIKey: conf.Conf.Meilisearch.APIKey, + }), + IndexUid: conf.Conf.Meilisearch.IndexPrefix + "alist", + FilterableAttributes: []string{"parent", "is_dir", "name"}, + SearchableAttributes: []string{"name"}, + } + + _, err := m.Client.GetIndex(m.IndexUid) + if err != nil { + var mErr *meilisearch.Error + ok := errors.As(err, &mErr) + if ok && mErr.MeilisearchApiError.Code == "index_not_found" { + task, err := m.Client.CreateIndex(&meilisearch.IndexConfig{ + Uid: m.IndexUid, + PrimaryKey: "id", + }) + if err != nil { + return nil, err + } + forTask, err := m.Client.WaitForTask(task.TaskUID) + if err != nil { + return nil, err + } + if forTask.Status != meilisearch.TaskStatusSucceeded { + return nil, fmt.Errorf("index creation failed, task status is %s", forTask.Status) + } + } else { + return nil, err + } + } + attributes, err := m.Client.Index(m.IndexUid).GetFilterableAttributes() + if err != nil { + return nil, err + } + if attributes == nil || !utils.SliceAllContains(*attributes, m.FilterableAttributes...) { + _, err = m.Client.Index(m.IndexUid).UpdateFilterableAttributes(&m.FilterableAttributes) + if err != nil { + return nil, err + } + } + + attributes, err = m.Client.Index(m.IndexUid).GetSearchableAttributes() + if err != nil { + return nil, err + } + if attributes == nil || !utils.SliceAllContains(*attributes, m.SearchableAttributes...) { + _, err = m.Client.Index(m.IndexUid).UpdateSearchableAttributes(&m.SearchableAttributes) + if err != nil { + return nil, err + } + } + + pagination, err := m.Client.Index(m.IndexUid).GetPagination() + if err != nil { + return nil, err + } + if pagination.MaxTotalHits != int64(model.MaxInt) { + _, err := m.Client.Index(m.IndexUid).UpdatePagination(&meilisearch.Pagination{ + MaxTotalHits: int64(model.MaxInt), + }) + if err != nil { + return nil, err + } + } + return &m, nil + }) +} diff --git a/internal/search/meilisearch/search.go b/internal/search/meilisearch/search.go new file mode 100644 index 00000000000..1516306b75f --- /dev/null +++ b/internal/search/meilisearch/search.go @@ -0,0 +1,227 @@ +package meilisearch + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/google/uuid" + "github.com/meilisearch/meilisearch-go" + "path" + "strings" + "time" +) + +type searchDocument struct { + ID string `json:"id"` + model.SearchNode +} + +type Meilisearch struct { + Client *meilisearch.Client + IndexUid string + FilterableAttributes []string + SearchableAttributes []string +} + +func (m *Meilisearch) Config() searcher.Config { + return config +} + +func (m *Meilisearch) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + mReq := &meilisearch.SearchRequest{ + AttributesToSearchOn: m.SearchableAttributes, + Page: int64(req.Page), + HitsPerPage: int64(req.PerPage), + } + if req.Scope != 0 { + mReq.Filter = fmt.Sprintf("is_dir = %v", req.Scope == 1) + } + search, err := m.Client.Index(m.IndexUid).Search(req.Keywords, mReq) + if err != nil { + return nil, 0, err + } + nodes, err := utils.SliceConvert(search.Hits, func(src any) (model.SearchNode, error) { + srcMap := src.(map[string]any) + return model.SearchNode{ + Parent: srcMap["parent"].(string), + Name: srcMap["name"].(string), + IsDir: srcMap["is_dir"].(bool), + Size: int64(srcMap["size"].(float64)), + }, nil + }) + if err != nil { + return nil, 0, err + } + return nodes, search.TotalHits, nil +} + +func (m *Meilisearch) Index(ctx context.Context, node model.SearchNode) error { + return m.BatchIndex(ctx, []model.SearchNode{node}) +} + +func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { + documents, _ := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { + + return &searchDocument{ + ID: uuid.NewString(), + SearchNode: src, + }, nil + }) + + _, err := m.Client.Index(m.IndexUid).AddDocuments(documents) + if err != nil { + return err + } + + //// Wait for the task to complete and check + //forTask, err := m.Client.WaitForTask(task.TaskUID, meilisearch.WaitParams{ + // Context: ctx, + // Interval: time.Millisecond * 50, + //}) + //if err != nil { + // return err + //} + //if forTask.Status != meilisearch.TaskStatusSucceeded { + // return fmt.Errorf("BatchIndex failed, task status is %s", forTask.Status) + //} + return nil +} + +func (m *Meilisearch) getDocumentsByParent(ctx context.Context, parent string) ([]*searchDocument, error) { + var result meilisearch.DocumentsResult + err := m.Client.Index(m.IndexUid).GetDocuments(&meilisearch.DocumentsQuery{ + Filter: fmt.Sprintf("parent = '%s'", strings.ReplaceAll(parent, "'", "\\'")), + Limit: int64(model.MaxInt), + }, &result) + if err != nil { + return nil, err + } + return utils.SliceConvert(result.Results, func(src map[string]any) (*searchDocument, error) { + return &searchDocument{ + ID: src["id"].(string), + SearchNode: model.SearchNode{ + Parent: src["parent"].(string), + Name: src["name"].(string), + IsDir: src["is_dir"].(bool), + Size: int64(src["size"].(float64)), + }, + }, nil + }) +} + +func (m *Meilisearch) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { + result, err := m.getDocumentsByParent(ctx, parent) + if err != nil { + return nil, err + } + return utils.SliceConvert(result, func(src *searchDocument) (model.SearchNode, error) { + return src.SearchNode, nil + }) + +} + +func (m *Meilisearch) getParentsByPrefix(ctx context.Context, parent string) ([]string, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + parents := []string{parent} + get, err := m.getDocumentsByParent(ctx, parent) + if err != nil { + return nil, err + } + for _, node := range get { + if node.IsDir { + arr, err := m.getParentsByPrefix(ctx, path.Join(node.Parent, node.Name)) + if err != nil { + return nil, err + } + parents = append(parents, arr...) + } + } + return parents, nil + } +} + +func (m *Meilisearch) DelDirChild(ctx context.Context, prefix string) error { + dfs, err := m.getParentsByPrefix(ctx, utils.FixAndCleanPath(prefix)) + if err != nil { + return err + } + utils.SliceReplace(dfs, func(src string) string { + return "'" + strings.ReplaceAll(src, "'", "\\'") + "'" + }) + s := fmt.Sprintf("parent IN [%s]", strings.Join(dfs, ",")) + task, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilter(s) + if err != nil { + return err + } + taskStatus, err := m.getTaskStatus(ctx, task.TaskUID) + if err != nil { + return err + } + if taskStatus != meilisearch.TaskStatusSucceeded { + return fmt.Errorf("DelDir failed, task status is %s", taskStatus) + } + return nil +} + +func (m *Meilisearch) Del(ctx context.Context, prefix string) error { + prefix = utils.FixAndCleanPath(prefix) + dir, name := path.Split(prefix) + get, err := m.getDocumentsByParent(ctx, dir[:len(dir)-1]) + if err != nil { + return err + } + var document *searchDocument + for _, v := range get { + if v.Name == name { + document = v + break + } + } + if document == nil { + // Defensive programming. Document may be the folder, try deleting Child + return m.DelDirChild(ctx, prefix) + } + if document.IsDir { + err = m.DelDirChild(ctx, prefix) + if err != nil { + return err + } + } + task, err := m.Client.Index(m.IndexUid).DeleteDocument(document.ID) + if err != nil { + return err + } + taskStatus, err := m.getTaskStatus(ctx, task.TaskUID) + if err != nil { + return err + } + if taskStatus != meilisearch.TaskStatusSucceeded { + return fmt.Errorf("DelDir failed, task status is %s", taskStatus) + } + return nil +} + +func (m *Meilisearch) Release(ctx context.Context) error { + return nil +} + +func (m *Meilisearch) Clear(ctx context.Context) error { + _, err := m.Client.Index(m.IndexUid).DeleteAllDocuments() + return err +} + +func (m *Meilisearch) getTaskStatus(ctx context.Context, taskUID int64) (meilisearch.TaskStatus, error) { + forTask, err := m.Client.WaitForTask(taskUID, meilisearch.WaitParams{ + Context: ctx, + Interval: time.Second, + }) + if err != nil { + return meilisearch.TaskStatusUnknown, err + } + return forTask.Status, nil +} diff --git a/pkg/utils/slice.go b/pkg/utils/slice.go index 73bac93b4d0..842995daaf1 100644 --- a/pkg/utils/slice.go +++ b/pkg/utils/slice.go @@ -29,6 +29,20 @@ func SliceContains[T comparable](arr []T, v T) bool { return false } +// SliceAllContains check if slice all contains elements +func SliceAllContains[T comparable](arr []T, vs ...T) bool { + vsMap := make(map[T]struct{}) + for _, v := range arr { + vsMap[v] = struct{}{} + } + for _, v := range vs { + if _, ok := vsMap[v]; !ok { + return false + } + } + return true +} + // SliceConvert convert slice to another type slice func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D, error) { res := make([]D, 0, len(srcS)) @@ -79,3 +93,9 @@ func SliceFilter[T any](arr []T, filter func(src T) bool) []T { } return res } + +func SliceReplace[T any](arr []T, replace func(src T) T) { + for i, src := range arr { + arr[i] = replace(src) + } +} From 742335f80ecfd0fe44cc71cb16dec6c4fb6dade6 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 23 Feb 2024 15:42:52 +0800 Subject: [PATCH 135/659] fix: don't push docker on pr due to security --- .github/workflows/build_docker.yml | 5 +++-- .github/workflows/del-pr-docker.yml | 24 ------------------------ 2 files changed, 3 insertions(+), 26 deletions(-) delete mode 100644 .github/workflows/del-pr-docker.yml diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index ec56d426ce1..dbd48976fe8 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -46,6 +46,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to DockerHub + if: github.event_name == 'push' uses: docker/login-action@v3 with: username: xhofe @@ -57,7 +58,7 @@ jobs: with: context: . file: Dockerfile.ci - push: true + push: ${{ github.event_name == 'push' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x @@ -67,7 +68,7 @@ jobs: uses: docker/build-push-action@v5 with: file: Dockerfile.ffmpeg - push: true + push: ${{ github.event_name == 'push' }} tags: ${{ steps.meta-ffmpeg.outputs.tags }} labels: ${{ steps.meta-ffmpeg.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x diff --git a/.github/workflows/del-pr-docker.yml b/.github/workflows/del-pr-docker.yml deleted file mode 100644 index 82ae69cf894..00000000000 --- a/.github/workflows/del-pr-docker.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: delete closed pr docker tag - -on: - pull_request: - types: [closed] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - del_docker_tag: - name: Build Docker - runs-on: ubuntu-latest - steps: - - - name: Delete docker tag - id: del_docker_tag - uses: xhofe/del-docker-tag@main - with: - username: xhofe - password: ${{ secrets.DOCKERHUB_TOKEN }} - # token: ${{ secrets.DOCKER_TOKEN }} - tags: xhofe/alist:pr-${{ github.event.pull_request.number }},xhofe/alist:pr-${{ github.event.pull_request.number }}-ffmpeg \ No newline at end of file From e66abb3f5899b0970006322a249b71c57c1e5c64 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 23 Feb 2024 20:56:14 +0800 Subject: [PATCH 136/659] fix(deps): update module github.com/aws/aws-sdk-go to v1.50.24 (#5873) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a2953ad78cc..3fad42257c5 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Xhofe/wopan-sdk-go v0.1.2 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.49.18 + github.com/aws/aws-sdk-go v1.50.24 github.com/blevesearch/bleve/v2 v2.3.10 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.17.1 diff --git a/go.sum b/go.sum index 9ad983a75b7..c1442c3b057 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.49.18 h1:g/iMXkfXeJQ7MvnLwroxWsTTNkHtdVJGxIgrAIEG62M= github.com/aws/aws-sdk-go v1.49.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.50.24 h1:3o2Pg7mOoVL0jv54vWtuafoZqAeEXLhm1tltWA2GcEw= +github.com/aws/aws-sdk-go v1.50.24/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= From 94a80bccfeb70c2a51d1451fd65acaf8550a6f5d Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 24 Feb 2024 18:04:08 +0800 Subject: [PATCH 137/659] fix(feiji): unable to get link (close #6082) --- drivers/ilanzou/driver.go | 17 ++++++++++++++--- drivers/ilanzou/meta.go | 33 ++++++++++++++++++--------------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 080e3b55129..875300e1820 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -123,14 +123,14 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) query.Set("devType", "6") query.Set("devCode", d.UUID) query.Set("devModel", "chrome") - query.Set("devVersion", "120") + query.Set("devVersion", d.conf.devVersion) query.Set("appVersion", "") ts, err := getTimestamp(d.conf.secret) if err != nil { return nil, err } query.Set("timestamp", ts) - //query.Set("appToken", d.Token) + query.Set("appToken", d.Token) query.Set("enable", "1") downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) if err != nil { @@ -143,7 +143,18 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } query.Set("auth", hex.EncodeToString(auth)) u.RawQuery = query.Encode() - link := model.Link{URL: u.String()} + realURL := u.String() + // get the url after redirect + res, err := base.NoRedirectClient.R().Get(realURL) + if err != nil { + return nil, err + } + if res.StatusCode() == 302 { + realURL = res.Header().Get("location") + } else { + return nil, fmt.Errorf("redirect failed, status: %d", res.StatusCode()) + } + link := model.Link{URL: realURL} return &link, nil } diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go index f7c61e5a7fe..ca813c5efad 100644 --- a/drivers/ilanzou/meta.go +++ b/drivers/ilanzou/meta.go @@ -15,11 +15,12 @@ type Addition struct { } type Conf struct { - base string - secret []byte - bucket string - unproved string - proved string + base string + secret []byte + bucket string + unproved string + proved string + devVersion string } func init() { @@ -39,11 +40,12 @@ func init() { NoOverwriteUpload: false, }, conf: Conf{ - base: "https://api.ilanzou.com", - secret: []byte("lanZouY-disk-app"), - bucket: "wpanstore-lanzou", - unproved: "unproved", - proved: "proved", + base: "https://api.ilanzou.com", + secret: []byte("lanZouY-disk-app"), + bucket: "wpanstore-lanzou", + unproved: "unproved", + proved: "proved", + devVersion: "120", }, } }) @@ -63,11 +65,12 @@ func init() { NoOverwriteUpload: false, }, conf: Conf{ - base: "https://api.feijipan.com", - secret: []byte("dingHao-disk-app"), - bucket: "wpanstore", - unproved: "ws", - proved: "app", + base: "https://api.feijipan.com", + secret: []byte("dingHao-disk-app"), + bucket: "wpanstore", + unproved: "ws", + proved: "app", + devVersion: "121", }, } }) From 7e6522c81e382e86e10dab86acd3c3b53b9cafc5 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 24 Feb 2024 18:10:45 +0800 Subject: [PATCH 138/659] ci: build ffmpeg image with dev version --- .github/workflows/build_docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index dbd48976fe8..baedcaab693 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -63,6 +63,10 @@ jobs: labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + - name: Replace dockerfile tag + run: | + sed -i -e "s/latest/main/g" Dockerfile.ffmpeg + - name: Build and push with ffmpeg id: docker_build_ffmpeg uses: docker/build-push-action@v5 From 71e4e1ab6eb3539df0196d6d7068084ac3e0b7b4 Mon Sep 17 00:00:00 2001 From: Kevin Z Date: Thu, 29 Feb 2024 22:37:09 -0700 Subject: [PATCH 139/659] fix(chaoxing): json cannot unmarshal content.uploadDate (close #6119 in #6124) --- drivers/chaoxing/types.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go index ba636ec1dd0..c3074bbb721 100644 --- a/drivers/chaoxing/types.go +++ b/drivers/chaoxing/types.go @@ -139,7 +139,7 @@ type File struct { Topsort int `json:"topsort"` Restype string `json:"restype"` Size int_str `json:"size"` - UploadDate string `json:"uploadDate"` + UploadDate int64 `json:"uploadDate"` FileSize string `json:"fileSize"` Name string `json:"name"` FileID string `json:"fileId"` @@ -265,10 +265,7 @@ func fileToObj(f File) *model.Object { IsFolder: true, } } - paserTime, err := time.Parse("2006-01-02 15:04", f.Content.UploadDate) - if err != nil { - paserTime = time.Now() - } + paserTime := time.UnixMilli(f.Content.UploadDate) return &model.Object{ ID: fmt.Sprintf("%d$%s", f.ID, f.Content.FileID), Name: f.Content.Name, From f8b1f87a5fec23bf20485c409c0d216c4ab8e7fc Mon Sep 17 00:00:00 2001 From: wolfsilver Date: Sat, 2 Mar 2024 14:59:55 +0800 Subject: [PATCH 140/659] fix: support for Microsoft WebDAV (#6133 close #6104) * Add support for Microsoft WebDAV * add import --- server/webdav/prop.go | 5 +++++ server/webdav/webdav.go | 3 +++ 2 files changed, 8 insertions(+) diff --git a/server/webdav/prop.go b/server/webdav/prop.go index 466644a0eef..b1474ea3e95 100644 --- a/server/webdav/prop.go +++ b/server/webdav/prop.go @@ -14,6 +14,7 @@ import ( "net/http" "path" "strconv" + "strings" "time" "github.com/alist-org/alist/v3/internal/model" @@ -385,6 +386,10 @@ func findLastModified(ctx context.Context, ls LockSystem, name string, fi model. return fi.ModTime().UTC().Format(http.TimeFormat), nil } func findCreationDate(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { + userAgent := ctx.Value("userAgent").(string) + if strings.Contains(strings.ToLower(userAgent), "microsoft-webdav") { + return fi.CreateTime().UTC().Format(http.TimeFormat), nil + } return fi.CreateTime().UTC().Format(time.RFC3339), nil } diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 509a7f1cf6d..390e5409976 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -6,6 +6,7 @@ package webdav // import "golang.org/x/net/webdav" import ( + "context" "errors" "fmt" "net/http" @@ -619,6 +620,8 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status return status, err } ctx := r.Context() + userAgent := r.Header.Get("User-Agent") + ctx = context.WithValue(ctx, "userAgent", userAgent) user := ctx.Value("user").(*model.User) reqPath, err = user.JoinPath(reqPath) if err != nil { From d0f88bd1cbae88b689811bc175c891745186bb86 Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Sat, 2 Mar 2024 15:35:10 +0800 Subject: [PATCH 141/659] feat: s3 server support (#6088 close #5186) Currently tested: List, Get, Remove --- go.mod | 14 +- go.sum | 28 +- internal/bootstrap/data/setting.go | 6 + internal/conf/const.go | 6 + internal/model/setting.go | 1 + server/router.go | 1 + server/s3.go | 29 ++ server/s3/backend.go | 432 +++++++++++++++++++++++++++++ server/s3/ioutils.go | 36 +++ server/s3/list.go | 53 ++++ server/s3/logger.go | 27 ++ server/s3/pager.go | 67 +++++ server/s3/server.go | 27 ++ server/s3/utils.go | 164 +++++++++++ 14 files changed, 875 insertions(+), 16 deletions(-) create mode 100644 server/s3.go create mode 100644 server/s3/backend.go create mode 100644 server/s3/ioutils.go create mode 100644 server/s3/list.go create mode 100644 server/s3/logger.go create mode 100644 server/s3/pager.go create mode 100644 server/s3/server.go create mode 100644 server/s3/utils.go diff --git a/go.mod b/go.mod index 3fad42257c5..7fc142d44b4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/alist-org/alist/v3 go 1.21 require ( + github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 github.com/SheltonZhu/115driver v1.0.22 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 @@ -39,6 +40,7 @@ require ( github.com/meilisearch/meilisearch-go v0.26.1 github.com/minio/sio v0.3.0 github.com/natefinch/lumberjack v2.0.0+incompatible + github.com/ncw/swift/v2 v2.0.2 github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 @@ -137,8 +139,8 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect - github.com/klauspost/compress v1.16.5 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect @@ -153,7 +155,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -172,7 +174,6 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/ncw/swift/v2 v2.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect @@ -184,6 +185,8 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/rfjakob/eme v1.1.2 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect @@ -201,10 +204,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.5.0 // indirect - golang.org/x/sync v0.3.0 // indirect + golang.org/x/sync v0.5.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect + golang.org/x/tools v0.16.0 // indirect google.golang.org/api v0.134.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/grpc v1.57.0 // indirect diff --git a/go.sum b/go.sum index c1442c3b057..008dfa92708 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= +github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q= +github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U= @@ -32,8 +34,6 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.49.18 h1:g/iMXkfXeJQ7MvnLwroxWsTTNkHtdVJGxIgrAIEG62M= -github.com/aws/aws-sdk-go v1.49.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.50.24 h1:3o2Pg7mOoVL0jv54vWtuafoZqAeEXLhm1tltWA2GcEw= github.com/aws/aws-sdk-go v1.50.24/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -262,12 +262,11 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -315,8 +314,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/meilisearch/meilisearch-go v0.26.1 h1:3bmo2uLijX7kvBmiZ9LupVfC95TFcRJDgrRTzbOoE4A= github.com/meilisearch/meilisearch-go v0.26.1/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -408,6 +407,10 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= +github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -531,8 +534,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -589,9 +592,12 @@ golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 4fd99ca2383..8adc713d6d6 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -176,6 +176,12 @@ func InitialSettings() []model.SettingItem { {Key: conf.LdapDefaultDir, Value: "/", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapDefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapLoginTips, Value: "login with ldap", Type: conf.TypeString, Group: model.LDAP, Flag: model.PUBLIC}, + + //s3 settings + {Key: conf.S3Enabled, Value: "false", Type: conf.TypeBool, Group: model.S3, Flag: model.PRIVATE}, + {Key: conf.S3AccessKeyId, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, + {Key: conf.S3SecretAccessKey, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, + {Key: conf.S3Buckets, Value: "[]", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, } initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { diff --git a/internal/conf/const.go b/internal/conf/const.go index 5ffdef2b577..a5d95e5d8a9 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -84,6 +84,12 @@ const ( LdapDefaultDir = "ldap_default_dir" LdapLoginTips = "ldap_login_tips" + //s3 + S3Enabled = "s3_enabled" + S3AccessKeyId = "s3_access_key_id" + S3SecretAccessKey = "s3_secret_access_key" + S3Buckets = "s3_buckets" + // qbittorrent QbittorrentUrl = "qbittorrent_url" QbittorrentSeedtime = "qbittorrent_seedtime" diff --git a/internal/model/setting.go b/internal/model/setting.go index b561ad6b221..1a47cf5c062 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -10,6 +10,7 @@ const ( INDEX SSO LDAP + S3 ) const ( diff --git a/server/router.go b/server/router.go index 1421f66595d..b0b66294272 100644 --- a/server/router.go +++ b/server/router.go @@ -36,6 +36,7 @@ func Init(e *gin.Engine) { g.Use(middlewares.MaxAllowed(conf.Conf.MaxConnections)) } WebDav(g.Group("/dav")) + S3(g.Group("/s3")) g.GET("/d/*path", middlewares.Down, handles.Down) g.GET("/p/*path", middlewares.Down, handles.Proxy) diff --git a/server/s3.go b/server/s3.go new file mode 100644 index 00000000000..5a70cf2aa76 --- /dev/null +++ b/server/s3.go @@ -0,0 +1,29 @@ +package server + +import ( + "context" + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/server/common" + "github.com/alist-org/alist/v3/server/s3" + "github.com/gin-gonic/gin" +) + +func S3(g *gin.RouterGroup) { + if !setting.GetBool(conf.S3Enabled) { + g.Any("/*path", func(c *gin.Context) { + common.ErrorStrResp(c, "S3 server is not enabled", 403) + }) + return + } + h, _ := s3.NewServer(context.Background(), []string{setting.GetStr(conf.S3AccessKeyId) + "," + setting.GetStr(conf.S3SecretAccessKey)}) + + g.Any("/*path", func(c *gin.Context) { + adjustedPath := strings.TrimPrefix(c.Request.URL.Path, path.Join(conf.URL.Path, "/s3")) + c.Request.URL.Path = adjustedPath + gin.WrapH(h)(c) + }) +} diff --git a/server/s3/backend.go b/server/s3/backend.go new file mode 100644 index 00000000000..c73405252c7 --- /dev/null +++ b/server/s3/backend.go @@ -0,0 +1,432 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "context" + "encoding/hex" + "fmt" + "io" + "path" + "strings" + "sync" + "time" + + "github.com/Mikubill/gofakes3" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/ncw/swift/v2" +) + +var ( + emptyPrefix = &gofakes3.Prefix{} + timeFormat = "Mon, 2 Jan 2006 15:04:05.999999999 GMT" +) + +// s3Backend implements the gofacess3.Backend interface to make an S3 +// backend for gofakes3 +type s3Backend struct { + meta *sync.Map +} + +// newBackend creates a new SimpleBucketBackend. +func newBackend() gofakes3.Backend { + return &s3Backend{ + meta: new(sync.Map), + } +} + +// ListBuckets always returns the default bucket. +func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) { + buckets, err := getAndParseBuckets() + if err != nil { + return nil, err + } + var response []gofakes3.BucketInfo + ctx := context.Background() + for _, b := range buckets { + node, _ := fs.Get(ctx, b.Path, &fs.GetArgs{}) + response = append(response, gofakes3.BucketInfo{ + // Name: gofakes3.URLEncode(b.Name), + Name: b.Name, + CreationDate: gofakes3.NewContentTime(node.ModTime()), + }) + } + return response, nil +} + +// ListBucket lists the objects in the given bucket. +func (b *s3Backend) ListBucket(bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { + bucket, err := getBucketByName(bucketName) + if err != nil { + return nil, err + } + bucketPath := bucket.Path + + if prefix == nil { + prefix = emptyPrefix + } + + // workaround + if strings.TrimSpace(prefix.Prefix) == "" { + prefix.HasPrefix = false + } + if strings.TrimSpace(prefix.Delimiter) == "" { + prefix.HasDelimiter = false + } + + response := gofakes3.NewObjectList() + path, remaining := prefixParser(prefix) + + err = b.entryListR(bucketPath, path, remaining, prefix.HasDelimiter, response) + if err == gofakes3.ErrNoSuchKey { + // AWS just returns an empty list + response = gofakes3.NewObjectList() + } else if err != nil { + return nil, err + } + + return b.pager(response, page) +} + +// HeadObject returns the fileinfo for the given object name. +// +// Note that the metadata is not supported yet. +func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object, error) { + ctx := context.Background() + bucket, err := getBucketByName(bucketName) + if err != nil { + return nil, err + } + bucketPath := bucket.Path + + fp := path.Join(bucketPath, objectName) + fmeta, _ := op.GetNearestMeta(fp) + node, err := fs.Get(context.WithValue(ctx, "meta", fmeta), fp, &fs.GetArgs{}) + if err != nil { + return nil, gofakes3.KeyNotFound(objectName) + } + + if node.IsDir() { + return nil, gofakes3.KeyNotFound(objectName) + } + + size := node.GetSize() + // hash := getFileHashByte(fobj) + + meta := map[string]string{ + "Last-Modified": node.ModTime().Format(timeFormat), + "Content-Type": utils.GetMimeType(fp), + } + + if val, ok := b.meta.Load(fp); ok { + metaMap := val.(map[string]string) + for k, v := range metaMap { + meta[k] = v + } + } + + return &gofakes3.Object{ + Name: objectName, + // Hash: hash, + Metadata: meta, + Size: size, + Contents: noOpReadCloser{}, + }, nil +} + +// GetObject fetchs the object from the filesystem. +func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) { + ctx := context.Background() + bucket, err := getBucketByName(bucketName) + if err != nil { + return nil, err + } + bucketPath := bucket.Path + + fp := path.Join(bucketPath, objectName) + fmeta, _ := op.GetNearestMeta(fp) + node, err := fs.Get(context.WithValue(ctx, "meta", fmeta), fp, &fs.GetArgs{}) + if err != nil { + return nil, gofakes3.KeyNotFound(objectName) + } + + if node.IsDir() { + return nil, gofakes3.KeyNotFound(objectName) + } + + link, file, err := fs.Link(ctx, fp, model.LinkArgs{}) + if err != nil { + return nil, err + } + + size := file.GetSize() + rnge, err := rangeRequest.Range(size) + if err != nil { + return nil, err + } + + if link.RangeReadCloser == nil && link.MFile == nil && len(link.URL) == 0 { + return nil, fmt.Errorf("the remote storage driver need to be enhanced to support s3") + } + remoteFileSize := file.GetSize() + remoteClosers := utils.EmptyClosers() + rangeReaderFunc := func(ctx context.Context, start, length int64) (io.ReadCloser, error) { + if length >= 0 && start+length >= remoteFileSize { + length = -1 + } + rrc := link.RangeReadCloser + if len(link.URL) > 0 { + + rangedRemoteLink := &model.Link{ + URL: link.URL, + Header: link.Header, + } + var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, rangedRemoteLink) + if err != nil { + return nil, err + } + rrc = converted + } + if rrc != nil { + remoteReader, err := rrc.RangeRead(ctx, http_range.Range{Start: start, Length: length}) + remoteClosers.AddClosers(rrc.GetClosers()) + if err != nil { + return nil, err + } + return remoteReader, nil + } + if link.MFile != nil { + _, err := link.MFile.Seek(start, io.SeekStart) + if err != nil { + return nil, err + } + //remoteClosers.Add(remoteLink.MFile) + //keep reuse same MFile and close at last. + remoteClosers.Add(link.MFile) + return io.NopCloser(link.MFile), nil + } + return nil, errs.NotSupport + } + + var rdr io.ReadCloser + if rnge != nil { + rdr, err = rangeReaderFunc(ctx, rnge.Start, rnge.Length) + if err != nil { + return nil, err + } + } else { + rdr, err = rangeReaderFunc(ctx, 0, -1) + if err != nil { + return nil, err + } + } + + meta := map[string]string{ + "Last-Modified": node.ModTime().Format(timeFormat), + "Content-Type": utils.GetMimeType(fp), + } + + if val, ok := b.meta.Load(fp); ok { + metaMap := val.(map[string]string) + for k, v := range metaMap { + meta[k] = v + } + } + + return &gofakes3.Object{ + // Name: gofakes3.URLEncode(objectName), + Name: objectName, + // Hash: "", + Metadata: meta, + Size: size, + Range: rnge, + Contents: rdr, + }, nil +} + +// TouchObject creates or updates meta on specified object. +func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) { + //TODO: implement + return result, gofakes3.ErrNotImplemented +} + +// PutObject creates or overwrites the object with the given name. +func (b *s3Backend) PutObject( + bucketName, objectName string, + meta map[string]string, + input io.Reader, size int64, +) (result gofakes3.PutObjectResult, err error) { + ctx := context.Background() + bucket, err := getBucketByName(bucketName) + if err != nil { + return result, err + } + bucketPath := bucket.Path + + fp := path.Join(bucketPath, objectName) + reqPath := path.Dir(fp) + fmeta, _ := op.GetNearestMeta(fp) + _, err = fs.Get(context.WithValue(ctx, "meta", fmeta), reqPath, &fs.GetArgs{}) + if err != nil { + return result, gofakes3.KeyNotFound(objectName) + } + + var ti time.Time + + if val, ok := meta["X-Amz-Meta-Mtime"]; ok { + ti, _ = swift.FloatStringToTime(val) + } + + if val, ok := meta["mtime"]; ok { + ti, _ = swift.FloatStringToTime(val) + } + + obj := model.Object{ + Name: path.Base(fp), + Size: size, + Modified: ti, + Ctime: time.Now(), + } + stream := &stream.FileStream{ + Obj: &obj, + Reader: input, + Mimetype: meta["Content-Type"], + } + + err = fs.PutDirectly(ctx, path.Dir(reqPath), stream) + if err != nil { + return result, err + } + + if err := stream.Close(); err != nil { + // remove file when close error occurred (FsPutErr) + _ = fs.Remove(ctx, fp) + return result, err + } + + b.meta.Store(fp, meta) + + return result, nil +} + +// DeleteMulti deletes multiple objects in a single request. +func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) { + for _, object := range objects { + if err := b.deleteObject(bucketName, object); err != nil { + utils.Log.Errorf("serve s3", "delete object failed: %v", err) + result.Error = append(result.Error, gofakes3.ErrorResult{ + Code: gofakes3.ErrInternal, + Message: gofakes3.ErrInternal.Message(), + Key: object, + }) + } else { + result.Deleted = append(result.Deleted, gofakes3.ObjectID{ + Key: object, + }) + } + } + + return result, nil +} + +// DeleteObject deletes the object with the given name. +func (b *s3Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { + return result, b.deleteObject(bucketName, objectName) +} + +// deleteObject deletes the object from the filesystem. +func (b *s3Backend) deleteObject(bucketName, objectName string) error { + ctx := context.Background() + bucket, err := getBucketByName(bucketName) + if err != nil { + return err + } + bucketPath := bucket.Path + + fp := path.Join(bucketPath, objectName) + fmeta, _ := op.GetNearestMeta(fp) + // S3 does not report an error when attemping to delete a key that does not exist, so + // we need to skip IsNotExist errors. + if _, err := fs.Get(context.WithValue(ctx, "meta", fmeta), fp, &fs.GetArgs{}); err != nil && !errs.IsObjectNotFound(err) { + return err + } + + fs.Remove(ctx, fp) + return nil +} + +// CreateBucket creates a new bucket. +func (b *s3Backend) CreateBucket(name string) error { + return gofakes3.ErrNotImplemented +} + +// DeleteBucket deletes the bucket with the given name. +func (b *s3Backend) DeleteBucket(name string) error { + return gofakes3.ErrNotImplemented +} + +// BucketExists checks if the bucket exists. +func (b *s3Backend) BucketExists(name string) (exists bool, err error) { + buckets, err := getAndParseBuckets() + if err != nil { + return false, err + } + for _, b := range buckets { + if b.Name == name { + return true, nil + } + } + return false, nil +} + +// CopyObject copy specified object from srcKey to dstKey. +func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { + if srcBucket == dstBucket && srcKey == dstKey { + //TODO: update meta + return result, nil + } + + ctx := context.Background() + srcB, err := getBucketByName(srcBucket) + if err != nil { + return result, err + } + srcBucketPath := srcB.Path + + srcFp := path.Join(srcBucketPath, srcKey) + fmeta, _ := op.GetNearestMeta(srcFp) + srcNode, err := fs.Get(context.WithValue(ctx, "meta", fmeta), srcFp, &fs.GetArgs{}) + + c, err := b.GetObject(srcBucket, srcKey, nil) + if err != nil { + return + } + defer func() { + _ = c.Contents.Close() + }() + + for k, v := range c.Metadata { + if _, found := meta[k]; !found && k != "X-Amz-Acl" { + meta[k] = v + } + } + if _, ok := meta["mtime"]; !ok { + meta["mtime"] = swift.TimeToFloatString(srcNode.ModTime()) + } + + _, err = b.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) + if err != nil { + return + } + + return gofakes3.CopyObjectResult{ + ETag: `"` + hex.EncodeToString(c.Hash) + `"`, + LastModified: gofakes3.NewContentTime(srcNode.ModTime()), + }, nil +} diff --git a/server/s3/ioutils.go b/server/s3/ioutils.go new file mode 100644 index 00000000000..6b49cacc708 --- /dev/null +++ b/server/s3/ioutils.go @@ -0,0 +1,36 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import "io" + +type noOpReadCloser struct{} + +type readerWithCloser struct { + io.Reader + closer func() error +} + +var _ io.ReadCloser = &readerWithCloser{} + +func (d noOpReadCloser) Read(b []byte) (n int, err error) { + return 0, io.EOF +} + +func (d noOpReadCloser) Close() error { + return nil +} + +func limitReadCloser(rdr io.Reader, closer func() error, sz int64) io.ReadCloser { + return &readerWithCloser{ + Reader: io.LimitReader(rdr, sz), + closer: closer, + } +} + +func (rwc *readerWithCloser) Close() error { + if rwc.closer != nil { + return rwc.closer() + } + return nil +} diff --git a/server/s3/list.go b/server/s3/list.go new file mode 100644 index 00000000000..bce870ca9ed --- /dev/null +++ b/server/s3/list.go @@ -0,0 +1,53 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "path" + "strings" + + "github.com/Mikubill/gofakes3" +) + +func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error { + fp := path.Join(bucket, fdPath) + + dirEntries, err := getDirEntries(fp) + if err != nil { + return err + } + + for _, entry := range dirEntries { + object := entry.GetName() + + // workround for control-chars detect + objectPath := path.Join(fdPath, object) + + if !strings.HasPrefix(object, name) { + continue + } + + if entry.IsDir() { + if addPrefix { + // response.AddPrefix(gofakes3.URLEncode(objectPath)) + response.AddPrefix(objectPath) + continue + } + err := b.entryListR(bucket, path.Join(fdPath, object), "", false, response) + if err != nil { + return err + } + } else { + item := &gofakes3.Content{ + // Key: gofakes3.URLEncode(objectPath), + Key: objectPath, + LastModified: gofakes3.NewContentTime(entry.ModTime()), + ETag: getFileHash(entry), + Size: entry.GetSize(), + StorageClass: gofakes3.StorageStandard, + } + response.Add(item) + } + } + return nil +} diff --git a/server/s3/logger.go b/server/s3/logger.go new file mode 100644 index 00000000000..7566fa8a116 --- /dev/null +++ b/server/s3/logger.go @@ -0,0 +1,27 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "fmt" + + "github.com/Mikubill/gofakes3" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// logger output formatted message +type logger struct{} + +// print log message +func (l logger) Print(level gofakes3.LogLevel, v ...interface{}) { + switch level { + default: + fallthrough + case gofakes3.LogErr: + utils.Log.Errorf("serve s3: %s", fmt.Sprintln(v...)) + case gofakes3.LogWarn: + utils.Log.Infof("serve s3: %s", fmt.Sprintln(v...)) + case gofakes3.LogInfo: + utils.Log.Debugf("serve s3: %s", fmt.Sprintln(v...)) + } +} diff --git a/server/s3/pager.go b/server/s3/pager.go new file mode 100644 index 00000000000..3268b0ca234 --- /dev/null +++ b/server/s3/pager.go @@ -0,0 +1,67 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "sort" + + "github.com/Mikubill/gofakes3" +) + +// pager splits the object list into smulitply pages. +func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { + // sort by alphabet + sort.Slice(list.CommonPrefixes, func(i, j int) bool { + return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix + }) + // sort by modtime + sort.Slice(list.Contents, func(i, j int) bool { + return list.Contents[i].LastModified.Before(list.Contents[j].LastModified.Time) + }) + tokens := page.MaxKeys + if tokens == 0 { + tokens = 1000 + } + if page.HasMarker { + for i, obj := range list.Contents { + if obj.Key == page.Marker { + list.Contents = list.Contents[i+1:] + break + } + } + for i, obj := range list.CommonPrefixes { + if obj.Prefix == page.Marker { + list.CommonPrefixes = list.CommonPrefixes[i+1:] + break + } + } + } + + response := gofakes3.NewObjectList() + for _, obj := range list.CommonPrefixes { + if tokens <= 0 { + break + } + response.AddPrefix(obj.Prefix) + tokens-- + } + + for _, obj := range list.Contents { + if tokens <= 0 { + break + } + response.Add(obj) + tokens-- + } + + if len(list.CommonPrefixes)+len(list.Contents) > int(page.MaxKeys) { + response.IsTruncated = true + if len(response.Contents) > 0 { + response.NextMarker = response.Contents[len(response.Contents)-1].Key + } else { + response.NextMarker = response.CommonPrefixes[len(response.CommonPrefixes)-1].Prefix + } + } + + return response, nil +} diff --git a/server/s3/server.go b/server/s3/server.go new file mode 100644 index 00000000000..2cb1f36d5ca --- /dev/null +++ b/server/s3/server.go @@ -0,0 +1,27 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "context" + "math/rand" + "net/http" + + "github.com/Mikubill/gofakes3" +) + +// Make a new S3 Server to serve the remote +func NewServer(ctx context.Context, authpair []string) (h http.Handler, err error) { + var newLogger logger + faker := gofakes3.New( + newBackend(), + // gofakes3.WithHostBucket(!opt.pathBucketMode), + gofakes3.WithLogger(newLogger), + gofakes3.WithRequestID(rand.Uint64()), + gofakes3.WithoutVersioning(), + gofakes3.WithV4Auth(authlistResolver(authpair)), + gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied + ) + + return faker.Server(), nil +} diff --git a/server/s3/utils.go b/server/s3/utils.go new file mode 100644 index 00000000000..88fab1ad9ca --- /dev/null +++ b/server/s3/utils.go @@ -0,0 +1,164 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/Mikubill/gofakes3" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Bucket struct { + Name string `json:"name"` + Path string `json:"path"` +} + +func getAndParseBuckets() ([]Bucket, error) { + var res []Bucket + err := json.Unmarshal([]byte(setting.GetStr(conf.S3Buckets)), &res) + return res, err +} + +func getBucketByName(name string) (Bucket, error) { + buckets, err := getAndParseBuckets() + if err != nil { + return Bucket{}, err + } + for _, b := range buckets { + if b.Name == name { + return b, nil + } + } + return Bucket{}, gofakes3.BucketNotFound(name) +} + +func getDirEntries(path string) ([]model.Obj, error) { + ctx := context.Background() + meta, _ := op.GetNearestMeta(path) + fi, err := fs.Get(context.WithValue(ctx, "meta", meta), path, &fs.GetArgs{}) + if errs.IsNotFoundError(err) { + return nil, gofakes3.ErrNoSuchKey + } else if err != nil { + return nil, gofakes3.ErrNoSuchKey + } + + if !fi.IsDir() { + return nil, gofakes3.ErrNoSuchKey + } + + dirEntries, err := fs.List(context.WithValue(ctx, "meta", meta), path, &fs.ListArgs{}) + if err != nil { + return nil, err + } + + return dirEntries, nil +} + +// func getFileHashByte(node interface{}) []byte { +// b, err := hex.DecodeString(getFileHash(node)) +// if err != nil { +// return nil +// } +// return b +// } + +func getFileHash(node interface{}) string { + // var o fs.Object + + // switch b := node.(type) { + // case vfs.Node: + // fsObj, ok := b.DirEntry().(fs.Object) + // if !ok { + // fs.Debugf("serve s3", "File uploading - reading hash from VFS cache") + // in, err := b.Open(os.O_RDONLY) + // if err != nil { + // return "" + // } + // defer func() { + // _ = in.Close() + // }() + // h, err := hash.NewMultiHasherTypes(hash.NewHashSet(hash.MD5)) + // if err != nil { + // return "" + // } + // _, err = io.Copy(h, in) + // if err != nil { + // return "" + // } + // return h.Sums()[hash.MD5] + // } + // o = fsObj + // case fs.Object: + // o = b + // } + + // hash, err := o.Hash(context.Background(), hash.MD5) + // if err != nil { + // return "" + // } + // return hash + return "" +} + +func prefixParser(p *gofakes3.Prefix) (path, remaining string) { + idx := strings.LastIndexByte(p.Prefix, '/') + if idx < 0 { + return "", p.Prefix + } + return p.Prefix[:idx], p.Prefix[idx+1:] +} + +// // FIXME this could be implemented by VFS.MkdirAll() +// func mkdirRecursive(path string, VFS *vfs.VFS) error { +// path = strings.Trim(path, "/") +// dirs := strings.Split(path, "/") +// dir := "" +// for _, d := range dirs { +// dir += "/" + d +// if _, err := VFS.Stat(dir); err != nil { +// err := VFS.Mkdir(dir, 0777) +// if err != nil { +// return err +// } +// } +// } +// return nil +// } + +// func rmdirRecursive(p string, VFS *vfs.VFS) { +// dir := path.Dir(p) +// if !strings.ContainsAny(dir, "/\\") { +// // might be bucket(root) +// return +// } +// if _, err := VFS.Stat(dir); err == nil { +// err := VFS.Remove(dir) +// if err != nil { +// return +// } +// rmdirRecursive(dir, VFS) +// } +// } + +func authlistResolver(list []string) map[string]string { + authList := make(map[string]string) + for _, v := range list { + parts := strings.Split(v, ",") + if len(parts) != 2 { + utils.Log.Infof(fmt.Sprintf("Ignored: invalid auth pair %s", v)) + continue + } + authList[parts[0]] = parts[1] + } + return authList +} From ae6984714d984e4c5e8f9c2f9bedee580960bfd9 Mon Sep 17 00:00:00 2001 From: Sukka Date: Sat, 2 Mar 2024 15:36:28 +0800 Subject: [PATCH 142/659] fix: remove default polyfill (#6130 close #6100) * refactor(setting): replace `polyfill.io`` * fix: remove default polyfill --------- Co-authored-by: Andy Hsu --- internal/bootstrap/data/setting.go | 5 ++--- internal/model/setting.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 8adc713d6d6..c6b771b45f3 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -21,7 +21,6 @@ func initSettings() { if err != nil { utils.Log.Fatalf("failed get settings: %+v", err) } - for i := range settings { if !isActive(settings[i].Key) && settings[i].Flag != model.DEPRECATED { settings[i].Flag = model.DEPRECATED @@ -42,7 +41,7 @@ func initSettings() { continue } // save - if stored != nil && item.Key != conf.VERSION { + if stored != nil && item.Key != conf.VERSION && stored.Value != item.DeprecatedValue { item.Value = stored.Value } if stored == nil || *item != *stored { @@ -129,7 +128,7 @@ func InitialSettings() []model.SettingItem { // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, - {Key: conf.CustomizeHead, Value: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.CustomizeHead, DeprecatedValue: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.CustomizeBody, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.LinkExpiration, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.SignAll, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, diff --git a/internal/model/setting.go b/internal/model/setting.go index 1a47cf5c062..9ef6fdf7d78 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -21,13 +21,14 @@ const ( ) type SettingItem struct { - Key string `json:"key" gorm:"primaryKey" binding:"required"` // unique key - Value string `json:"value"` // value - Help string `json:"help"` // help message - Type string `json:"type"` // string, number, bool, select - Options string `json:"options"` // values for select - Group int `json:"group"` // use to group setting in frontend - Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. + Key string `json:"key" gorm:"primaryKey" binding:"required"` // unique key + Value string `json:"value"` // value + DeprecatedValue string `json:"deprecated_value" gorm:"-:all"` // deprecated value + Help string `json:"help"` // help message + Type string `json:"type"` // string, number, bool, select + Options string `json:"options"` // values for select + Group int `json:"group"` // use to group setting in frontend + Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. } func (s SettingItem) IsDeprecated() bool { From 7d9ecba99c5c156967bacb3769cf026cdffea861 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 4 Mar 2024 14:23:44 +0800 Subject: [PATCH 143/659] fix: add `m3u8` to default video types (close #6142) --- internal/bootstrap/data/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index c6b771b45f3..b12973d7a2f 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -97,7 +97,7 @@ func InitialSettings() []model.SettingItem { // preview settings {Key: conf.TextTypes, Value: "txt,htm,html,xml,java,properties,sql,js,md,json,conf,ini,vue,php,py,bat,gitignore,yml,go,sh,c,cpp,h,hpp,tsx,vtt,srt,ass,rs,lrc", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.AudioTypes, Value: "mp3,flac,ogg,m4a,wav,opus,wma", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, - {Key: conf.VideoTypes, Value: "mp4,mkv,avi,mov,rmvb,webm,flv", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, + {Key: conf.VideoTypes, Value: "mp4,mkv,avi,mov,rmvb,webm,flv,m3u8", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ImageTypes, Value: "jpg,tiff,jpeg,png,gif,bmp,svg,ico,swf,webp", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, //{Key: conf.OfficeTypes, Value: "doc,docx,xls,xlsx,ppt,pptx", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ProxyTypes, Value: "m3u8", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, From 6f6a8e6dfc031838e5d7a57bdf6b010483a8c284 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:12:22 +0800 Subject: [PATCH 144/659] fix(deps): update github.com/t3rm1n4l/go-mega digest to d494b6a (#6081) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7fc142d44b4..dd3e1d4205c 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 github.com/stretchr/testify v1.8.4 - github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca + github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 diff --git a/go.sum b/go.sum index 008dfa92708..014d3f21997 100644 --- a/go.sum +++ b/go.sum @@ -449,6 +449,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4UvMT7OmKhH9sOIvS8iXkxfPdnn9wQA= github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= +github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= +github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= From 2a17d0c2cd09a05c5066e7a7676acddaf8625c0a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 5 Mar 2024 16:29:26 +0800 Subject: [PATCH 145/659] fix: settings reset to default after restart if set to empty (close #6143) --- internal/bootstrap/data/setting.go | 7 +++++-- internal/model/setting.go | 16 ++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index b12973d7a2f..f85903c09dd 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -34,6 +34,9 @@ func initSettings() { // create or save setting for i := range initialSettingItems { item := &initialSettingItems[i] + if item.PreDefault == "" { + item.PreDefault = item.Value + } // err stored, err := op.GetSettingItemByKey(item.Key) if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { @@ -41,7 +44,7 @@ func initSettings() { continue } // save - if stored != nil && item.Key != conf.VERSION && stored.Value != item.DeprecatedValue { + if stored != nil && item.Key != conf.VERSION && stored.Value != item.PreDefault { item.Value = stored.Value } if stored == nil || *item != *stored { @@ -128,7 +131,7 @@ func InitialSettings() []model.SettingItem { // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, - {Key: conf.CustomizeHead, DeprecatedValue: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.CustomizeHead, PreDefault: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.CustomizeBody, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.LinkExpiration, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.SignAll, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, diff --git a/internal/model/setting.go b/internal/model/setting.go index 9ef6fdf7d78..9893124890e 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -21,14 +21,14 @@ const ( ) type SettingItem struct { - Key string `json:"key" gorm:"primaryKey" binding:"required"` // unique key - Value string `json:"value"` // value - DeprecatedValue string `json:"deprecated_value" gorm:"-:all"` // deprecated value - Help string `json:"help"` // help message - Type string `json:"type"` // string, number, bool, select - Options string `json:"options"` // values for select - Group int `json:"group"` // use to group setting in frontend - Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. + Key string `json:"key" gorm:"primaryKey" binding:"required"` // unique key + Value string `json:"value"` // value + PreDefault string `json:"-" gorm:"-:all"` // deprecated value + Help string `json:"help"` // help message + Type string `json:"type"` // string, number, bool, select + Options string `json:"options"` // values for select + Group int `json:"group"` // use to group setting in frontend + Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. } func (s SettingItem) IsDeprecated() bool { From ac68079a768962f4265599c8451541f0053d6549 Mon Sep 17 00:00:00 2001 From: mlkt <365690226@qq.com> Date: Fri, 8 Mar 2024 15:33:42 +0800 Subject: [PATCH 146/659] feat(seafile): improve features, support access to encrypted library, etc (#6160) --- drivers/seafile/driver.go | 110 +++++++++++++++++++++++++++++-------- drivers/seafile/meta.go | 3 +- drivers/seafile/types.go | 34 +++++++++++- drivers/seafile/util.go | 112 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 234 insertions(+), 25 deletions(-) diff --git a/drivers/seafile/driver.go b/drivers/seafile/driver.go index 49cf3386058..6d1f16dad3b 100644 --- a/drivers/seafile/driver.go +++ b/drivers/seafile/driver.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "net/http" - "path/filepath" "strings" "time" @@ -19,6 +18,7 @@ type Seafile struct { Addition authorization string + libraryMap map[string]*LibraryInfo } func (d *Seafile) Config() driver.Config { @@ -31,6 +31,8 @@ func (d *Seafile) GetAddition() driver.Additional { func (d *Seafile) Init(ctx context.Context) error { d.Address = strings.TrimSuffix(d.Address, "/") + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + d.libraryMap = make(map[string]*LibraryInfo) return d.getToken() } @@ -38,10 +40,37 @@ func (d *Seafile) Drop(ctx context.Context) error { return nil } -func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { +func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) (result []model.Obj, err error) { path := dir.GetPath() + if path == d.RootFolderPath { + libraries, err := d.listLibraries() + if err != nil { + return nil, err + } + if path == "/" && d.RepoId == "" { + return utils.SliceConvert(libraries, func(f LibraryItemResp) (model.Obj, error) { + return &model.Object{ + Name: f.Name, + Modified: time.Unix(f.Modified, 0), + Size: f.Size, + IsFolder: true, + }, nil + }) + } + } + var repo *LibraryInfo + repo, path, err = d.getRepoAndPath(path) + if err != nil { + return nil, err + } + if repo.Encrypted { + err = d.decryptLibrary(repo) + if err != nil { + return nil, err + } + } var resp []RepoDirItemResp - _, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", d.Addition.RepoId), func(req *resty.Request) { + _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", repo.Id), func(req *resty.Request) { req.SetResult(&resp).SetQueryParams(map[string]string{ "p": path, }) @@ -63,9 +92,13 @@ func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) } func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(file.GetPath()) + if err != nil { + return nil, err + } + res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": file.GetPath(), + "p": path, "reuse": "1", }) }) @@ -78,9 +111,14 @@ func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(parentDir.GetPath()) + if err != nil { + return err + } + path, _ = utils.JoinBasePath(path, dirName) + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": filepath.Join(parentDir.GetPath(), dirName), + "p": path, }).SetFormData(map[string]string{ "operation": "mkdir", }) @@ -89,22 +127,34 @@ func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri } func (d *Seafile) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(srcObj.GetPath()) + if err != nil { + return err + } + dstRepo, dstPath, err := d.getRepoAndPath(dstDir.GetPath()) + if err != nil { + return err + } + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": srcObj.GetPath(), + "p": path, }).SetFormData(map[string]string{ "operation": "move", - "dst_repo": d.Addition.RepoId, - "dst_dir": dstDir.GetPath(), + "dst_repo": dstRepo.Id, + "dst_dir": dstPath, }) }, true) return err } func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(srcObj.GetPath()) + if err != nil { + return err + } + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": srcObj.GetPath(), + "p": path, }).SetFormData(map[string]string{ "operation": "rename", "newname": newName, @@ -114,31 +164,47 @@ func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *Seafile) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(srcObj.GetPath()) + if err != nil { + return err + } + dstRepo, dstPath, err := d.getRepoAndPath(dstDir.GetPath()) + if err != nil { + return err + } + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": srcObj.GetPath(), + "p": path, }).SetFormData(map[string]string{ "operation": "copy", - "dst_repo": d.Addition.RepoId, - "dst_dir": dstDir.GetPath(), + "dst_repo": dstRepo.Id, + "dst_dir": dstPath, }) }) return err } func (d *Seafile) Remove(ctx context.Context, obj model.Obj) error { - _, err := d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(obj.GetPath()) + if err != nil { + return err + } + _, err = d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": obj.GetPath(), + "p": path, }) }) return err } func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(dstDir.GetPath()) + if err != nil { + return err + } + res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": dstDir.GetPath(), + "p": path, }) }) if err != nil { @@ -150,7 +216,7 @@ func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt _, err = d.request(http.MethodPost, u, func(req *resty.Request) { req.SetFileReader("file", stream.GetName(), stream). SetFormData(map[string]string{ - "parent_dir": dstDir.GetPath(), + "parent_dir": path, "replace": "1", }) }) diff --git a/drivers/seafile/meta.go b/drivers/seafile/meta.go index e333fd905bb..9d84aee182f 100644 --- a/drivers/seafile/meta.go +++ b/drivers/seafile/meta.go @@ -11,7 +11,8 @@ type Addition struct { Address string `json:"address" required:"true"` UserName string `json:"username" required:"true"` Password string `json:"password" required:"true"` - RepoId string `json:"repoId" required:"true"` + RepoId string `json:"repoId" required:"false"` + RepoPwd string `json:"repoPwd" required:"false"` } var config = driver.Config{ diff --git a/drivers/seafile/types.go b/drivers/seafile/types.go index 5c5b528d31f..47cb322df4a 100644 --- a/drivers/seafile/types.go +++ b/drivers/seafile/types.go @@ -1,14 +1,44 @@ package seafile +import "time" + type AuthTokenResp struct { Token string `json:"token"` } -type RepoDirItemResp struct { +type RepoItemResp struct { Id string `json:"id"` - Type string `json:"type"` // dir, file + Type string `json:"type"` // repo, dir, file Name string `json:"name"` Size int64 `json:"size"` Modified int64 `json:"mtime"` Permission string `json:"permission"` } + +type LibraryItemResp struct { + RepoItemResp + OwnerContactEmail string `json:"owner_contact_email"` + OwnerName string `json:"owner_name"` + Owner string `json:"owner"` + ModifierEmail string `json:"modifier_email"` + ModifierContactEmail string `json:"modifier_contact_email"` + ModifierName string `json:"modifier_name"` + Virtual bool `json:"virtual"` + MtimeRelative string `json:"mtime_relative"` + Encrypted bool `json:"encrypted"` + Version int `json:"version"` + HeadCommitId string `json:"head_commit_id"` + Root string `json:"root"` + Salt string `json:"salt"` + SizeFormatted string `json:"size_formatted"` +} + +type RepoDirItemResp struct { + RepoItemResp +} + +type LibraryInfo struct { + LibraryItemResp + decryptedTime time.Time + decryptedSuccess bool +} \ No newline at end of file diff --git a/drivers/seafile/util.go b/drivers/seafile/util.go index 23255545c3a..681953103c3 100644 --- a/drivers/seafile/util.go +++ b/drivers/seafile/util.go @@ -1,8 +1,13 @@ package seafile import ( + "errors" "fmt" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/pkg/utils" + "net/http" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/go-resty/resty/v2" @@ -60,3 +65,110 @@ func (d *Seafile) request(method string, pathname string, callback base.ReqCallb } return res.Body(), nil } + +func (d *Seafile) getRepoAndPath(fullPath string) (repo *LibraryInfo, path string, err error) { + libraryMap := d.libraryMap + repoId := d.Addition.RepoId + if repoId != "" { + if len(repoId) == 36 /* uuid */ { + for _, library := range libraryMap { + if library.Id == repoId { + return library, fullPath, nil + } + } + } + } else { + var repoName string + str := fullPath[1:] + pos := strings.IndexRune(str, '/') + if pos == -1 { + repoName = str + } else { + repoName = str[:pos] + } + path = utils.FixAndCleanPath(fullPath[1+len(repoName):]) + if library, ok := libraryMap[repoName]; ok { + return library, path, nil + } + } + return nil, "", errs.ObjectNotFound +} + +func (d *Seafile) listLibraries() (resp []LibraryItemResp, err error) { + repoId := d.Addition.RepoId + if repoId == "" { + _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { + req.SetResult(&resp) + }) + } else { + var oneResp LibraryItemResp + _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/", repoId), func(req *resty.Request) { + req.SetResult(&oneResp) + }) + if err == nil { + resp = append(resp, oneResp) + } + } + if err != nil { + return nil, err + } + libraryMap := make(map[string]*LibraryInfo) + var putLibraryMap func(library LibraryItemResp, index int) + putLibraryMap = func(library LibraryItemResp, index int) { + name := library.Name + if index > 0 { + name = fmt.Sprintf("%s (%d)", name, index) + } + if _, exist := libraryMap[name]; exist { + putLibraryMap(library, index+1) + } else { + libraryInfo := LibraryInfo{} + data, _ := utils.Json.Marshal(library) + _ = utils.Json.Unmarshal(data, &libraryInfo) + libraryMap[name] = &libraryInfo + } + } + for _, library := range resp { + putLibraryMap(library, 0) + } + d.libraryMap = libraryMap + return resp, nil +} + +var repoPwdNotConfigured = errors.New("library password not configured") +var repoPwdIncorrect = errors.New("library password is incorrect") + +func (d *Seafile) decryptLibrary(repo *LibraryInfo) (err error) { + if !repo.Encrypted { + return nil + } + if d.RepoPwd == "" { + return repoPwdNotConfigured + } + now := time.Now() + decryptedTime := repo.decryptedTime + if repo.decryptedSuccess { + if now.Sub(decryptedTime).Minutes() <= 30 { + return nil + } + } else { + if now.Sub(decryptedTime).Seconds() <= 10 { + return repoPwdIncorrect + } + } + var resp string + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/", repo.Id), func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "password": d.RepoPwd, + }) + }) + repo.decryptedTime = time.Now() + if err != nil || !strings.Contains(resp, "success") { + repo.decryptedSuccess = false + return err + } + repo.decryptedSuccess = true + return nil +} + + From 45e009a22c57116d2438f9c336170257d2a068a7 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sat, 9 Mar 2024 14:54:49 +0800 Subject: [PATCH 147/659] fix(mopan): upload error (close #6158 in #6166) --- drivers/mopan/driver.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index b5441bff2ea..369ec83b64d 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -295,7 +295,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre } if !initUpdload.FileDataExists { - utils.Log.Error(d.client.CloudDiskStartBusiness()) + // utils.Log.Error(d.client.CloudDiskStartBusiness()) threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, retry.Attempts(3), @@ -323,6 +323,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if err != nil { return err } + req.ContentLength = byteSize resp, err := base.HttpClient.Do(req) if err != nil { return err From 82222840fe6188dff62431bca8c6d9103a225d91 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 14:58:36 +0800 Subject: [PATCH 148/659] fix(deps): update golang.org/x/exp digest to 814bf88 (#6144) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 14 +++++++------- go.sum | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index dd3e1d4205c..85aa4ec3406 100644 --- a/go.mod +++ b/go.mod @@ -54,10 +54,10 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 - golang.org/x/crypto v0.18.0 - golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e + golang.org/x/crypto v0.19.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/image v0.15.0 - golang.org/x/net v0.20.0 + golang.org/x/net v0.21.0 golang.org/x/oauth2 v0.16.0 golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 @@ -204,11 +204,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.5.0 // indirect - golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.16.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.17.0 // indirect + golang.org/x/term v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.16.0 // indirect + golang.org/x/tools v0.18.0 // indirect google.golang.org/api v0.134.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect google.golang.org/grpc v1.57.0 // indirect diff --git a/go.sum b/go.sum index 014d3f21997..a46f0d55cce 100644 --- a/go.sum +++ b/go.sum @@ -509,8 +509,12 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= @@ -530,6 +534,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -538,6 +544,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -567,6 +575,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -575,6 +585,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -600,6 +612,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= +golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= From bdfc1591bdc5c3414e8e79518243bb0fb87b55a1 Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Sun, 10 Mar 2024 16:48:25 +0800 Subject: [PATCH 149/659] fix: webauthn logspam (#6181) --- internal/bootstrap/data/user.go | 19 +++++++++++++++++++ server/handles/user.go | 1 + 2 files changed, 20 insertions(+) diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 451c60a327f..3b71e498206 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -31,6 +31,7 @@ func initUser() { PwdHash: model.TwoHashPwd(adminPassword, salt), Role: model.ADMIN, BasePath: "/", + Authn: "[]", } if err := op.CreateUser(admin); err != nil { panic(err) @@ -53,6 +54,7 @@ func initUser() { BasePath: "/", Permission: 0, Disabled: true, + Authn: "[]", } if err := db.CreateUser(guest); err != nil { utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) @@ -62,6 +64,7 @@ func initUser() { } } hashPwdForOldVersion() + updateAuthnForOldVersion() } func hashPwdForOldVersion() { @@ -80,3 +83,19 @@ func hashPwdForOldVersion() { } } } + +func updateAuthnForOldVersion() { + users, _, err := op.GetUsers(1, -1) + if err != nil { + utils.Log.Fatalf("[update authn for old version] failed get users: %v", err) + } + for i := range users { + user := users[i] + if user.Authn == "" { + user.Authn = "[]" + if err := db.UpdateUser(&user); err != nil { + utils.Log.Fatalf("[update authn for old version] failed update user: %v", err) + } + } + } +} diff --git a/server/handles/user.go b/server/handles/user.go index 2220648f750..4d404a4c652 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -41,6 +41,7 @@ func CreateUser(c *gin.Context) { } req.SetPassword(req.Password) req.Password = "" + req.Authn = "[]" if err := op.CreateUser(&req); err != nil { common.ErrorResp(c, err, 500, true) } else { From 195c869272479df859756441bac6fe573f4c37a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=81=A5=E5=BF=98=E7=97=87?= <50003754+zzc10086@users.noreply.github.com> Date: Mon, 11 Mar 2024 20:10:26 +0800 Subject: [PATCH 150/659] feat(139): refresh token periodically (#6146) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 139定时刷新token * fix build fail --- drivers/139/driver.go | 13 +++++++++++++ drivers/139/types.go | 13 +++++++++++++ drivers/139/util.go | 27 +++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 9d8cbd523e6..d33c3d77ebf 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -8,18 +8,21 @@ import ( "net/http" "strconv" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/cron" log "github.com/sirupsen/logrus" ) type Yun139 struct { model.Storage Addition + cron *cron.Cron Account string } @@ -35,6 +38,13 @@ func (d *Yun139) Init(ctx context.Context) error { if d.Authorization == "" { return fmt.Errorf("authorization is empty") } + d.cron = cron.NewCron(time.Hour * 24 * 7) + d.cron.Do(func() { + err := d.refreshToken() + if err != nil { + log.Errorf("%+v", err) + } + }) switch d.Addition.Type { case MetaPersonalNew: if len(d.Addition.RootFolderID) == 0 { @@ -72,6 +82,9 @@ func (d *Yun139) Init(ctx context.Context) error { } func (d *Yun139) Drop(ctx context.Context) error { + if d.cron != nil { + d.cron.Stop() + } return nil } diff --git a/drivers/139/types.go b/drivers/139/types.go index 841aa9d3871..f797196624b 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -1,5 +1,9 @@ package _139 +import ( + "encoding/xml" +) + const ( MetaPersonal string = "personal" MetaFamily string = "family" @@ -230,3 +234,12 @@ type PersonalUploadResp struct { UploadId string `json:"uploadId"` } } + +type RefreshTokenResp struct { + XMLName xml.Name `xml:"root"` + Return string `xml:"return"` + Token string `xml:"token"` + Expiretime int32 `xml:"expiretime"` + AccessToken string `xml:"accessToken"` + Desc string `xml:"desc"` +} diff --git a/drivers/139/util.go b/drivers/139/util.go index a3627b6c8ae..3cd2966e06b 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -15,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/internal/op" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" @@ -52,6 +53,32 @@ func getTime(t string) time.Time { return stamp } +func (d *Yun139) refreshToken() error { + url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do" + var resp RefreshTokenResp + decode, err := base64.StdEncoding.DecodeString(d.Authorization) + if err != nil { + return err + } + decodeStr := string(decode) + splits := strings.Split(decodeStr, ":") + reqBody := "" + splits[2] + "" + splits[1] + "656" + _, err = base.RestyClient.R(). + //ForceContentType("application/json"). + SetBody(reqBody). + SetResult(&resp). + Post(url) + if err != nil { + return err + } + if resp.Return != "0" { + return fmt.Errorf("failed to refresh token: %s", resp.Desc) + } + d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + splits[1] + ":" + resp.Token)) + op.MustSaveDriverStorage(d) + return nil +} + func (d *Yun139) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { url := "https://yun.139.com" + pathname req := base.RestyClient.R() From 9a0a63d34cca4dbcc58fe5f6ad1fade346559e9b Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 11 Mar 2024 20:30:22 +0800 Subject: [PATCH 151/659] fix(ilanzou): add referer to request header (close #6171) --- drivers/ilanzou/driver.go | 5 ++++- drivers/ilanzou/meta.go | 5 ++++- drivers/ilanzou/util.go | 6 +++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 875300e1820..341136da1cd 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -145,7 +145,10 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) u.RawQuery = query.Encode() realURL := u.String() // get the url after redirect - res, err := base.NoRedirectClient.R().Get(realURL) + res, err := base.NoRedirectClient.R().SetHeaders(map[string]string{ + //"Origin": d.conf.site, + "Referer": d.conf.site + "/", + }).Get(realURL) if err != nil { return nil, err } diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go index ca813c5efad..ed5b2edb52e 100644 --- a/drivers/ilanzou/meta.go +++ b/drivers/ilanzou/meta.go @@ -21,6 +21,7 @@ type Conf struct { unproved string proved string devVersion string + site string } func init() { @@ -45,7 +46,8 @@ func init() { bucket: "wpanstore-lanzou", unproved: "unproved", proved: "proved", - devVersion: "120", + devVersion: "122", + site: "https://www.ilanzou.com", }, } }) @@ -71,6 +73,7 @@ func init() { unproved: "ws", proved: "app", devVersion: "121", + site: "https://www.feijipan.com", }, } }) diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index d8995523ea0..37e111ad53e 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -52,12 +52,16 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr "devType": "6", "devCode": d.UUID, "devModel": "chrome", - "devVersion": "120", + "devVersion": d.conf.devVersion, "appVersion": "", "timestamp": ts, //"appToken": d.Token, "extra": "2", }) + req.SetHeaders(map[string]string{ + "Origin": d.conf.site, + "Referer": d.conf.site + "/", + }) if proved { req.SetQueryParam("appToken", d.Token) } From b07ddfbc130829b7b0b8e75837aae780c9863444 Mon Sep 17 00:00:00 2001 From: Mmx Date: Wed, 13 Mar 2024 15:11:21 +0800 Subject: [PATCH 152/659] fix(ci): replace dockerfile tag step may have no effect (#6206) --- .github/workflows/build_docker.yml | 1 + .github/workflows/release_docker.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index baedcaab693..9c740ba1fa4 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -71,6 +71,7 @@ jobs: id: docker_build_ffmpeg uses: docker/build-push-action@v5 with: + context: . file: Dockerfile.ffmpeg push: ${{ github.event_name == 'push' }} tags: ${{ steps.meta-ffmpeg.outputs.tags }} diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 10f8554220b..d172c27f625 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -62,6 +62,7 @@ jobs: id: docker_build_ffmpeg uses: docker/build-push-action@v5 with: + context: . file: Dockerfile.ffmpeg push: true tags: ${{ steps.meta-ffmpeg.outputs.tags }} From 88947f6676cdaa3dc9cea1fa4b3e2510addcd7de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E4=BD=B3?= Date: Sun, 24 Mar 2024 11:03:18 +0800 Subject: [PATCH 153/659] fix(ipfs): url escape filename (#6245 close #6027) This resolves #6027 --- drivers/ipfs_api/driver.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/ipfs_api/driver.go b/drivers/ipfs_api/driver.go index cf21e62da59..f6f81305e20 100644 --- a/drivers/ipfs_api/driver.go +++ b/drivers/ipfs_api/driver.go @@ -62,7 +62,7 @@ func (d *IPFS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] for _, file := range dirs { gateurl := *d.gateURL gateurl.Path = "ipfs/" + file.Hash - gateurl.RawQuery = "filename=" + file.Name + gateurl.RawQuery = "filename=" + url.PathEscape(file.Name) objlist = append(objlist, &model.ObjectURL{ Object: model.Object{ID: file.Hash, Name: file.Name, Size: int64(file.Size), IsFolder: file.Type == 1}, Url: model.Url{Url: gateurl.String()}, @@ -73,7 +73,7 @@ func (d *IPFS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] } func (d *IPFS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - link := d.Gateway + "/ipfs/" + file.GetID() + "/?filename=" + file.GetName() + link := d.Gateway + "/ipfs/" + file.GetID() + "/?filename=" + url.PathEscape(file.GetName()) return &model.Link{URL: link}, nil } From 022e0ca292715a6303045072e772fc6fbb34c6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=81=A5=E5=BF=98=E7=97=87?= <50003754+zzc10086@users.noreply.github.com> Date: Sun, 24 Mar 2024 11:04:55 +0800 Subject: [PATCH 154/659] fix(139): incorrect refreshTokenResp serialization (#6248) --- drivers/139/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/139/util.go b/drivers/139/util.go index 3cd2966e06b..5918e4c5305 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -64,7 +64,7 @@ func (d *Yun139) refreshToken() error { splits := strings.Split(decodeStr, ":") reqBody := "" + splits[2] + "" + splits[1] + "656" _, err = base.RestyClient.R(). - //ForceContentType("application/json"). + ForceContentType("application/xml"). SetBody(reqBody). SetResult(&resp). Post(url) @@ -74,7 +74,7 @@ func (d *Yun139) refreshToken() error { if resp.Return != "0" { return fmt.Errorf("failed to refresh token: %s", resp.Desc) } - d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + splits[1] + ":" + resp.Token)) + d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + ":" + splits[1] + ":" + resp.Token)) op.MustSaveDriverStorage(d) return nil } From 9c84b6596f4dd470497f61db1dd8e5955397a9a6 Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Sun, 24 Mar 2024 15:16:00 +0800 Subject: [PATCH 155/659] feat: stand-alone port s3 server (#6242) * feat: single port s3 server * fix: unable to PUT files if not in root dir --- cmd/server.go | 21 +++++++++++++++++++++ internal/bootstrap/data/setting.go | 1 - internal/conf/config.go | 15 ++++++++++++++- internal/conf/const.go | 3 +-- server/router.go | 5 +++++ server/s3.go | 22 +++++++++++++++++++--- server/s3/backend.go | 2 +- server/s3/server.go | 4 ++-- server/s3/utils.go | 18 +++++++----------- 9 files changed, 70 insertions(+), 21 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index d03a9d8099c..3c7137bcf6e 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -91,6 +91,27 @@ the address is defined in config file`, } }() } + s3r := gin.New() + s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) + server.InitS3(s3r) + if conf.Conf.S3.Port != -1 { + s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) + utils.Log.Infof("start S3 server @ %s", s3Base) + go func() { + var err error + if conf.Conf.S3.SSL { + httpsSrv = &http.Server{Addr: s3Base, Handler: s3r} + err = httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) + } + if !conf.Conf.S3.SSL { + httpSrv = &http.Server{Addr: s3Base, Handler: s3r} + err = httpSrv.ListenAndServe() + } + if err != nil && !errors.Is(err, http.ErrServerClosed) { + utils.Log.Fatalf("failed to start s3 server: %s", err.Error()) + } + }() + } // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index f85903c09dd..93a588e7622 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -180,7 +180,6 @@ func InitialSettings() []model.SettingItem { {Key: conf.LdapLoginTips, Value: "login with ldap", Type: conf.TypeString, Group: model.LDAP, Flag: model.PUBLIC}, //s3 settings - {Key: conf.S3Enabled, Value: "false", Type: conf.TypeBool, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3AccessKeyId, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3SecretAccessKey, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3Buckets, Value: "[]", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, diff --git a/internal/conf/config.go b/internal/conf/config.go index 0f1e0048c75..ef41192a778 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -1,9 +1,10 @@ package conf import ( + "path/filepath" + "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/pkg/utils/random" - "path/filepath" ) type Database struct { @@ -63,6 +64,12 @@ type Cors struct { AllowHeaders []string `json:"allow_headers" env:"ALLOW_HEADERS"` } +type S3 struct { + Enable bool `json:"enable" env:"ENABLE"` + Port int `json:"port" env:"PORT"` + SSL bool `json:"ssl" env:"SSL"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -81,6 +88,7 @@ type Config struct { TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"` Cors Cors `json:"cors" envPrefix:"CORS_"` + S3 S3 `json:"s3" envPrefix:"S3_"` } func DefaultConfig() *Config { @@ -142,5 +150,10 @@ func DefaultConfig() *Config { AllowMethods: []string{"*"}, AllowHeaders: []string{"*"}, }, + S3: S3{ + Enable: false, + Port: 5246, + SSL: false, + }, } } diff --git a/internal/conf/const.go b/internal/conf/const.go index a5d95e5d8a9..2d53702e91a 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -85,10 +85,9 @@ const ( LdapLoginTips = "ldap_login_tips" //s3 - S3Enabled = "s3_enabled" + S3Buckets = "s3_buckets" S3AccessKeyId = "s3_access_key_id" S3SecretAccessKey = "s3_secret_access_key" - S3Buckets = "s3_buckets" // qbittorrent QbittorrentUrl = "qbittorrent_url" diff --git a/server/router.go b/server/router.go index b0b66294272..5f784aa4b7d 100644 --- a/server/router.go +++ b/server/router.go @@ -171,3 +171,8 @@ func Cors(r *gin.Engine) { config.AllowMethods = conf.Conf.Cors.AllowMethods r.Use(cors.New(config)) } + +func InitS3(e *gin.Engine) { + Cors(e) + S3Server(e.Group("/")) +} diff --git a/server/s3.go b/server/s3.go index 5a70cf2aa76..1a9f2e038c9 100644 --- a/server/s3.go +++ b/server/s3.go @@ -6,20 +6,25 @@ import ( "strings" "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/s3" "github.com/gin-gonic/gin" ) func S3(g *gin.RouterGroup) { - if !setting.GetBool(conf.S3Enabled) { + if !conf.Conf.S3.Enable { g.Any("/*path", func(c *gin.Context) { common.ErrorStrResp(c, "S3 server is not enabled", 403) }) return } - h, _ := s3.NewServer(context.Background(), []string{setting.GetStr(conf.S3AccessKeyId) + "," + setting.GetStr(conf.S3SecretAccessKey)}) + if conf.Conf.S3.Port != -1 { + g.Any("/*path", func(c *gin.Context) { + common.ErrorStrResp(c, "S3 server bound to single port", 403) + }) + return + } + h, _ := s3.NewServer(context.Background()) g.Any("/*path", func(c *gin.Context) { adjustedPath := strings.TrimPrefix(c.Request.URL.Path, path.Join(conf.URL.Path, "/s3")) @@ -27,3 +32,14 @@ func S3(g *gin.RouterGroup) { gin.WrapH(h)(c) }) } + +func S3Server(g *gin.RouterGroup) { + if !conf.Conf.S3.Enable { + g.Any("/*path", func(c *gin.Context) { + common.ErrorStrResp(c, "S3 server is not enabled", 403) + }) + return + } + h, _ := s3.NewServer(context.Background()) + g.Any("/*path", gin.WrapH(h)) +} diff --git a/server/s3/backend.go b/server/s3/backend.go index c73405252c7..75c6b28b1ef 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -299,7 +299,7 @@ func (b *s3Backend) PutObject( Mimetype: meta["Content-Type"], } - err = fs.PutDirectly(ctx, path.Dir(reqPath), stream) + err = fs.PutDirectly(ctx, reqPath, stream) if err != nil { return result, err } diff --git a/server/s3/server.go b/server/s3/server.go index 2cb1f36d5ca..19df735fb5d 100644 --- a/server/s3/server.go +++ b/server/s3/server.go @@ -11,7 +11,7 @@ import ( ) // Make a new S3 Server to serve the remote -func NewServer(ctx context.Context, authpair []string) (h http.Handler, err error) { +func NewServer(ctx context.Context) (h http.Handler, err error) { var newLogger logger faker := gofakes3.New( newBackend(), @@ -19,7 +19,7 @@ func NewServer(ctx context.Context, authpair []string) (h http.Handler, err erro gofakes3.WithLogger(newLogger), gofakes3.WithRequestID(rand.Uint64()), gofakes3.WithoutVersioning(), - gofakes3.WithV4Auth(authlistResolver(authpair)), + gofakes3.WithV4Auth(authlistResolver()), gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied ) diff --git a/server/s3/utils.go b/server/s3/utils.go index 88fab1ad9ca..98c271f76a3 100644 --- a/server/s3/utils.go +++ b/server/s3/utils.go @@ -5,7 +5,6 @@ package s3 import ( "context" "encoding/json" - "fmt" "strings" "github.com/Mikubill/gofakes3" @@ -15,7 +14,6 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/pkg/utils" ) type Bucket struct { @@ -150,15 +148,13 @@ func prefixParser(p *gofakes3.Prefix) (path, remaining string) { // } // } -func authlistResolver(list []string) map[string]string { - authList := make(map[string]string) - for _, v := range list { - parts := strings.Split(v, ",") - if len(parts) != 2 { - utils.Log.Infof(fmt.Sprintf("Ignored: invalid auth pair %s", v)) - continue - } - authList[parts[0]] = parts[1] +func authlistResolver() map[string]string { + s3accesskeyid := setting.GetStr(conf.S3AccessKeyId) + s3secretaccesskey := setting.GetStr(conf.S3SecretAccessKey) + if s3accesskeyid == "" && s3secretaccesskey == "" { + return nil } + authList := make(map[string]string) + authList[s3accesskeyid] = s3secretaccesskey return authList } From cf08aa3668d319b4001268f94ef13ed76c1afd70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=8C=E4=B8=AB=E8=AE=B2=E6=A2=B5?= Date: Mon, 25 Mar 2024 22:53:44 +0800 Subject: [PATCH 156/659] feat: add doge driver (#6201) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add doge driver * doc: 补充readme文档 * fix: 对齐meta信息 * fix: 调整结构体名字,与driver保持一致 * perf: merge to s3 * Rename goge.go to doge.go --------- Co-authored-by: Andy Hsu --- .air.toml | 44 +++++++++++++++++++++++++++++++ .gitignore | 1 + README.md | 1 + README_cn.md | 1 + README_ja.md | 1 + drivers/s3/doge.go | 62 ++++++++++++++++++++++++++++++++++++++++++++ drivers/s3/driver.go | 4 ++- drivers/s3/meta.go | 26 +++++++++++++------ drivers/s3/util.go | 12 +++++++-- go.sum | 16 ------------ 10 files changed, 141 insertions(+), 27 deletions(-) create mode 100644 .air.toml create mode 100644 drivers/s3/doge.go diff --git a/.air.toml b/.air.toml new file mode 100644 index 00000000000..52d17fb43da --- /dev/null +++ b/.air.toml @@ -0,0 +1,44 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] +args_bin = ["server"] +bin = "./tmp/main" +cmd = "go build -o ./tmp/main ." +delay = 0 +exclude_dir = ["assets", "tmp", "vendor", "testdata"] +exclude_file = [] +exclude_regex = ["_test.go"] +exclude_unchanged = false +follow_symlink = false +full_bin = "" +include_dir = [] +include_ext = ["go", "tpl", "tmpl", "html"] +include_file = [] +kill_delay = "0s" +log = "build-errors.log" +poll = false +poll_interval = 0 +rerun = false +rerun_delay = 500 +send_interrupt = false +stop_on_error = false + +[color] +app = "" +build = "yellow" +main = "magenta" +runner = "green" +watcher = "cyan" + +[log] +main_only = false +time = false + +[misc] +clean_on_exit = false + +[screen] +clear_on_rebuild = false +keep_scroll = true diff --git a/.gitignore b/.gitignore index 9f5ade897ca..1d71f0d608c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ output/ *.json /build /data/ +/tmp/ /log/ /lang/ /daemon/ diff --git a/README.md b/README.md index 431b32128a5..0935b9cc1f1 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) - [x] [FeijiPan](https://www.feijipan.com/) + - [x] [dogecloud](https://www.dogecloud.com/product/oss) - [x] Easy to deploy and out-of-the-box - [x] File preview (PDF, markdown, code, plain text, ...) - [x] Image preview in gallery mode diff --git a/README_cn.md b/README_cn.md index db8455e1eb8..e6feab09936 100644 --- a/README_cn.md +++ b/README_cn.md @@ -75,6 +75,7 @@ - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) - [x] [飞机盘](https://www.feijipan.com/) + - [x] [多吉云](https://www.dogecloud.com/product/oss) - [x] 部署方便，开箱即用 - [x] 文件预览（PDF、markdown、代码、纯文本……） - [x] 画廊模式下的图像预览 diff --git a/README_ja.md b/README_ja.md index 67b2840a586..2ad984365d7 100644 --- a/README_ja.md +++ b/README_ja.md @@ -76,6 +76,7 @@ - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) - [x] [FeijiPan](https://www.feijipan.com/) + - [x] [dogecloud](https://www.dogecloud.com/product/oss) - [x] デプロイが簡単で、すぐに使える - [x] ファイルプレビュー (PDF, マークダウン, コード, プレーンテキスト, ...) - [x] ギャラリーモードでの画像プレビュー diff --git a/drivers/s3/doge.go b/drivers/s3/doge.go new file mode 100644 index 00000000000..8b516d97e23 --- /dev/null +++ b/drivers/s3/doge.go @@ -0,0 +1,62 @@ +package s3 + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "strings" +) + +type TmpTokenResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data TmpTokenResponseData `json:"data,omitempty"` +} +type TmpTokenResponseData struct { + Credentials Credentials `json:"Credentials"` +} +type Credentials struct { + AccessKeyId string `json:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty"` + SessionToken string `json:"sessionToken,omitempty"` +} + +func getCredentials(AccessKey, SecretKey string) (rst Credentials, err error) { + apiPath := "/auth/tmp_token.json" + reqBody, err := json.Marshal(map[string]interface{}{"channel": "OSS_FULL", "scopes": []string{"*"}}) + if err != nil { + return rst, err + } + + signStr := apiPath + "\n" + string(reqBody) + hmacObj := hmac.New(sha1.New, []byte(SecretKey)) + hmacObj.Write([]byte(signStr)) + sign := hex.EncodeToString(hmacObj.Sum(nil)) + Authorization := "TOKEN " + AccessKey + ":" + sign + + req, err := http.NewRequest("POST", "https://api.dogecloud.com"+apiPath, strings.NewReader(string(reqBody))) + if err != nil { + return rst, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", Authorization) + client := http.Client{} + resp, err := client.Do(req) + if err != nil { + return rst, err + } + defer resp.Body.Close() + ret, err := io.ReadAll(resp.Body) + if err != nil { + return rst, err + } + var tmpTokenResp TmpTokenResponse + err = json.Unmarshal(ret, &tmpTokenResp) + if err != nil { + return rst, err + } + return tmpTokenResp.Data.Credentials, nil +} diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index c8099ee43dd..3209a476c33 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -26,10 +26,12 @@ type S3 struct { Session *session.Session client *s3.S3 linkClient *s3.S3 + + config driver.Config } func (d *S3) Config() driver.Config { - return config + return d.config } func (d *S3) GetAddition() driver.Additional { diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 453f4db72e8..4436c61508e 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -22,15 +22,25 @@ type Addition struct { AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` } -var config = driver.Config{ - Name: "S3", - DefaultRoot: "/", - LocalSort: true, - CheckStatus: true, -} - func init() { op.RegisterDriver(func() driver.Driver { - return &S3{} + return &S3{ + config: driver.Config{ + Name: "S3", + DefaultRoot: "/", + LocalSort: true, + CheckStatus: true, + }, + } + }) + op.RegisterDriver(func() driver.Driver { + return &S3{ + config: driver.Config{ + Name: "Doge", + DefaultRoot: "/", + LocalSort: true, + CheckStatus: true, + }, + } }) } diff --git a/drivers/s3/util.go b/drivers/s3/util.go index 5578176a7a3..31e658bdcab 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -21,13 +21,21 @@ import ( // do others that not defined in Driver interface func (d *S3) initSession() error { + var err error + accessKeyID, secretAccessKey, sessionToken := d.AccessKeyID, d.SecretAccessKey, d.SessionToken + if d.config.Name == "Doge" { + credentialsTmp, err := getCredentials(d.AccessKeyID, d.SecretAccessKey) + if err != nil { + return err + } + accessKeyID, secretAccessKey, sessionToken = credentialsTmp.AccessKeyId, credentialsTmp.SecretAccessKey, credentialsTmp.SessionToken + } cfg := &aws.Config{ - Credentials: credentials.NewStaticCredentials(d.AccessKeyID, d.SecretAccessKey, d.SessionToken), + Credentials: credentials.NewStaticCredentials(accessKeyID, secretAccessKey, sessionToken), Region: &d.Region, Endpoint: &d.Endpoint, S3ForcePathStyle: aws.Bool(d.ForcePathStyle), } - var err error d.Session, err = session.NewSession(cfg) return err } diff --git a/go.sum b/go.sum index a46f0d55cce..3ea10e49e0b 100644 --- a/go.sum +++ b/go.sum @@ -447,8 +447,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4UvMT7OmKhH9sOIvS8iXkxfPdnn9wQA= -github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= @@ -507,12 +505,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= -golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= -golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -532,8 +526,6 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= -golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= @@ -542,8 +534,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -573,8 +563,6 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -583,8 +571,6 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= -golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -610,8 +596,6 @@ golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 8a18f47e68d145b3c5287c7767dc3a33a54ae083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=8C=E4=B8=AB=E8=AE=B2=E6=A2=B5?= Date: Wed, 27 Mar 2024 14:22:26 +0800 Subject: [PATCH 157/659] fix(doge): the temporary access key is only valid for two hours (#6273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add doge driver * doc: 补充readme文档 * fix: 对齐meta信息 * fix: 调整结构体名字,与driver保持一致 * perf: merge to s3 * Rename goge.go to doge.go * fix: 解决多吉云临时秘钥两个小时过期的问题 * fix: 定时任务在Drop中Stop --------- Co-authored-by: Andy Hsu --- drivers/s3/doge.go | 1 + drivers/s3/driver.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/drivers/s3/doge.go b/drivers/s3/doge.go index 8b516d97e23..12a584ca4f2 100644 --- a/drivers/s3/doge.go +++ b/drivers/s3/doge.go @@ -17,6 +17,7 @@ type TmpTokenResponse struct { } type TmpTokenResponseData struct { Credentials Credentials `json:"Credentials"` + ExpiredAt int `json:"ExpiredAt"` } type Credentials struct { AccessKeyId string `json:"accessKeyId,omitempty"` diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 3209a476c33..bc9b42f37f1 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -11,6 +11,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" @@ -28,6 +29,7 @@ type S3 struct { linkClient *s3.S3 config driver.Config + cron *cron.Cron } func (d *S3) Config() driver.Config { @@ -42,6 +44,15 @@ func (d *S3) Init(ctx context.Context) error { if d.Region == "" { d.Region = "alist" } + if d.config.Name == "Doge" { + d.cron = cron.NewCron(time.Minute * 118) + d.cron.Do(func() { + err := d.initSession() + if err != nil { + log.Errorln("Doge init session error:", err) + } + }) + } err := d.initSession() if err != nil { return err @@ -52,6 +63,9 @@ func (d *S3) Init(ctx context.Context) error { } func (d *S3) Drop(ctx context.Context) error { + if d.cron != nil { + d.cron.Stop() + } return nil } From d517adde71d2f71b4bcef3084b976c0e4e7cadfa Mon Sep 17 00:00:00 2001 From: jwcesign Date: Fri, 29 Mar 2024 14:40:43 +0800 Subject: [PATCH 158/659] docs: use width instead of height for image in Readme (#6282) * Update README.md * Update README_cn.md * Update README_ja.md --- README.md | 2 +- README_cn.md | 2 +- README_ja.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0935b9cc1f1..5c557956ffb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- logo + logo

🗂️A file list program that supports multiple storages, powered by Gin and Solidjs.

diff --git a/README_cn.md b/README_cn.md index e6feab09936..90995f8e67b 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@
- logo + logo

🗂一个支持多存储的文件列表程序，使用 Gin 和 Solidjs。

diff --git a/README_ja.md b/README_ja.md index 2ad984365d7..f79dcf204a6 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,5 +1,5 @@
- logo + logo

🗂️Gin と Solidjs による、複数のストレージをサポートするファイルリストプログラム。

From e37465e67e441f767f9032a989c6bb0a419a18e9 Mon Sep 17 00:00:00 2001 From: NewbieOrange Date: Fri, 29 Mar 2024 14:42:01 +0800 Subject: [PATCH 159/659] feat(crypt): force stream upload for supported drivers (#6270) --- drivers/189pc/driver.go | 8 ++++++-- drivers/aliyundrive_open/upload.go | 15 +++++++++------ drivers/crypt/driver.go | 13 +++++++------ internal/model/obj.go | 1 + internal/stream/stream.go | 12 +++++++++--- 5 files changed, 32 insertions(+), 17 deletions(-) diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index f0977995105..382f710bc0a 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -312,13 +312,17 @@ func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // 响应时间长,按需启用 - if y.Addition.RapidUpload { + if y.Addition.RapidUpload && !stream.IsForceStreamUpload() { if newObj, err := y.RapidUpload(ctx, dstDir, stream); err == nil { return newObj, nil } } - switch y.UploadMethod { + uploadMethod := y.UploadMethod + if stream.IsForceStreamUpload() { + uploadMethod = "stream" + } + switch uploadMethod { case "old": return y.OldUpload(ctx, dstDir, stream, up) case "rapid": diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 3b224e7d225..5f57e8b5620 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -164,7 +164,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m count := int(math.Ceil(float64(stream.GetSize()) / float64(partSize))) createData["part_info_list"] = makePartInfos(count) // rapid upload - rapidUpload := stream.GetSize() > 100*utils.KB && d.RapidUpload + rapidUpload := !stream.IsForceStreamUpload() && stream.GetSize() > 100*utils.KB && d.RapidUpload if rapidUpload { log.Debugf("[aliyundrive_open] start cal pre_hash") // read 1024 bytes to calculate pre hash @@ -242,13 +242,16 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m if remain := stream.GetSize() - offset; length > remain { length = remain } - //rd := utils.NewMultiReadable(io.LimitReader(stream, partSize)) - rd, err := stream.RangeRead(http_range.Range{Start: offset, Length: length}) - if err != nil { - return nil, err + rd := utils.NewMultiReadable(io.LimitReader(stream, partSize)) + if rapidUpload { + srd, err := stream.RangeRead(http_range.Range{Start: offset, Length: length}) + if err != nil { + return nil, err + } + rd = utils.NewMultiReadable(srd) } err = retry.Do(func() error { - //rd.Reset() + rd.Reset() return d.uploadPart(ctx, rd, createResp.PartInfoList[i]) }, retry.Attempts(3), diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index 649f47e58c1..b0325db4956 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -3,7 +3,6 @@ package crypt import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" stdpath "path" "regexp" @@ -14,6 +13,7 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -160,7 +160,7 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ // discarding hash as it's encrypted } if d.Thumbnail && thumb == "" { - thumb = utils.EncodePath(common.GetApiUrl(nil) + stdpath.Join("/d", args.ReqPath, ".thumbnails", name+".webp"), true) + thumb = utils.EncodePath(common.GetApiUrl(nil)+stdpath.Join("/d", args.ReqPath, ".thumbnails", name+".webp"), true) } if !ok && !d.Thumbnail { result = append(result, &objRes) @@ -389,10 +389,11 @@ func (d *Crypt) Put(ctx context.Context, dstDir model.Obj, streamer model.FileSt Modified: streamer.ModTime(), IsFolder: streamer.IsDir(), }, - Reader: wrappedIn, - Mimetype: "application/octet-stream", - WebPutAsTask: streamer.NeedStore(), - Exist: streamer.GetExist(), + Reader: wrappedIn, + Mimetype: "application/octet-stream", + WebPutAsTask: streamer.NeedStore(), + ForceStreamUpload: true, + Exist: streamer.GetExist(), } err = op.Put(ctx, d.remoteStorage, dstDirActualPath, streamOut, up, false) if err != nil { diff --git a/internal/model/obj.go b/internal/model/obj.go index dbcdcaf12a3..122fb546278 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -41,6 +41,7 @@ type FileStreamer interface { GetMimetype() string //SetReader(io.Reader) NeedStore() bool + IsForceStreamUpload() bool GetExist() Obj SetExist(Obj) //for a non-seekable Stream, RangeRead supports peeking some data, and CacheFullInTempFile still works diff --git a/internal/stream/stream.go b/internal/stream/stream.go index a45735d035a..4b882c519e0 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -18,9 +18,10 @@ type FileStream struct { Ctx context.Context model.Obj io.Reader - Mimetype string - WebPutAsTask bool - Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it + Mimetype string + WebPutAsTask bool + ForceStreamUpload bool + Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it utils.Closers tmpFile *os.File //if present, tmpFile has full content, it will be deleted at last peekBuff *bytes.Reader @@ -43,6 +44,11 @@ func (f *FileStream) GetMimetype() string { func (f *FileStream) NeedStore() bool { return f.WebPutAsTask } + +func (f *FileStream) IsForceStreamUpload() bool { + return f.ForceStreamUpload +} + func (f *FileStream) Close() error { var err1, err2 error err1 = f.Closers.Close() From 0e86036874a7576c99991dcf6b462a0cce51bb8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=BA=8C=E4=B8=AB=E8=AE=B2=E6=A2=B5?= Date: Fri, 29 Mar 2024 14:56:49 +0800 Subject: [PATCH 160/659] fix(doge): reget client after refresh session (#6277) --- drivers/s3/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index bc9b42f37f1..728c642038c 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -45,12 +45,15 @@ func (d *S3) Init(ctx context.Context) error { d.Region = "alist" } if d.config.Name == "Doge" { + // 多吉云每次临时生成的秘钥有效期为 2h，所以这里设置为 118 分钟重新生成一次 d.cron = cron.NewCron(time.Minute * 118) d.cron.Do(func() { err := d.initSession() if err != nil { log.Errorln("Doge init session error:", err) } + d.client = d.getClient(false) + d.linkClient = d.getClient(true) }) } err := d.initSession() From 2880ed70ce5cae15ffb049c38369b63e06c9a427 Mon Sep 17 00:00:00 2001 From: guangwu Date: Tue, 2 Apr 2024 16:50:30 +0800 Subject: [PATCH 161/659] fix: some typos (#6283) Signed-off-by: guoguangwu --- internal/net/util.go | 2 +- server/handles/ssologin.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/net/util.go b/internal/net/util.go index 99f95c9a420..4347e2c404d 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -327,7 +327,7 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea length_int = int(length) if offset > 100*1024*1024 { - log.Warnf("offset is more than 100MB, if loading data from internet, high-latency and wasting of bandwith is expected") + log.Warnf("offset is more than 100MB, if loading data from internet, high-latency and wasting of bandwidth is expected") } if _, err := io.Copy(io.Discard, io.LimitReader(readCloser, offset)); err != nil { diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index b71179b6496..70298a9c3f0 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -398,7 +398,7 @@ func SSOLoginCallback(c *gin.Context) { } userID := utils.Json.Get(resp.Body(), idField).ToString() if utils.SliceContains([]string{"", "0"}, userID) { - common.ErrorResp(c, errors.New("error occured"), 400) + common.ErrorResp(c, errors.New("error occurred"), 400) return } if argument == "get_sso_id" { From d8e190406a67be0aee4fbc218a80ae69fd22efba Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Tue, 2 Apr 2024 16:51:02 +0800 Subject: [PATCH 162/659] feat(189pc): add family transfer upload (#6288) * feat(189pc): add family transfer upload * fix(189):family transfer file delete --- drivers/189pc/driver.go | 185 ++++++++++++++----------- drivers/189pc/help.go | 16 +++ drivers/189pc/meta.go | 1 + drivers/189pc/types.go | 18 ++- drivers/189pc/utils.go | 299 ++++++++++++++++++++++++++++++++-------- pkg/utils/time.go | 25 ++++ 6 files changed, 406 insertions(+), 138 deletions(-) diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 382f710bc0a..9c01a50fd86 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -1,6 +1,7 @@ package _189pc import ( + "container/ring" "context" "net/http" "strconv" @@ -28,6 +29,9 @@ type Cloud189PC struct { uploadThread int + familyTransferFolder *ring.Ring + cleanFamilyTransferFile func() + storageConfig driver.Config } @@ -52,7 +56,6 @@ func (y *Cloud189PC) Init(ctx context.Context) (err error) { } if !y.isFamily() && y.RootFolderID == "" { y.RootFolderID = "-11" - y.FamilyID = "" } // 限制上传线程数 @@ -79,11 +82,24 @@ func (y *Cloud189PC) Init(ctx context.Context) (err error) { } // 处理家庭云ID - if y.isFamily() && y.FamilyID == "" { + if y.FamilyID == "" { if y.FamilyID, err = y.getFamilyID(); err != nil { return err } } + + // 创建中转文件夹,防止重名文件 + if y.FamilyTransfer { + if y.familyTransferFolder, err = y.createFamilyTransferFolder(32); err != nil { + return err + } + } + + y.cleanFamilyTransferFile = utils.NewThrottle2(time.Minute, func() { + if err := y.cleanFamilyTransfer(context.TODO()); err != nil { + utils.Log.Errorf("cleanFamilyTransferFolderError:%s", err) + } + }) return } @@ -92,7 +108,7 @@ func (y *Cloud189PC) Drop(ctx context.Context) error { } func (y *Cloud189PC) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - return y.getFiles(ctx, dir.GetID()) + return y.getFiles(ctx, dir.GetID(), y.isFamily()) } func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -100,8 +116,9 @@ func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkAr URL string `json:"fileDownloadUrl"` } + isFamily := y.isFamily() fullUrl := API_URL - if y.isFamily() { + if isFamily { fullUrl += "/family/file" } fullUrl += "/getFileDownloadUrl.action" @@ -109,7 +126,7 @@ func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkAr _, err := y.get(fullUrl, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParam("fileId", file.GetID()) - if y.isFamily() { + if isFamily { r.SetQueryParams(map[string]string{ "familyId": y.FamilyID, }) @@ -119,7 +136,7 @@ func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkAr "flag": "1", }) } - }, &downloadUrl) + }, &downloadUrl, isFamily) if err != nil { return nil, err } @@ -156,8 +173,9 @@ func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkAr } func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + isFamily := y.isFamily() fullUrl := API_URL - if y.isFamily() { + if isFamily { fullUrl += "/family/file" } fullUrl += "/createFolder.action" @@ -169,7 +187,7 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s "folderName": dirName, "relativePath": "", }) - if y.isFamily() { + if isFamily { req.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "parentId": parentDir.GetID(), @@ -179,7 +197,7 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s "parentFolderId": parentDir.GetID(), }) } - }, &newFolder) + }, &newFolder, isFamily) if err != nil { return nil, err } @@ -187,27 +205,14 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s } func (y *Cloud189PC) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - var resp CreateBatchTaskResp - _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { - req.SetContext(ctx) - req.SetFormData(map[string]string{ - "type": "MOVE", - "taskInfos": MustString(utils.Json.MarshalToString( - []BatchTaskInfo{ - { - FileId: srcObj.GetID(), - FileName: srcObj.GetName(), - IsFolder: BoolToNumber(srcObj.IsDir()), - }, - })), - "targetFolderId": dstDir.GetID(), - }) - if y.isFamily() { - req.SetFormData(map[string]string{ - "familyId": y.FamilyID, - }) - } - }, &resp) + isFamily := y.isFamily() + other := map[string]string{"targetFileName": dstDir.GetName()} + + resp, err := y.CreateBatchTask("MOVE", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), other, BatchTaskInfo{ + FileId: srcObj.GetID(), + FileName: srcObj.GetName(), + IsFolder: BoolToNumber(srcObj.IsDir()), + }) if err != nil { return nil, err } @@ -218,10 +223,11 @@ func (y *Cloud189PC) Move(ctx context.Context, srcObj, dstDir model.Obj) (model. } func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + isFamily := y.isFamily() queryParam := make(map[string]string) fullUrl := API_URL method := http.MethodPost - if y.isFamily() { + if isFamily { fullUrl += "/family/file" method = http.MethodGet queryParam["familyId"] = y.FamilyID @@ -245,7 +251,7 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin _, err := y.request(fullUrl, method, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(queryParam) - }, nil, newObj) + }, nil, newObj, isFamily) if err != nil { return nil, err } @@ -253,28 +259,15 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin } func (y *Cloud189PC) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - var resp CreateBatchTaskResp - _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { - req.SetContext(ctx) - req.SetFormData(map[string]string{ - "type": "COPY", - "taskInfos": MustString(utils.Json.MarshalToString( - []BatchTaskInfo{ - { - FileId: srcObj.GetID(), - FileName: srcObj.GetName(), - IsFolder: BoolToNumber(srcObj.IsDir()), - }, - })), - "targetFolderId": dstDir.GetID(), - "targetFileName": dstDir.GetName(), - }) - if y.isFamily() { - req.SetFormData(map[string]string{ - "familyId": y.FamilyID, - }) - } - }, &resp) + isFamily := y.isFamily() + other := map[string]string{"targetFileName": dstDir.GetName()} + + resp, err := y.CreateBatchTask("COPY", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), other, BatchTaskInfo{ + FileId: srcObj.GetID(), + FileName: srcObj.GetName(), + IsFolder: BoolToNumber(srcObj.IsDir()), + }) + if err != nil { return err } @@ -282,27 +275,13 @@ func (y *Cloud189PC) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { - var resp CreateBatchTaskResp - _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { - req.SetContext(ctx) - req.SetFormData(map[string]string{ - "type": "DELETE", - "taskInfos": MustString(utils.Json.MarshalToString( - []*BatchTaskInfo{ - { - FileId: obj.GetID(), - FileName: obj.GetName(), - IsFolder: BoolToNumber(obj.IsDir()), - }, - })), - }) + isFamily := y.isFamily() - if y.isFamily() { - req.SetFormData(map[string]string{ - "familyId": y.FamilyID, - }) - } - }, &resp) + resp, err := y.CreateBatchTask("DELETE", IF(isFamily, y.FamilyID, ""), "", nil, BatchTaskInfo{ + FileId: obj.GetID(), + FileName: obj.GetName(), + IsFolder: BoolToNumber(obj.IsDir()), + }) if err != nil { return err } @@ -310,10 +289,13 @@ func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200) } -func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) { + overwrite := true + isFamily := y.isFamily() + // 响应时间长,按需启用 if y.Addition.RapidUpload && !stream.IsForceStreamUpload() { - if newObj, err := y.RapidUpload(ctx, dstDir, stream); err == nil { + if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { return newObj, nil } } @@ -322,17 +304,58 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if stream.IsForceStreamUpload() { uploadMethod = "stream" } + + // 旧版上传家庭云也有限制 + if uploadMethod == "old" { + return y.OldUpload(ctx, dstDir, stream, up, isFamily, overwrite) + } + + // 开启家庭云转存 + if !isFamily && y.FamilyTransfer { + // 修改上传目标为家庭云文件夹 + transferDstDir := dstDir + dstDir = (y.familyTransferFolder.Value).(*Cloud189Folder) + y.familyTransferFolder = y.familyTransferFolder.Next() + + isFamily = true + overwrite = false + + defer func() { + if newObj != nil { + // 批量任务有概率删不掉 + y.cleanFamilyTransferFile() + + // 转存家庭云文件到个人云 + err = y.SaveFamilyFileToPersonCloud(context.TODO(), y.FamilyID, newObj, transferDstDir, true) + + task := BatchTaskInfo{ + FileId: newObj.GetID(), + FileName: newObj.GetName(), + IsFolder: BoolToNumber(newObj.IsDir()), + } + + // 删除源文件 + if resp, err := y.CreateBatchTask("DELETE", y.FamilyID, "", nil, task); err == nil { + y.WaitBatchTask("DELETE", resp.TaskID, time.Second) + // 永久删除 + if resp, err := y.CreateBatchTask("CLEAR_RECYCLE", y.FamilyID, "", nil, task); err == nil { + y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) + } + } + newObj = nil + } + }() + } + switch uploadMethod { - case "old": - return y.OldUpload(ctx, dstDir, stream, up) case "rapid": - return y.FastUpload(ctx, dstDir, stream, up) + return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) case "stream": if stream.GetSize() == 0 { - return y.FastUpload(ctx, dstDir, stream, up) + return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) } fallthrough default: - return y.StreamUpload(ctx, dstDir, stream, up) + return y.StreamUpload(ctx, dstDir, stream, up, isFamily, overwrite) } } diff --git a/drivers/189pc/help.go b/drivers/189pc/help.go index ba1f3f08e39..49f957fab1d 100644 --- a/drivers/189pc/help.go +++ b/drivers/189pc/help.go @@ -192,3 +192,19 @@ func partSize(size int64) int64 { } return DEFAULT } + +func isBool(bs ...bool) bool { + for _, b := range bs { + if b { + return true + } + } + return false +} + +func IF[V any](o bool, t V, f V) V { + if o { + return t + } + return f +} diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 079ac7cc2b9..1891c5c0ccd 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -16,6 +16,7 @@ type Addition struct { FamilyID string `json:"family_id"` UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + FamilyTransfer bool `json:"family_transfer"` RapidUpload bool `json:"rapid_upload"` NoUseOcr bool `json:"no_use_ocr"` } diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index d779659ed16..a1b3810fd23 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -3,10 +3,11 @@ package _189pc import ( "encoding/xml" "fmt" - "github.com/alist-org/alist/v3/pkg/utils" "sort" "strings" "time" + + "github.com/alist-org/alist/v3/pkg/utils" ) // 居然有四种返回方式 @@ -242,7 +243,12 @@ type BatchTaskInfo struct { // IsFolder 是否是文件夹，0-否，1-是 IsFolder int `json:"isFolder"` // SrcParentId 文件所在父目录ID - //SrcParentId string `json:"srcParentId"` + SrcParentId string `json:"srcParentId,omitempty"` + + /* 冲突管理 */ + // 1 -> 跳过 2 -> 保留 3 -> 覆盖 + DealWay int `json:"dealWay,omitempty"` + IsConflict int `json:"isConflict,omitempty"` } /* 上传部分 */ @@ -355,6 +361,14 @@ type BatchTaskStateResp struct { TaskStatus int `json:"taskStatus"` //1 初始化 2 存在冲突 3 执行中，4 完成 } +type BatchTaskConflictTaskInfoResp struct { + SessionKey string `json:"sessionKey"` + TargetFolderID int `json:"targetFolderId"` + TaskID string `json:"taskId"` + TaskInfos []BatchTaskInfo + TaskType int `json:"taskType"` +} + /* query 加密参数*/ type Params map[string]string diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 5e403a830e4..ee96af3e160 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -2,6 +2,7 @@ package _189pc import ( "bytes" + "container/ring" "context" "crypto/md5" "encoding/base64" @@ -54,11 +55,11 @@ const ( CHANNEL_ID = "web_cloud.189.cn" ) -func (y *Cloud189PC) SignatureHeader(url, method, params string) map[string]string { +func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { dateOfGmt := getHttpDateStr() sessionKey := y.tokenInfo.SessionKey sessionSecret := y.tokenInfo.SessionSecret - if y.isFamily() { + if isFamily { sessionKey = y.tokenInfo.FamilySessionKey sessionSecret = y.tokenInfo.FamilySessionSecret } @@ -72,9 +73,9 @@ func (y *Cloud189PC) SignatureHeader(url, method, params string) map[string]stri return header } -func (y *Cloud189PC) EncryptParams(params Params) string { +func (y *Cloud189PC) EncryptParams(params Params, isFamily bool) string { sessionSecret := y.tokenInfo.SessionSecret - if y.isFamily() { + if isFamily { sessionSecret = y.tokenInfo.FamilySessionSecret } if params != nil { @@ -83,17 +84,17 @@ func (y *Cloud189PC) EncryptParams(params Params) string { return "" } -func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, params Params, resp interface{}) ([]byte, error) { +func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, params Params, resp interface{}, isFamily ...bool) ([]byte, error) { req := y.client.R().SetQueryParams(clientSuffix()) // 设置params - paramsData := y.EncryptParams(params) + paramsData := y.EncryptParams(params, isBool(isFamily...)) if paramsData != "" { req.SetQueryParam("params", paramsData) } // Signature - req.SetHeaders(y.SignatureHeader(url, method, paramsData)) + req.SetHeaders(y.SignatureHeader(url, method, paramsData, isBool(isFamily...))) var erron RespErr req.SetError(&erron) @@ -129,15 +130,15 @@ func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, para return res.Body(), nil } -func (y *Cloud189PC) get(url string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - return y.request(url, http.MethodGet, callback, nil, resp) +func (y *Cloud189PC) get(url string, callback base.ReqCallback, resp interface{}, isFamily ...bool) ([]byte, error) { + return y.request(url, http.MethodGet, callback, nil, resp, isFamily...) } -func (y *Cloud189PC) post(url string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - return y.request(url, http.MethodPost, callback, nil, resp) +func (y *Cloud189PC) post(url string, callback base.ReqCallback, resp interface{}, isFamily ...bool) ([]byte, error) { + return y.request(url, http.MethodPost, callback, nil, resp, isFamily...) } -func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]string, sign bool, file io.Reader) ([]byte, error) { +func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]string, sign bool, file io.Reader, isFamily bool) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, file) if err != nil { return nil, err @@ -154,7 +155,7 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str } if sign { - for key, value := range y.SignatureHeader(url, http.MethodPut, "") { + for key, value := range y.SignatureHeader(url, http.MethodPut, "", isFamily) { req.Header.Add(key, value) } } @@ -181,9 +182,9 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str } return body, nil } -func (y *Cloud189PC) getFiles(ctx context.Context, fileId string) ([]model.Obj, error) { +func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) ([]model.Obj, error) { fullUrl := API_URL - if y.isFamily() { + if isFamily { fullUrl += "/family/file" } fullUrl += "/listFiles.action" @@ -201,7 +202,7 @@ func (y *Cloud189PC) getFiles(ctx context.Context, fileId string) ([]model.Obj, "pageNum": fmt.Sprint(pageNum), "pageSize": "130", }) - if y.isFamily() { + if isFamily { r.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "orderBy": toFamilyOrderBy(y.OrderBy), @@ -214,7 +215,7 @@ func (y *Cloud189PC) getFiles(ctx context.Context, fileId string) ([]model.Obj, "descending": toDesc(y.OrderDirection), }) } - }, &resp) + }, &resp, isFamily) if err != nil { return nil, err } @@ -437,7 +438,7 @@ func (y *Cloud189PC) refreshSession() (err error) { // 普通上传 // 无法上传大小为0的文件 -func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { var sliceSize = partSize(file.GetSize()) count := int(math.Ceil(float64(file.GetSize()) / float64(sliceSize))) lastPartSize := file.GetSize() % sliceSize @@ -454,7 +455,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo } fullUrl := UPLOAD_URL - if y.isFamily() { + if isFamily { params.Set("familyId", y.FamilyID) fullUrl += "/family" } else { @@ -466,7 +467,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo var initMultiUpload InitMultiUploadResp _, err := y.request(fullUrl+"/initMultiUpload", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, params, &initMultiUpload) + }, params, &initMultiUpload, isFamily) if err != nil { return nil, err } @@ -502,14 +503,14 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo partInfo := fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) threadG.Go(func(ctx context.Context) error { - uploadUrls, err := y.GetMultiUploadUrls(ctx, initMultiUpload.Data.UploadFileID, partInfo) + uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo) if err != nil { return err } // step.4 上传切片 uploadUrl := uploadUrls[0] - _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, bytes.NewReader(byteData)) + _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, bytes.NewReader(byteData), isFamily) if err != nil { return err } @@ -538,21 +539,21 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo "sliceMd5": sliceMd5Hex, "lazyCheck": "1", "isLog": "0", - "opertype": "3", - }, &resp) + "opertype": IF(overwrite, "3", "1"), + }, &resp, isFamily) if err != nil { return nil, err } return resp.toFile(), nil } -func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer) (model.Obj, error) { +func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, isFamily bool, overwrite bool) (model.Obj, error) { fileMd5 := stream.GetHash().GetHash(utils.MD5) if len(fileMd5) < utils.MD5.Width { return nil, errors.New("invalid hash") } - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, stream.GetName(), fmt.Sprint(stream.GetSize())) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, stream.GetName(), fmt.Sprint(stream.GetSize()), isFamily) if err != nil { return nil, err } @@ -561,11 +562,11 @@ func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream m return nil, errors.New("rapid upload fail") } - return y.OldUploadCommit(ctx, uploadInfo.FileCommitUrl, uploadInfo.UploadFileId) + return y.OldUploadCommit(ctx, uploadInfo.FileCommitUrl, uploadInfo.UploadFileId, isFamily, overwrite) } // 快传 -func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { tempFile, err := file.CacheFullInTempFile() if err != nil { return nil, err @@ -609,7 +610,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } fullUrl := UPLOAD_URL - if y.isFamily() { + if isFamily { fullUrl += "/family" } else { //params.Set("extend", `{"opScene":"1","relativepath":"","rootfolderid":""}`) @@ -628,13 +629,13 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode "sliceSize": fmt.Sprint(sliceSize), "sliceMd5": sliceMd5Hex, } - if y.isFamily() { + if isFamily { params.Set("familyId", y.FamilyID) } var uploadInfo InitMultiUploadResp _, err = y.request(fullUrl+"/initMultiUpload", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, params, &uploadInfo) + }, params, &uploadInfo, isFamily) if err != nil { return nil, err } @@ -659,7 +660,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode i, uploadPart := i, uploadPart threadG.Go(func(ctx context.Context) error { // step.3 获取上传链接 - uploadUrls, err := y.GetMultiUploadUrls(ctx, uploadInfo.UploadFileID, uploadPart) + uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, uploadInfo.UploadFileID, uploadPart) if err != nil { return err } @@ -671,7 +672,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } // step.4 上传切片 - _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, io.NewSectionReader(tempFile, offset, byteSize)) + _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, io.NewSectionReader(tempFile, offset, byteSize), isFamily) if err != nil { return err } @@ -698,8 +699,8 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode }, Params{ "uploadFileId": uploadInfo.UploadFileID, "isLog": "0", - "opertype": "3", - }, &resp) + "opertype": IF(overwrite, "3", "1"), + }, &resp, isFamily) if err != nil { return nil, err } @@ -708,9 +709,9 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode // 获取上传切片信息 // 对http body有大小限制，分片信息太多会出错 -func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, uploadFileId string, partInfo ...string) ([]UploadUrlInfo, error) { +func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, isFamily bool, uploadFileId string, partInfo ...string) ([]UploadUrlInfo, error) { fullUrl := UPLOAD_URL - if y.isFamily() { + if isFamily { fullUrl += "/family" } else { fullUrl += "/person" @@ -723,7 +724,7 @@ func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, uploadFileId string }, Params{ "uploadFileId": uploadFileId, "partInfo": strings.Join(partInfo, ","), - }, &uploadUrlsResp) + }, &uploadUrlsResp, isFamily) if err != nil { return nil, err } @@ -752,7 +753,7 @@ func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, uploadFileId string } // 旧版本上传，家庭云不支持覆盖 -func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { tempFile, err := file.CacheFullInTempFile() if err != nil { return nil, err @@ -763,7 +764,7 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model } // 创建上传会话 - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize())) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize()), isFamily) if err != nil { return nil, err } @@ -780,14 +781,14 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model "Expect": "100-continue", } - if y.isFamily() { + if isFamily { header["FamilyId"] = fmt.Sprint(y.FamilyID) header["UploadFileId"] = fmt.Sprint(status.UploadFileId) } else { header["Edrive-UploadFileId"] = fmt.Sprint(status.UploadFileId) } - _, err := y.put(ctx, status.FileUploadUrl, header, true, io.NopCloser(tempFile)) + _, err := y.put(ctx, status.FileUploadUrl, header, true, io.NopCloser(tempFile), isFamily) if err, ok := err.(*RespErr); ok && err.Code != "InputStreamReadError" { return nil, err } @@ -802,10 +803,10 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model "uploadFileId": fmt.Sprint(status.UploadFileId), "resumePolicy": "1", }) - if y.isFamily() { + if isFamily { req.SetQueryParam("familyId", fmt.Sprint(y.FamilyID)) } - }, &status) + }, &status, isFamily) if err != nil { return nil, err } @@ -815,20 +816,20 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model up(float64(status.GetSize()) / float64(file.GetSize()) * 100) } - return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId) + return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId, isFamily, overwrite) } // 创建上传会话 -func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileMd5, fileName, fileSize string) (*CreateUploadFileResp, error) { +func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileMd5, fileName, fileSize string, isFamily bool) (*CreateUploadFileResp, error) { var uploadInfo CreateUploadFileResp fullUrl := API_URL + "/createUploadFile.action" - if y.isFamily() { + if isFamily { fullUrl = API_URL + "/family/file/createFamilyFile.action" } _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) - if y.isFamily() { + if isFamily { req.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "parentId": parentID, @@ -849,7 +850,7 @@ func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileM "isLog": "0", }) } - }, &uploadInfo) + }, &uploadInfo, isFamily) if err != nil { return nil, err @@ -858,11 +859,11 @@ func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileM } // 提交上传文件 -func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string, uploadFileID int64) (model.Obj, error) { +func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string, uploadFileID int64, isFamily bool, overwrite bool) (model.Obj, error) { var resp OldCommitUploadFileResp _, err := y.post(fileCommitUrl, func(req *resty.Request) { req.SetContext(ctx) - if y.isFamily() { + if isFamily { req.SetHeaders(map[string]string{ "ResumePolicy": "1", "UploadFileId": fmt.Sprint(uploadFileID), @@ -870,13 +871,13 @@ func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string, }) } else { req.SetFormData(map[string]string{ - "opertype": "3", + "opertype": IF(overwrite, "3", "1"), "resumePolicy": "1", "uploadFileId": fmt.Sprint(uploadFileID), "isLog": "0", }) } - }, &resp) + }, &resp, isFamily) if err != nil { return nil, err } @@ -895,10 +896,100 @@ func (y *Cloud189PC) isLogin() bool { return err == nil } +// 创建家庭云中转文件夹 +func (y *Cloud189PC) createFamilyTransferFolder(count int) (*ring.Ring, error) { + folders := ring.New(count) + var rootFolder Cloud189Folder + _, err := y.post(API_URL+"/family/file/createFolder.action", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "folderName": "FamilyTransferFolder", + "familyId": y.FamilyID, + }) + }, &rootFolder, true) + if err != nil { + return nil, err + } + + folderCount := 0 + + // 获取已有目录 + files, err := y.getFiles(context.TODO(), rootFolder.GetID(), true) + if err != nil { + return nil, err + } + for _, file := range files { + if folder, ok := file.(*Cloud189Folder); ok { + folders.Value = folder + folders = folders.Next() + folderCount++ + } + } + + // 创建新的目录 + for folderCount < count { + var newFolder Cloud189Folder + _, err := y.post(API_URL+"/family/file/createFolder.action", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "folderName": uuid.NewString(), + "familyId": y.FamilyID, + "parentId": rootFolder.GetID(), + }) + }, &newFolder, true) + if err != nil { + return nil, err + } + folders.Value = &newFolder + folders = folders.Next() + folderCount++ + } + return folders, nil +} + +// 清理中转文件夹 +func (y *Cloud189PC) cleanFamilyTransfer(ctx context.Context) error { + var tasks []BatchTaskInfo + r := y.familyTransferFolder + for p := r.Next(); p != r; p = p.Next() { + folder := p.Value.(*Cloud189Folder) + + files, err := y.getFiles(ctx, folder.GetID(), true) + if err != nil { + return err + } + for _, file := range files { + tasks = append(tasks, BatchTaskInfo{ + FileId: file.GetID(), + FileName: file.GetName(), + IsFolder: BoolToNumber(file.IsDir()), + }) + } + } + + if len(tasks) > 0 { + // 删除 + resp, err := y.CreateBatchTask("DELETE", y.FamilyID, "", nil, tasks...) + if err != nil { + return err + } + err = y.WaitBatchTask("DELETE", resp.TaskID, time.Second) + if err != nil { + return err + } + // 永久删除 + resp, err = y.CreateBatchTask("CLEAR_RECYCLE", y.FamilyID, "", nil, tasks...) + if err != nil { + return err + } + err = y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) + return err + } + return nil +} + // 获取家庭云所有用户信息 func (y *Cloud189PC) getFamilyInfoList() ([]FamilyInfoResp, error) { var resp FamilyInfoListResp - _, err := y.get(API_URL+"/family/manage/getFamilyList.action", nil, &resp) + _, err := y.get(API_URL+"/family/manage/getFamilyList.action", nil, &resp, true) if err != nil { return nil, err } @@ -922,6 +1013,73 @@ func (y *Cloud189PC) getFamilyID() (string, error) { return fmt.Sprint(infos[0].FamilyID), nil } +// 保存家庭云中的文件到个人云 +func (y *Cloud189PC) SaveFamilyFileToPersonCloud(ctx context.Context, familyId string, srcObj, dstDir model.Obj, overwrite bool) error { + // _, err := y.post(API_URL+"/family/file/saveFileToMember.action", func(req *resty.Request) { + // req.SetQueryParams(map[string]string{ + // "channelId": "home", + // "familyId": familyId, + // "destParentId": destParentId, + // "fileIdList": familyFileId, + // }) + // }, nil) + // return err + + task := BatchTaskInfo{ + FileId: srcObj.GetID(), + FileName: srcObj.GetName(), + IsFolder: BoolToNumber(srcObj.IsDir()), + } + resp, err := y.CreateBatchTask("COPY", familyId, dstDir.GetID(), map[string]string{ + "groupId": "null", + "copyType": "2", + "shareId": "null", + }, task) + if err != nil { + return err + } + + for { + state, err := y.CheckBatchTask("COPY", resp.TaskID) + if err != nil { + return err + } + switch state.TaskStatus { + case 2: + task.DealWay = IF(overwrite, 3, 2) + // 冲突时覆盖文件 + if err := y.ManageBatchTask("COPY", resp.TaskID, dstDir.GetID(), task); err != nil { + return err + } + case 4: + return nil + } + time.Sleep(time.Millisecond * 400) + } +} + +func (y *Cloud189PC) CreateBatchTask(aType string, familyID string, targetFolderId string, other map[string]string, taskInfos ...BatchTaskInfo) (*CreateBatchTaskResp, error) { + var resp CreateBatchTaskResp + _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { + req.SetFormData(map[string]string{ + "type": aType, + "taskInfos": MustString(utils.Json.MarshalToString(taskInfos)), + }) + if targetFolderId != "" { + req.SetFormData(map[string]string{"targetFolderId": targetFolderId}) + } + if familyID != "" { + req.SetFormData(map[string]string{"familyId": familyID}) + } + req.SetFormData(other) + }, &resp, familyID != "") + if err != nil { + return nil, err + } + return &resp, nil +} + +// 检测任务状态 func (y *Cloud189PC) CheckBatchTask(aType string, taskID string) (*BatchTaskStateResp, error) { var resp BatchTaskStateResp _, err := y.post(API_URL+"/batch/checkBatchTask.action", func(req *resty.Request) { @@ -936,6 +1094,37 @@ func (y *Cloud189PC) CheckBatchTask(aType string, taskID string) (*BatchTaskStat return &resp, nil } +// 获取冲突的任务信息 +func (y *Cloud189PC) GetConflictTaskInfo(aType string, taskID string) (*BatchTaskConflictTaskInfoResp, error) { + var resp BatchTaskConflictTaskInfoResp + _, err := y.post(API_URL+"/batch/getConflictTaskInfo.action", func(req *resty.Request) { + req.SetFormData(map[string]string{ + "type": aType, + "taskId": taskID, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +// 处理冲突 +func (y *Cloud189PC) ManageBatchTask(aType string, taskID string, targetFolderId string, taskInfos ...BatchTaskInfo) error { + _, err := y.post(API_URL+"/batch/manageBatchTask.action", func(req *resty.Request) { + req.SetFormData(map[string]string{ + "targetFolderId": targetFolderId, + "type": aType, + "taskId": taskID, + "taskInfos": MustString(utils.Json.MarshalToString(taskInfos)), + }) + }, nil) + return err +} + +var ErrIsConflict = errors.New("there is a conflict with the target object") + +// 等待任务完成 func (y *Cloud189PC) WaitBatchTask(aType string, taskID string, t time.Duration) error { for { state, err := y.CheckBatchTask(aType, taskID) @@ -944,7 +1133,7 @@ func (y *Cloud189PC) WaitBatchTask(aType string, taskID string, t time.Duration) } switch state.TaskStatus { case 2: - return errors.New("there is a conflict with the target object") + return ErrIsConflict case 4: return nil } diff --git a/pkg/utils/time.go b/pkg/utils/time.go index a9d9b5b674c..aa7069282fb 100644 --- a/pkg/utils/time.go +++ b/pkg/utils/time.go @@ -37,3 +37,28 @@ func NewDebounce2(interval time.Duration, f func()) func() { (*time.Timer)(timer).Reset(interval) } } + +func NewThrottle(interval time.Duration) func(func()) { + var lastCall time.Time + + return func(fn func()) { + now := time.Now() + if now.Sub(lastCall) < interval { + return + } + time.AfterFunc(interval, fn) + lastCall = now + } +} + +func NewThrottle2(interval time.Duration, fn func()) func() { + var lastCall time.Time + return func() { + now := time.Now() + if now.Sub(lastCall) < interval { + return + } + time.AfterFunc(interval, fn) + lastCall = now + } +} From 58c3cb3cf65655627db5c6c9970fa98aa71f8d6d Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Wed, 3 Apr 2024 10:09:48 +0800 Subject: [PATCH 163/659] fix(s3): don't bind s3 port if s3 is not enabled (#6291) --- cmd/server.go | 8 ++++---- server/s3.go | 6 ------ 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 3c7137bcf6e..8a7beafa7fd 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -91,10 +91,10 @@ the address is defined in config file`, } }() } - s3r := gin.New() - s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) - server.InitS3(s3r) - if conf.Conf.S3.Port != -1 { + if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { + s3r := gin.New() + s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) + server.InitS3(s3r) s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) utils.Log.Infof("start S3 server @ %s", s3Base) go func() { diff --git a/server/s3.go b/server/s3.go index 1a9f2e038c9..21b95527ded 100644 --- a/server/s3.go +++ b/server/s3.go @@ -34,12 +34,6 @@ func S3(g *gin.RouterGroup) { } func S3Server(g *gin.RouterGroup) { - if !conf.Conf.S3.Enable { - g.Any("/*path", func(c *gin.Context) { - common.ErrorStrResp(c, "S3 server is not enabled", 403) - }) - return - } h, _ := s3.NewServer(context.Background()) g.Any("/*path", gin.WrapH(h)) } From 1756036a21af133a47d834dd121a47f0bb91a4ca Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Wed, 3 Apr 2024 14:33:19 +0800 Subject: [PATCH 164/659] fix(authn): subfolder api is considered as a wrong origin(closes #6294 in #6301) --- internal/authn/authn.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/authn/authn.go b/internal/authn/authn.go index df1d1fc6ff2..ea621d048af 100644 --- a/internal/authn/authn.go +++ b/internal/authn/authn.go @@ -1,6 +1,7 @@ package authn import ( + "fmt" "net/http" "net/url" @@ -19,7 +20,7 @@ func NewAuthnInstance(r *http.Request) (*webauthn.WebAuthn, error) { RPDisplayName: setting.GetStr(conf.SiteTitle), RPID: siteUrl.Hostname(), //RPOrigin: siteUrl.String(), - RPOrigins: []string{siteUrl.String()}, + RPOrigins: []string{fmt.Sprintf("%s://%s", siteUrl.Scheme, siteUrl.Host)}, // RPOrigin: "http://localhost:5173" }) } From cd5a8a011db03fbb13e002e38bcc8c08672de556 Mon Sep 17 00:00:00 2001 From: Mix <32300164+mnixry@users.noreply.github.com> Date: Mon, 8 Apr 2024 18:35:23 +0800 Subject: [PATCH 165/659] fix: typo about env of `Meilisearch` (#6316) --- internal/conf/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/conf/config.go b/internal/conf/config.go index ef41192a778..742aab1f265 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -77,7 +77,7 @@ type Config struct { JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` Database Database `json:"database" envPrefix:"DB_"` - Meilisearch Meilisearch `json:"meilisearch" env:"MEILISEARCH"` + Meilisearch Meilisearch `json:"meilisearch" envPrefix:"MEILISEARCH_"` Scheme Scheme `json:"scheme"` TempDir string `json:"temp_dir" env:"TEMP_DIR"` BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` From c3c518184706e195ea14d127b726c73bad4a4a49 Mon Sep 17 00:00:00 2001 From: tukipona <161995125+tukipona@users.noreply.github.com> Date: Wed, 10 Apr 2024 21:50:30 +0800 Subject: [PATCH 166/659] feat(Seafile): add token login (#6324 close #5302) --- drivers/seafile/meta.go | 5 +++-- drivers/seafile/util.go | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/seafile/meta.go b/drivers/seafile/meta.go index 9d84aee182f..fd5255f592b 100644 --- a/drivers/seafile/meta.go +++ b/drivers/seafile/meta.go @@ -9,8 +9,9 @@ type Addition struct { driver.RootPath Address string `json:"address" required:"true"` - UserName string `json:"username" required:"true"` - Password string `json:"password" required:"true"` + UserName string `json:"username" required:"false"` + Password string `json:"password" required:"false"` + Token string `json:"token" required:"false"` RepoId string `json:"repoId" required:"false"` RepoPwd string `json:"repoPwd" required:"false"` } diff --git a/drivers/seafile/util.go b/drivers/seafile/util.go index 681953103c3..89b7b0fc39f 100644 --- a/drivers/seafile/util.go +++ b/drivers/seafile/util.go @@ -14,6 +14,10 @@ import ( ) func (d *Seafile) getToken() error { + if d.Token != "" { + d.authorization = fmt.Sprintf("Token %s", d.Token) + return nil + } var authResp AuthTokenResp res, err := base.RestyClient.R(). SetResult(&authResp). From 793a4ea6ca45e0b9fd44d0cbcddd563d83749d8f Mon Sep 17 00:00:00 2001 From: tukipona <161995125+tukipona@users.noreply.github.com> Date: Fri, 12 Apr 2024 21:45:16 +0800 Subject: [PATCH 167/659] fix(cloudreve): add domain to the download url if not exists (#6339 close #6265) * fix: correct the download url got by Cloudreve driver * fix: add an condition to the correction --- drivers/cloudreve/driver.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 49c2d5f00f2..dc6d1b13213 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -71,6 +71,9 @@ func (d *Cloudreve) Link(ctx context.Context, file model.Obj, args model.LinkArg if err != nil { return nil, err } + if strings.HasPrefix(dUrl, "/api") { + dUrl = d.Address + dUrl + } return &model.Link{ URL: dUrl, }, nil From 0c9dcec9cdcb0416e630116adee090b4640635e8 Mon Sep 17 00:00:00 2001 From: jack roble <74554363+0daysseus@users.noreply.github.com> Date: Fri, 19 Apr 2024 05:22:16 -0400 Subject: [PATCH 168/659] fix: init storages in order (#6346) --- internal/bootstrap/storage.go | 4 ++-- internal/db/storage.go | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/storage.go b/internal/bootstrap/storage.go index 44af99c56e1..86288edcb16 100644 --- a/internal/bootstrap/storage.go +++ b/internal/bootstrap/storage.go @@ -21,8 +21,8 @@ func LoadStorages() { if err != nil { utils.Log.Errorf("failed get enabled storages: %+v", err) } else { - utils.Log.Infof("success load storage: [%s], driver: [%s]", - storages[i].MountPath, storages[i].Driver) + utils.Log.Infof("success load storage: [%s], driver: [%s], order: [%d]", + storages[i].MountPath, storages[i].Driver, storages[i].Order) } } conf.StoragesLoaded = true diff --git a/internal/db/storage.go b/internal/db/storage.go index 105bc0aafda..d4e0730f064 100644 --- a/internal/db/storage.go +++ b/internal/db/storage.go @@ -2,6 +2,7 @@ package db import ( "fmt" + "sort" "github.com/alist-org/alist/v3/internal/model" "github.com/pkg/errors" @@ -65,5 +66,8 @@ func GetEnabledStorages() ([]model.Storage, error) { if err := db.Where(fmt.Sprintf("%s = ?", columnName("disabled")), false).Find(&storages).Error; err != nil { return nil, errors.WithStack(err) } + sort.Slice(storages, func(i, j int) bool { + return storages[i].Order < storages[j].Order + }) return storages, nil } From 32ddab9b0131885045971b3d6bd59605d4ffc9ac Mon Sep 17 00:00:00 2001 From: Xiaoran Studio Date: Wed, 24 Apr 2024 14:54:01 +0800 Subject: [PATCH 169/659] feat(123_share): add access token (#6357) --- drivers/123_share/driver.go | 11 ++++++++ drivers/123_share/meta.go | 3 ++- drivers/123_share/util.go | 50 +++++++++++++++++++++++++++++++------ 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/drivers/123_share/driver.go b/drivers/123_share/driver.go index b2fd4313331..7fca7cc145e 100644 --- a/drivers/123_share/driver.go +++ b/drivers/123_share/driver.go @@ -4,8 +4,11 @@ import ( "context" "encoding/base64" "fmt" + "golang.org/x/time/rate" "net/http" "net/url" + "sync" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -19,6 +22,7 @@ import ( type Pan123Share struct { model.Storage Addition + apiRateLimit sync.Map } func (d *Pan123Share) Config() driver.Config { @@ -146,4 +150,11 @@ func (d *Pan123Share) Put(ctx context.Context, dstDir model.Obj, stream model.Fi // return nil, errs.NotSupport //} +func (d *Pan123Share) APIRateLimit(api string) bool { + limiter, _ := d.apiRateLimit.LoadOrStore(api, + rate.NewLimiter(rate.Every(time.Millisecond*700), 1)) + ins := limiter.(*rate.Limiter) + return ins.Allow() +} + var _ driver.Driver = (*Pan123Share)(nil) diff --git a/drivers/123_share/meta.go b/drivers/123_share/meta.go index a4bb14a9593..ce39b7eee07 100644 --- a/drivers/123_share/meta.go +++ b/drivers/123_share/meta.go @@ -7,10 +7,11 @@ import ( type Addition struct { ShareKey string `json:"sharekey" required:"true"` - SharePwd string `json:"sharepassword" required:"true"` + SharePwd string `json:"sharepassword"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + AccessToken string `json:"accesstoken" type:"text"` } var config = driver.Config{ diff --git a/drivers/123_share/util.go b/drivers/123_share/util.go index bfce54f3cc0..b22b7cc4547 100644 --- a/drivers/123_share/util.go +++ b/drivers/123_share/util.go @@ -2,8 +2,15 @@ package _123Share import ( "errors" + "fmt" + "hash/crc32" + "math" + "math/rand" "net/http" + "net/url" "strconv" + "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" @@ -15,20 +22,45 @@ const ( Api = "https://www.123pan.com/api" AApi = "https://www.123pan.com/a/api" BApi = "https://www.123pan.com/b/api" - MainApi = Api + MainApi = BApi FileList = MainApi + "/share/get" DownloadInfo = MainApi + "/share/download/info" //AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) +func signPath(path string, os string, version string) (k string, v string) { + table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'} + random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64())) + now := time.Now().In(time.FixedZone("CST", 8*3600)) + timestamp := fmt.Sprint(now.Unix()) + nowStr := []byte(now.Format("200601021504")) + for i := 0; i < len(nowStr); i++ { + nowStr[i] = table[nowStr[i]-48] + } + timeSign := fmt.Sprint(crc32.ChecksumIEEE(nowStr)) + data := strings.Join([]string{timestamp, random, path, os, version, timeSign}, "|") + dataSign := fmt.Sprint(crc32.ChecksumIEEE([]byte(data))) + return timeSign, strings.Join([]string{timestamp, random, dataSign}, "-") +} + +func GetApi(rawUrl string) string { + u, _ := url.Parse(rawUrl) + query := u.Query() + query.Add(signPath(u.Path, "web", "3")) + u.RawQuery = query.Encode() + return u.String() +} + func (d *Pan123Share) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ - "origin": "https://www.123pan.com", - "referer": "https://www.123pan.com/", - "user-agent": "Dart/2.19(dart:io)", - "platform": "android", - "app-version": "36", + "origin": "https://www.123pan.com", + "referer": "https://www.123pan.com/", + "authorization": "Bearer " + d.AccessToken, + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) alist-client", + "platform": "web", + "app-version": "3", + //"user-agent": base.UserAgent, }) if callback != nil { callback(req) @@ -36,7 +68,7 @@ func (d *Pan123Share) request(url string, method string, callback base.ReqCallba if resp != nil { req.SetResult(resp) } - res, err := req.Execute(method, url) + res, err := req.Execute(method, GetApi(url)) if err != nil { return nil, err } @@ -52,6 +84,10 @@ func (d *Pan123Share) getFiles(parentId string) ([]File, error) { page := 1 res := make([]File, 0) for { + if !d.APIRateLimit(FileList) { + time.Sleep(time.Millisecond * 200) + continue + } var resp Files query := map[string]string{ "limit": "100", From 479fc6d4663aa5fd137ea56f7b6ba81377c6442e Mon Sep 17 00:00:00 2001 From: potoo <34411681+potoo0@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:13:30 +0800 Subject: [PATCH 170/659] fix(webdav): make sure `Mtime` after `Ctime` (#6372 close #6371) * fix(server/webdav) make sure Mtime >= Ctime * fix(server/webdav) avoid variable 'stream' collides with imported package name --- server/webdav/util.go | 13 +++++++++---- server/webdav/webdav.go | 10 +++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/server/webdav/util.go b/server/webdav/util.go index 15d9e07cc56..a2a1641ce62 100644 --- a/server/webdav/util.go +++ b/server/webdav/util.go @@ -8,16 +8,21 @@ import ( ) func (h *Handler) getModTime(r *http.Request) time.Time { - return h.getHeaderTime(r, "X-OC-Mtime") + return h.getHeaderTime(r, "X-OC-Mtime", "") } -// owncloud/ nextcloud haven't impl this, but we can add the support since rclone may support this soon +// owncloud/ nextcloud haven't impl this, but we can add the support since rclone may support this soon. +// try ModTime if CreateTime not found in header func (h *Handler) getCreateTime(r *http.Request) time.Time { - return h.getHeaderTime(r, "X-OC-Ctime") + return h.getHeaderTime(r, "X-OC-Ctime", "X-OC-Mtime") } -func (h *Handler) getHeaderTime(r *http.Request, header string) time.Time { +func (h *Handler) getHeaderTime(r *http.Request, header, alternative string) time.Time { hVal := r.Header.Get(header) + // try alternative + if hVal == "" && alternative != "" { + hVal = r.Header.Get(alternative) + } if hVal != "" { modTimeUnix, err := strconv.ParseInt(hVal, 10, 64) if err == nil { diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 390e5409976..6054991a0c2 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -331,21 +331,21 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, Modified: h.getModTime(r), Ctime: h.getCreateTime(r), } - stream := &stream.FileStream{ + fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body, Mimetype: r.Header.Get("Content-Type"), } - if stream.Mimetype == "" { - stream.Mimetype = utils.GetMimeType(reqPath) + if fsStream.Mimetype == "" { + fsStream.Mimetype = utils.GetMimeType(reqPath) } - err = fs.PutDirectly(ctx, path.Dir(reqPath), stream) + err = fs.PutDirectly(ctx, path.Dir(reqPath), fsStream) if errs.IsNotFoundError(err) { return http.StatusNotFound, err } _ = r.Body.Close() - _ = stream.Close() + _ = fsStream.Close() // TODO(rost): Returning 405 Method Not Allowed might not be appropriate. if err != nil { return http.StatusMethodNotAllowed, err From ec08ecdf6cdb69bb776d5fbb539a842ef9c946d3 Mon Sep 17 00:00:00 2001 From: potoo <34411681+potoo0@users.noreply.github.com> Date: Thu, 25 Apr 2024 20:08:20 +0800 Subject: [PATCH 171/659] fix(baidu_netdisk): cached Ctime/Mtime (#6373 close #6370) (cherry picked from commit 23542541e4f343d484de1f83ee5c928d2ab6753c) --- drivers/baidu_netdisk/driver.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 20810a768de..43da834a143 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -165,9 +165,16 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo if err != nil { return nil, err } + // 修复时间，具体原因见 Put 方法注释的 **注意** + newFile.Ctime = stream.CreateTime().Unix() + newFile.Mtime = stream.ModTime().Unix() return fileToObj(newFile), nil } +// Put +// +// **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 +// 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // rapid upload if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { @@ -245,9 +252,9 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F log.Debugf("%+v", precreateResp) if precreateResp.ReturnType == 2 { //rapid upload, since got md5 match from baidu server - if err != nil { - return nil, err - } + // 修复时间，具体原因见 Put 方法注释的 **注意** + precreateResp.File.Ctime = ctime + precreateResp.File.Mtime = mtime return fileToObj(precreateResp.File), nil } } @@ -298,6 +305,9 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if err != nil { return nil, err } + // 修复时间，具体原因见 Put 方法注释的 **注意** + newFile.Ctime = ctime + newFile.Mtime = mtime return fileToObj(newFile), nil } From b95df1d7457c6a72cf7e8242061e8a52fc9a3095 Mon Sep 17 00:00:00 2001 From: Mmx Date: Thu, 25 Apr 2024 20:11:15 +0800 Subject: [PATCH 172/659] perf: use io copy with buffer pool (#6389) * feat: add io methods with buffer * chore: move io.Copy calls to utils.CopyWithBuffer --- drivers/123/driver.go | 2 +- drivers/189pc/utils.go | 2 +- drivers/aliyundrive/driver.go | 2 +- drivers/aliyundrive_open/upload.go | 2 +- drivers/baidu_netdisk/driver.go | 2 +- drivers/baidu_photo/driver.go | 2 +- drivers/chaoxing/driver.go | 2 +- drivers/ilanzou/driver.go | 2 +- drivers/mediatrack/driver.go | 2 +- drivers/pikpak/util.go | 3 ++- drivers/quark_uc/driver.go | 4 ++-- drivers/smb/util.go | 4 ++-- drivers/thunder/util.go | 2 +- internal/net/request.go | 3 ++- internal/net/serve.go | 4 ++-- internal/net/util.go | 3 ++- internal/stream/stream.go | 2 +- pkg/gowebdav/client.go | 3 ++- pkg/utils/file.go | 4 ++-- pkg/utils/hash.go | 2 +- pkg/utils/hash_test.go | 3 +-- pkg/utils/io.go | 31 +++++++++++++++++++++++++++++- 22 files changed, 59 insertions(+), 27 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index f5d981ef636..240027405d5 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -194,7 +194,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr defer func() { _ = tempFile.Close() }() - if _, err = io.Copy(h, tempFile); err != nil { + if _, err = utils.CopyWithBuffer(h, tempFile); err != nil { return err } _, err = tempFile.Seek(0, io.SeekStart) diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index ee96af3e160..a000a84e005 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -595,7 +595,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } silceMd5.Reset() - if _, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5), tempFile, byteSize); err != nil && err != io.EOF { + if _, err := utils.CopyWithBufferN(io.MultiWriter(fileMd5, silceMd5), tempFile, byteSize); err != nil && err != io.EOF { return nil, err } md5Byte := silceMd5.Sum(nil) diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index eab38f58e1c..2a977aa35e5 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -194,7 +194,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil } if d.RapidUpload { buf := bytes.NewBuffer(make([]byte, 0, 1024)) - io.CopyN(buf, file, 1024) + utils.CopyWithBufferN(buf, file, 1024) reqBody["pre_hash"] = utils.HashData(utils.SHA1, buf.Bytes()) if localFile != nil { if _, err := localFile.Seek(0, io.SeekStart); err != nil { diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 5f57e8b5620..d152836c075 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -136,7 +136,7 @@ func (d *AliyundriveOpen) calProofCode(stream model.FileStreamer) (string, error if err != nil { return "", err } - _, err = io.CopyN(buf, reader, length) + _, err = utils.CopyWithBufferN(buf, reader, length) if err != nil { return "", err } diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 43da834a143..ad52a4b5438 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -211,7 +211,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if i == count { byteSize = lastBlockSize } - _, err := io.CopyN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize) + _, err := utils.CopyWithBufferN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize) if err != nil && err != io.EOF { return nil, err } diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index c29bc110095..7477a8eb527 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -261,7 +261,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if i == count { byteSize = lastBlockSize } - _, err := io.CopyN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize) + _, err := utils.CopyWithBufferN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize) if err != nil && err != io.EOF { return nil, err } diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index 143235fa481..de122c36c4d 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -229,7 +229,7 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, stream model.FileS if err != nil { return err } - _, err = io.Copy(filePart, stream) + _, err = utils.CopyWithBuffer(filePart, stream) if err != nil { return err } diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 341136da1cd..1d8e5d36b09 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -271,7 +271,7 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt defer func() { _ = tempFile.Close() }() - if _, err = io.Copy(h, tempFile); err != nil { + if _, err = utils.CopyWithBuffer(h, tempFile); err != nil { return nil, err } _, err = tempFile.Seek(0, io.SeekStart) diff --git a/drivers/mediatrack/driver.go b/drivers/mediatrack/driver.go index ef571832eb7..f0f1ded0087 100644 --- a/drivers/mediatrack/driver.go +++ b/drivers/mediatrack/driver.go @@ -206,7 +206,7 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil return err } h := md5.New() - _, err = io.Copy(h, tempFile) + _, err = utils.CopyWithBuffer(h, tempFile) if err != nil { return err } diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 02b988bcd64..71ad1dca8a3 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -4,6 +4,7 @@ import ( "crypto/sha1" "encoding/hex" "errors" + "github.com/alist-org/alist/v3/pkg/utils" "io" "net/http" @@ -141,7 +142,7 @@ func getGcid(r io.Reader, size int64) (string, error) { readSize := calcBlockSize(size) for { hash2.Reset() - if n, err := io.CopyN(hash2, r, readSize); err != nil && n == 0 { + if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { if err != io.EOF { return "", err } diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 291189ce088..8674fbab26f 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -143,7 +143,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File _ = tempFile.Close() }() m := md5.New() - _, err = io.Copy(m, tempFile) + _, err = utils.CopyWithBuffer(m, tempFile) if err != nil { return err } @@ -153,7 +153,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File } md5Str := hex.EncodeToString(m.Sum(nil)) s := sha1.New() - _, err = io.Copy(s, tempFile) + _, err = utils.CopyWithBuffer(s, tempFile) if err != nil { return err } diff --git a/drivers/smb/util.go b/drivers/smb/util.go index f4605536da7..d9fbf6c5a5a 100644 --- a/drivers/smb/util.go +++ b/drivers/smb/util.go @@ -1,7 +1,7 @@ package smb import ( - "io" + "github.com/alist-org/alist/v3/pkg/utils" "io/fs" "net" "os" @@ -74,7 +74,7 @@ func (d *SMB) CopyFile(src, dst string) error { } defer dstfd.Close() - if _, err = io.Copy(dstfd, srcfd); err != nil { + if _, err = utils.CopyWithBuffer(dstfd, srcfd); err != nil { return err } if srcinfo, err = d.fs.Stat(src); err != nil { diff --git a/drivers/thunder/util.go b/drivers/thunder/util.go index f6dec3260cf..3ec8db58ffe 100644 --- a/drivers/thunder/util.go +++ b/drivers/thunder/util.go @@ -190,7 +190,7 @@ func getGcid(r io.Reader, size int64) (string, error) { readSize := calcBlockSize(size) for { hash2.Reset() - if n, err := io.CopyN(hash2, r, readSize); err != nil && n == 0 { + if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { if err != io.EOF { return "", err } diff --git a/internal/net/request.go b/internal/net/request.go index 71f45aa7afc..088ff66ab4f 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/alist-org/alist/v3/pkg/utils" "io" "math" "net/http" @@ -271,7 +272,7 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int } } - n, err := io.Copy(ch.buf, resp.Body) + n, err := utils.CopyWithBuffer(ch.buf, resp.Body) if err != nil { return n, &errReadingBody{err: err} diff --git a/internal/net/serve.go b/internal/net/serve.go index a0566780759..adee75ae1d6 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -162,7 +162,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time pw.CloseWithError(err) return } - if _, err := io.CopyN(part, reader, ra.Length); err != nil { + if _, err := utils.CopyWithBufferN(part, reader, ra.Length); err != nil { pw.CloseWithError(err) return } @@ -182,7 +182,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time w.WriteHeader(code) if r.Method != "HEAD" { - written, err := io.CopyN(w, sendContent, sendSize) + written, err := utils.CopyWithBufferN(w, sendContent, sendSize) if err != nil { log.Warnf("ServeHttp error. err: %s ", err) if written != sendSize { diff --git a/internal/net/util.go b/internal/net/util.go index 4347e2c404d..44201859487 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -2,6 +2,7 @@ package net import ( "fmt" + "github.com/alist-org/alist/v3/pkg/utils" "io" "math" "mime/multipart" @@ -330,7 +331,7 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea log.Warnf("offset is more than 100MB, if loading data from internet, high-latency and wasting of bandwidth is expected") } - if _, err := io.Copy(io.Discard, io.LimitReader(readCloser, offset)); err != nil { + if _, err := utils.CopyWithBuffer(io.Discard, io.LimitReader(readCloser, offset)); err != nil { return nil, err } diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 4b882c519e0..40482f45a36 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -104,7 +104,7 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if httpRange.Start == 0 && httpRange.Length <= InMemoryBufMaxSizeBytes && f.peekBuff == nil { bufSize := utils.Min(httpRange.Length, f.GetSize()) newBuf := bytes.NewBuffer(make([]byte, 0, bufSize)) - n, err := io.CopyN(newBuf, f.Reader, bufSize) + n, err := utils.CopyWithBufferN(newBuf, f.Reader, bufSize) if err != nil { return nil, err } diff --git a/pkg/gowebdav/client.go b/pkg/gowebdav/client.go index 2fca0b7f43d..cef501b9a15 100644 --- a/pkg/gowebdav/client.go +++ b/pkg/gowebdav/client.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/xml" "fmt" + "github.com/alist-org/alist/v3/pkg/utils" "io" "net/http" "net/url" @@ -419,7 +420,7 @@ func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadClos // stream in rs.Body if rs.StatusCode == 200 { // discard first 'offset' bytes. - if _, err := io.Copy(io.Discard, io.LimitReader(rs.Body, offset)); err != nil { + if _, err := utils.CopyWithBuffer(io.Discard, io.LimitReader(rs.Body, offset)); err != nil { return nil, newPathErrorErr("ReadStreamRange", path, err) } diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 7ae07158998..54247636dcb 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -32,7 +32,7 @@ func CopyFile(src, dst string) error { } defer dstfd.Close() - if _, err = io.Copy(dstfd, srcfd); err != nil { + if _, err = CopyWithBuffer(dstfd, srcfd); err != nil { return err } if srcinfo, err = os.Stat(src); err != nil { @@ -121,7 +121,7 @@ func CreateTempFile(r io.Reader, size int64) (*os.File, error) { if err != nil { return nil, err } - readBytes, err := io.Copy(f, r) + readBytes, err := CopyWithBuffer(f, r) if err != nil { _ = os.Remove(f.Name()) return nil, errs.NewErr(err, "CreateTempFile failed") diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index 8f8aaa26781..fa06bcc24c2 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -96,7 +96,7 @@ func HashData(hashType *HashType, data []byte, params ...any) string { // HashReader get hash of one hashType from a reader func HashReader(hashType *HashType, reader io.Reader, params ...any) (string, error) { h := hashType.NewFunc(params...) - _, err := io.Copy(h, reader) + _, err := CopyWithBuffer(h, reader) if err != nil { return "", errs.NewErr(err, "HashReader error") } diff --git a/pkg/utils/hash_test.go b/pkg/utils/hash_test.go index 55713c1afb3..0f5a2a3b14e 100644 --- a/pkg/utils/hash_test.go +++ b/pkg/utils/hash_test.go @@ -4,7 +4,6 @@ import ( "bytes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "io" "testing" ) @@ -36,7 +35,7 @@ var hashTestSet = []hashTest{ func TestMultiHasher(t *testing.T) { for _, test := range hashTestSet { mh := NewMultiHasher([]*HashType{MD5, SHA1, SHA256}) - n, err := io.Copy(mh, bytes.NewBuffer(test.input)) + n, err := CopyWithBuffer(mh, bytes.NewBuffer(test.input)) require.NoError(t, err) assert.Len(t, test.input, int(n)) hashInfo := mh.GetHashInfo() diff --git a/pkg/utils/io.go b/pkg/utils/io.go index 6852e28a83d..7be989c3fd7 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "sync" "time" "golang.org/x/exp/constraints" @@ -29,7 +30,7 @@ func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, p // possible in the call process. var finish int64 = 0 s := size / 100 - _, err := io.Copy(out, readerFunc(func(p []byte) (int, error) { + _, err := CopyWithBuffer(out, readerFunc(func(p []byte) (int, error) { // golang non-blocking channel: https://gobyexample.com/non-blocking-channel-operations select { // if context has been canceled @@ -204,3 +205,31 @@ func Max[T constraints.Ordered](a, b T) T { } return a } + +var IoBuffPool = &sync.Pool{ + New: func() interface{} { + return make([]byte, 32*1024*2) // Two times of size in io package + }, +} + +func CopyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) { + buff := IoBuffPool.Get().([]byte) + defer IoBuffPool.Put(buff) + written, err = io.CopyBuffer(dst, src, buff) + if err != nil { + return + } + return written, nil +} + +func CopyWithBufferN(dst io.Writer, src io.Reader, n int64) (written int64, err error) { + written, err = CopyWithBuffer(dst, io.LimitReader(src, n)) + if written == n { + return n, nil + } + if written < n && err == nil { + // src stopped early; must have been EOF. + err = io.EOF + } + return +} From 0e246a7b0cc1acaea18587d06d5b8a7ae3fce581 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 30 Apr 2024 14:22:26 +0800 Subject: [PATCH 173/659] chore: replace link of vidhub [skip ci] --- README.md | 2 +- README_cn.md | 2 +- README_ja.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5c557956ffb..702638e11c7 100644 --- a/README.md +++ b/README.md @@ -115,7 +115,7 @@ https://alist.nn.ci/guide/sponsor.html ### Special sponsors -- [VidHub](https://okaapps.com/product/1659622164?ref=alist) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. +- [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) - [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 diff --git a/README_cn.md b/README_cn.md index 90995f8e67b..f268d383c8b 100644 --- a/README_cn.md +++ b/README_cn.md @@ -113,7 +113,7 @@ AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我 ### 特别赞助 -- [VidHub](https://zh.okaapps.com/product/1659622164?ref=alist) - 苹果生态下优雅的网盘视频播放器，iPhone，iPad，Mac，Apple TV全平台支持。 +- [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - 苹果生态下优雅的网盘视频播放器，iPhone，iPad，Mac，Apple TV全平台支持。 - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (国内API服务器赞助) - [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 diff --git a/README_ja.md b/README_ja.md index f79dcf204a6..7cef979f75e 100644 --- a/README_ja.md +++ b/README_ja.md @@ -115,7 +115,7 @@ https://alist.nn.ci/guide/sponsor.html ### スペシャルスポンサー -- [VidHub](https://okaapps.com/product/1659622164?ref=alist) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. +- [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) - [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 From eecea3febd890294904d63f27fdbd977b77a8fa2 Mon Sep 17 00:00:00 2001 From: potoo <34411681+potoo0@users.noreply.github.com> Date: Thu, 2 May 2024 22:27:31 +0800 Subject: [PATCH 174/659] fix(onedrive): fix Ctime/Mtime (#6397) --- drivers/onedrive/driver.go | 1 + drivers/onedrive/types.go | 31 +++++++++++++++++++------ drivers/onedrive/util.go | 47 +++++++++++++++++++++++++++++++++++--- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index 319fd906b7d..adbe0342e1c 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -118,6 +118,7 @@ func (d *Onedrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName str "folder": base.Json{}, "@microsoft.graph.conflictBehavior": "rename", } + // todo 修复文件夹 ctime/mtime, onedrive 可在 data 里设置 fileSystemInfo 字段, 但是此接口未提供 ctime/mtime _, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) diff --git a/drivers/onedrive/types.go b/drivers/onedrive/types.go index 69264abcf03..aedd5a06802 100644 --- a/drivers/onedrive/types.go +++ b/drivers/onedrive/types.go @@ -24,12 +24,12 @@ type RespErr struct { } type File struct { - Id string `json:"id"` - Name string `json:"name"` - Size int64 `json:"size"` - LastModifiedDateTime time.Time `json:"lastModifiedDateTime"` - Url string `json:"@microsoft.graph.downloadUrl"` - File *struct { + Id string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` + Url string `json:"@microsoft.graph.downloadUrl"` + File *struct { MimeType string `json:"mimeType"` } `json:"file"` Thumbnails []struct { @@ -58,7 +58,7 @@ func fileToObj(f File, parentID string) *Object { ID: f.Id, Name: f.Name, Size: f.Size, - Modified: f.LastModifiedDateTime, + Modified: f.FileSystemInfo.LastModifiedDateTime, IsFolder: f.File == nil, }, Thumbnail: model.Thumbnail{Thumbnail: thumb}, @@ -72,3 +72,20 @@ type Files struct { Value []File `json:"value"` NextLink string `json:"@odata.nextLink"` } + +// Metadata represents a request to update Metadata. +// It includes only the writeable properties. +// omitempty is intentionally included for all, per https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update?view=odsp-graph-online#request-body +type Metadata struct { + Description string `json:"description,omitempty"` // Provides a user-visible description of the item. Read-write. Only on OneDrive Personal. Undocumented limit of 1024 characters. + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo,omitempty"` // File system information on client. Read-write. +} + +// FileSystemInfoFacet contains properties that are reported by the +// device's local file system for the local version of an item. This +// facet can be used to specify the last modified date or created date +// of the item as it was on the local device. +type FileSystemInfoFacet struct { + CreatedDateTime time.Time `json:"createdDateTime,omitempty"` // The UTC date and time the file was created on a client. + LastModifiedDateTime time.Time `json:"lastModifiedDateTime,omitempty"` // The UTC date and time the file was last modified on a client. +} diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index a0c6fa8fcbf..9ee2dae9cae 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -127,7 +127,7 @@ func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, func (d *Onedrive) getFiles(path string) ([]File, error) { var res []File - nextLink := d.GetMetaUrl(false, path) + "/children?$top=5000&$expand=thumbnails($select=medium)&$select=id,name,size,lastModifiedDateTime,content.downloadUrl,file,parentReference" + nextLink := d.GetMetaUrl(false, path) + "/children?$top=5000&$expand=thumbnails($select=medium)&$select=id,name,size,fileSystemInfo,content.downloadUrl,file,parentReference" for nextLink != "" { var files Files _, err := d.Request(nextLink, http.MethodGet, nil, &files) @@ -148,7 +148,10 @@ func (d *Onedrive) GetFile(path string) (*File, error) { } func (d *Onedrive) upSmall(ctx context.Context, dstDir model.Obj, stream model.FileStreamer) error { - url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/content" + filepath := stdpath.Join(dstDir.GetPath(), stream.GetName()) + // 1. upload new file + // ApiDoc: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online + url := d.GetMetaUrl(false, filepath) + "/content" data, err := io.ReadAll(stream) if err != nil { return err @@ -156,12 +159,50 @@ func (d *Onedrive) upSmall(ctx context.Context, dstDir model.Obj, stream model.F _, err = d.Request(url, http.MethodPut, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, nil) + if err != nil { + return fmt.Errorf("onedrive: Failed to upload new file(path=%v): %w", filepath, err) + } + + // 2. update metadata + err = d.updateMetadata(ctx, stream, filepath) + if err != nil { + return fmt.Errorf("onedrive: Failed to update file(path=%v) metadata: %w", filepath, err) + } + return nil +} + +func (d *Onedrive) updateMetadata(ctx context.Context, stream model.FileStreamer, filepath string) error { + url := d.GetMetaUrl(false, filepath) + metadata := toAPIMetadata(stream) + // ApiDoc: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update?view=odsp-graph-online + _, err := d.Request(url, http.MethodPatch, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) return err } +func toAPIMetadata(stream model.FileStreamer) Metadata { + metadata := Metadata{ + FileSystemInfo: &FileSystemInfoFacet{}, + } + if !stream.ModTime().IsZero() { + metadata.FileSystemInfo.LastModifiedDateTime = stream.ModTime() + } + if !stream.CreateTime().IsZero() { + metadata.FileSystemInfo.CreatedDateTime = stream.CreateTime() + } + if stream.CreateTime().IsZero() && !stream.ModTime().IsZero() { + metadata.FileSystemInfo.CreatedDateTime = stream.CreateTime() + } + return metadata +} + func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/createUploadSession" - res, err := d.Request(url, http.MethodPost, nil, nil) + metadata := map[string]interface{}{"item": toAPIMetadata(stream)} + res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) if err != nil { return err } From b704bba444147563d1f9c8e26458d8ec0fa405a3 Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Thu, 2 May 2024 22:27:55 +0800 Subject: [PATCH 175/659] fix(115): disable NoOverwriteUpload (#6409 close #6251) closed #6251 --- drivers/115/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/115/meta.go b/drivers/115/meta.go index 2afc57a78c9..5791f1bd140 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -19,7 +19,7 @@ var config = driver.Config{ DefaultRoot: "0", //OnlyProxy: true, //OnlyLocal: true, - NoOverwriteUpload: true, + //NoOverwriteUpload: true, } func init() { From 7bf5014417508073479b0cc3b1916fb9c3fa687b Mon Sep 17 00:00:00 2001 From: Mmx Date: Thu, 2 May 2024 22:28:13 +0800 Subject: [PATCH 176/659] ci: cache musl library in docker build workflow (#6392) * ci: add musl libs into action cache * build: update Dockerfile.ci --- .github/workflows/build_docker.yml | 13 ++++++++++++- .github/workflows/release_docker.yml | 13 ++++++++++++- Dockerfile.ci | 2 +- build.sh | 27 +++++++++++++++++++-------- 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 9c740ba1fa4..731b0159011 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -32,10 +32,21 @@ jobs: flavor: | suffix=-ffmpeg,onlatest=true - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version: 'stable' + - name: Cache Musl + id: cache-musl + uses: actions/cache@v4 + with: + path: build/musl-libs + key: docker-musl-libs + + - name: Download Musl Library + if: steps.cache-musl.outputs.cache-hit != 'true' + run: bash build.sh prepare docker-multiplatform + - name: Build go binary run: bash build.sh dev docker-multiplatform diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index d172c27f625..8bff6a3d9b3 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -13,10 +13,21 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version: 'stable' + - name: Cache Musl + id: cache-musl + uses: actions/cache@v4 + with: + path: build/musl-libs + key: docker-musl-libs + + - name: Download Musl Library + if: steps.cache-musl.outputs.cache-hit != 'true' + run: bash build.sh prepare docker-multiplatform + - name: Build go binary run: bash build.sh release docker-multiplatform diff --git a/Dockerfile.ci b/Dockerfile.ci index dc18d259f58..c25e2471b16 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -3,7 +3,7 @@ ARG TARGETPLATFORM LABEL MAINTAINER="i@nn.ci" VOLUME /opt/alist/data/ WORKDIR /opt/alist/ -COPY /${TARGETPLATFORM}/alist ./ +COPY /build/${TARGETPLATFORM}/alist ./ COPY entrypoint.sh /entrypoint.sh RUN apk update && \ apk upgrade --no-cache && \ diff --git a/build.sh b/build.sh index f036d714efa..368d2d2cfb0 100644 --- a/build.sh +++ b/build.sh @@ -96,17 +96,24 @@ BuildDocker() { go build -o ./bin/alist -ldflags="$ldflags" -tags=jsoniter . } -BuildDockerMultiplatform() { - PrepareBuildDocker - +PrepareBuildDockerMusl() { + mkdir -p build/musl-libs BASE="https://musl.cc/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" - curl -L -o "${i}.tgz" "${url}" - sudo tar xf "${i}.tgz" --strip-components 1 -C /usr/local - rm -f "${i}.tgz" + lib_tgz="build/${i}.tgz" + curl -L -o "${lib_tgz}" "${url}" + tar xf "${lib_tgz}" --strip-components 1 -C build/musl-libs + rm -f "${lib_tgz}" done +} + +BuildDockerMultiplatform() { + PrepareBuildDocker + + # run PrepareBuildDockerMusl before build + export PATH=$PATH:$PWD/build/musl-libs/bin docker_lflags="--extldflags '-static -fpic' $ldflags" export CGO_ENABLED=1 @@ -122,7 +129,7 @@ BuildDockerMultiplatform() { export GOARCH=$arch export CC=${cgo_cc} echo "building for $os_arch" - go build -o ./$os/$arch/alist -ldflags="$docker_lflags" -tags=jsoniter . + go build -o build/$os/$arch/alist -ldflags="$docker_lflags" -tags=jsoniter . done DOCKER_ARM_ARCHES=(linux-arm/v6 linux-arm/v7) @@ -136,7 +143,7 @@ BuildDockerMultiplatform() { export GOARM=${GO_ARM[$i]} export CC=${cgo_cc} echo "building for $docker_arch" - go build -o ./${docker_arch%%-*}/${docker_arch##*-}/alist -ldflags="$docker_lflags" -tags=jsoniter . + go build -o build/${docker_arch%%-*}/${docker_arch##*-}/alist -ldflags="$docker_lflags" -tags=jsoniter . done } @@ -289,6 +296,10 @@ elif [ "$1" = "release" ]; then BuildRelease MakeRelease "md5.txt" fi +elif [ "$1" = "prepare" ]; then + if [ "$2" = "docker-multiplatform" ]; then + PrepareBuildDockerMusl + fi else echo -e "Parameter error" fi From 4cbbda8832449d1ccd9b81b5dbec574912934408 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 2 May 2024 22:30:00 +0800 Subject: [PATCH 177/659] fix(baidu): custom upload part size (close #5757) --- drivers/baidu_netdisk/meta.go | 19 ++++++++++--------- drivers/baidu_netdisk/util.go | 3 +++ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index b257986b9c1..7e01767b2ed 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -8,15 +8,16 @@ import ( type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` driver.RootPath - OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` - ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` - ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` - CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` - AccessToken string - UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` - UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` + ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` + ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` + CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` + AccessToken string + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" default:"0" help:"0 for auto"` } var config = driver.Config{ diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 6c51156c22f..ac1f06e807e 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -249,6 +249,9 @@ const ( ) func (d *BaiduNetdisk) getSliceSize() int64 { + if d.CustomUploadPartSize != 0 { + return d.CustomUploadPartSize + } switch d.vipType { case 1: return VipSliceSize From 5f2853242341ef510a0a5897b2c1788ef97303b0 Mon Sep 17 00:00:00 2001 From: Moraxyc Date: Thu, 9 May 2024 14:22:19 +0800 Subject: [PATCH 178/659] fix(test): ensure `setupStorages` is executed once (#6422) In TestGetStorageVirtualFilesByPath() and TestGetBalancedStorage(), setupStorages() was being called twice, leading to a "UNIQUE constraint failed" error. --- internal/op/storage_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/op/storage_test.go b/internal/op/storage_test.go index f3d0b4e64b7..fddf2bb11e3 100644 --- a/internal/op/storage_test.go +++ b/internal/op/storage_test.go @@ -60,7 +60,6 @@ func TestGetStorageVirtualFilesByPath(t *testing.T) { } func TestGetBalancedStorage(t *testing.T) { - setupStorages(t) set := mapset.NewSet[string]() for i := 0; i < 5; i++ { storage := op.GetBalancedStorage("/a/d/e1") From 2313213f599795a0b4a8c5a0930fb7653dfaaaa6 Mon Sep 17 00:00:00 2001 From: meozky <19809896+meozky@users.noreply.github.com> Date: Thu, 9 May 2024 14:23:12 +0800 Subject: [PATCH 179/659] fix(189pc): `FamilyID` range overflow (#6427 close #6426) --- drivers/189pc/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index a1b3810fd23..2e9ed4c203d 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -143,7 +143,7 @@ type FamilyInfoListResp struct { type FamilyInfoResp struct { Count int `json:"count"` CreateTime string `json:"createTime"` - FamilyID int `json:"familyId"` + FamilyID int64 `json:"familyId"` RemarkName string `json:"remarkName"` Type int `json:"type"` UseFlag int `json:"useFlag"` From 7e7b9b9b48ac6ff6369929dc6c954a0b8ae34a4a Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Thu, 9 May 2024 14:28:59 +0800 Subject: [PATCH 180/659] feat(s3): server support generated url request (#6431) --- go.mod | 2 +- go.sum | 4 ++-- server/s3/backend.go | 44 +++++++++++++++++++------------------------- server/s3/list.go | 2 +- server/s3/logger.go | 2 +- server/s3/pager.go | 2 +- server/s3/server.go | 2 +- server/s3/utils.go | 2 +- 8 files changed, 27 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 85aa4ec3406..11a858a2ac9 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/alist-org/alist/v3 go 1.21 require ( - github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 github.com/SheltonZhu/115driver v1.0.22 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/Xhofe/wopan-sdk-go v0.1.2 + github.com/alist-org/gofakes3 v0.0.4 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.50.24 diff --git a/go.sum b/go.sum index 3ea10e49e0b..05a9b3385a1 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,6 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700 h1:r3fp2/Ro+0RtpjNY0/wsbN7vRmCW//dXTOZDQTct25Q= -github.com/Mikubill/gofakes3 v0.0.3-0.20230622102024-284c0f988700/go.mod h1:OSXqXEGUe9CmPiwLMMnVrbXonMf4BeLBkBdLufxxiyY= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U= @@ -25,6 +23,8 @@ github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0E github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= +github.com/alist-org/gofakes3 v0.0.4 h1:/ID4+1llsiB8EweLcC65rVmgBZKL95e3P7Wa+aJGUiE= +github.com/alist-org/gofakes3 v0.0.4/go.mod h1:bLPZXt45XYMgaoGGLe5t0d1p13oZTQTptTEDLrku070= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= diff --git a/server/s3/backend.go b/server/s3/backend.go index 75c6b28b1ef..c4c6c5a66c7 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -12,7 +12,6 @@ import ( "sync" "time" - "github.com/Mikubill/gofakes3" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" @@ -20,12 +19,13 @@ import ( "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/gofakes3" "github.com/ncw/swift/v2" ) var ( emptyPrefix = &gofakes3.Prefix{} - timeFormat = "Mon, 2 Jan 2006 15:04:05.999999999 GMT" + timeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" ) // s3Backend implements the gofacess3.Backend interface to make an S3 @@ -42,13 +42,12 @@ func newBackend() gofakes3.Backend { } // ListBuckets always returns the default bucket. -func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) { +func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, error) { buckets, err := getAndParseBuckets() if err != nil { return nil, err } var response []gofakes3.BucketInfo - ctx := context.Background() for _, b := range buckets { node, _ := fs.Get(ctx, b.Path, &fs.GetArgs{}) response = append(response, gofakes3.BucketInfo{ @@ -61,7 +60,7 @@ func (b *s3Backend) ListBuckets() ([]gofakes3.BucketInfo, error) { } // ListBucket lists the objects in the given bucket. -func (b *s3Backend) ListBucket(bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { +func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { bucket, err := getBucketByName(bucketName) if err != nil { return nil, err @@ -97,8 +96,7 @@ func (b *s3Backend) ListBucket(bucketName string, prefix *gofakes3.Prefix, page // HeadObject returns the fileinfo for the given object name. // // Note that the metadata is not supported yet. -func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object, error) { - ctx := context.Background() +func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName string) (*gofakes3.Object, error) { bucket, err := getBucketByName(bucketName) if err != nil { return nil, err @@ -141,8 +139,7 @@ func (b *s3Backend) HeadObject(bucketName, objectName string) (*gofakes3.Object, } // GetObject fetchs the object from the filesystem. -func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) { - ctx := context.Background() +func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) { bucket, err := getBucketByName(bucketName) if err != nil { return nil, err @@ -251,18 +248,17 @@ func (b *s3Backend) GetObject(bucketName, objectName string, rangeRequest *gofak } // TouchObject creates or updates meta on specified object. -func (b *s3Backend) TouchObject(fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) { +func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) { //TODO: implement return result, gofakes3.ErrNotImplemented } // PutObject creates or overwrites the object with the given name. func (b *s3Backend) PutObject( - bucketName, objectName string, + ctx context.Context, bucketName, objectName string, meta map[string]string, input io.Reader, size int64, ) (result gofakes3.PutObjectResult, err error) { - ctx := context.Background() bucket, err := getBucketByName(bucketName) if err != nil { return result, err @@ -316,9 +312,9 @@ func (b *s3Backend) PutObject( } // DeleteMulti deletes multiple objects in a single request. -func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) { +func (b *s3Backend) DeleteMulti(ctx context.Context, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) { for _, object := range objects { - if err := b.deleteObject(bucketName, object); err != nil { + if err := b.deleteObject(ctx, bucketName, object); err != nil { utils.Log.Errorf("serve s3", "delete object failed: %v", err) result.Error = append(result.Error, gofakes3.ErrorResult{ Code: gofakes3.ErrInternal, @@ -336,13 +332,12 @@ func (b *s3Backend) DeleteMulti(bucketName string, objects ...string) (result go } // DeleteObject deletes the object with the given name. -func (b *s3Backend) DeleteObject(bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { - return result, b.deleteObject(bucketName, objectName) +func (b *s3Backend) DeleteObject(ctx context.Context, bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { + return result, b.deleteObject(ctx, bucketName, objectName) } // deleteObject deletes the object from the filesystem. -func (b *s3Backend) deleteObject(bucketName, objectName string) error { - ctx := context.Background() +func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName string) error { bucket, err := getBucketByName(bucketName) if err != nil { return err @@ -362,17 +357,17 @@ func (b *s3Backend) deleteObject(bucketName, objectName string) error { } // CreateBucket creates a new bucket. -func (b *s3Backend) CreateBucket(name string) error { +func (b *s3Backend) CreateBucket(ctx context.Context, name string) error { return gofakes3.ErrNotImplemented } // DeleteBucket deletes the bucket with the given name. -func (b *s3Backend) DeleteBucket(name string) error { +func (b *s3Backend) DeleteBucket(ctx context.Context, name string) error { return gofakes3.ErrNotImplemented } // BucketExists checks if the bucket exists. -func (b *s3Backend) BucketExists(name string) (exists bool, err error) { +func (b *s3Backend) BucketExists(ctx context.Context, name string) (exists bool, err error) { buckets, err := getAndParseBuckets() if err != nil { return false, err @@ -386,13 +381,12 @@ func (b *s3Backend) BucketExists(name string) (exists bool, err error) { } // CopyObject copy specified object from srcKey to dstKey. -func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { +func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { if srcBucket == dstBucket && srcKey == dstKey { //TODO: update meta return result, nil } - ctx := context.Background() srcB, err := getBucketByName(srcBucket) if err != nil { return result, err @@ -403,7 +397,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta fmeta, _ := op.GetNearestMeta(srcFp) srcNode, err := fs.Get(context.WithValue(ctx, "meta", fmeta), srcFp, &fs.GetArgs{}) - c, err := b.GetObject(srcBucket, srcKey, nil) + c, err := b.GetObject(ctx, srcBucket, srcKey, nil) if err != nil { return } @@ -420,7 +414,7 @@ func (b *s3Backend) CopyObject(srcBucket, srcKey, dstBucket, dstKey string, meta meta["mtime"] = swift.TimeToFloatString(srcNode.ModTime()) } - _, err = b.PutObject(dstBucket, dstKey, meta, c.Contents, c.Size) + _, err = b.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size) if err != nil { return } diff --git a/server/s3/list.go b/server/s3/list.go index bce870ca9ed..40a9e8ab356 100644 --- a/server/s3/list.go +++ b/server/s3/list.go @@ -6,7 +6,7 @@ import ( "path" "strings" - "github.com/Mikubill/gofakes3" + "github.com/alist-org/gofakes3" ) func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error { diff --git a/server/s3/logger.go b/server/s3/logger.go index 7566fa8a116..798734c341d 100644 --- a/server/s3/logger.go +++ b/server/s3/logger.go @@ -5,8 +5,8 @@ package s3 import ( "fmt" - "github.com/Mikubill/gofakes3" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/gofakes3" ) // logger output formatted message diff --git a/server/s3/pager.go b/server/s3/pager.go index 3268b0ca234..27dd5c9a5cd 100644 --- a/server/s3/pager.go +++ b/server/s3/pager.go @@ -5,7 +5,7 @@ package s3 import ( "sort" - "github.com/Mikubill/gofakes3" + "github.com/alist-org/gofakes3" ) // pager splits the object list into smulitply pages. diff --git a/server/s3/server.go b/server/s3/server.go index 19df735fb5d..2f7d15c0804 100644 --- a/server/s3/server.go +++ b/server/s3/server.go @@ -7,7 +7,7 @@ import ( "math/rand" "net/http" - "github.com/Mikubill/gofakes3" + "github.com/alist-org/gofakes3" ) // Make a new S3 Server to serve the remote diff --git a/server/s3/utils.go b/server/s3/utils.go index 98c271f76a3..6636835a62b 100644 --- a/server/s3/utils.go +++ b/server/s3/utils.go @@ -7,13 +7,13 @@ import ( "encoding/json" "strings" - "github.com/Mikubill/gofakes3" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/gofakes3" ) type Bucket struct { From f261ef50ccad6e7ba1a8015503cccae49fb1c253 Mon Sep 17 00:00:00 2001 From: liuycy Date: Thu, 9 May 2024 14:29:35 +0800 Subject: [PATCH 181/659] feat: add supports for netease music driver (#6423 close #5364) --- drivers/all.go | 1 + drivers/netease_music/crypto.go | 135 ++++++++++++++++++ drivers/netease_music/driver.go | 110 ++++++++++++++ drivers/netease_music/meta.go | 32 +++++ drivers/netease_music/types.go | 116 +++++++++++++++ drivers/netease_music/upload.go | 208 +++++++++++++++++++++++++++ drivers/netease_music/util.go | 246 ++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 + 9 files changed, 851 insertions(+) create mode 100644 drivers/netease_music/crypto.go create mode 100644 drivers/netease_music/driver.go create mode 100644 drivers/netease_music/meta.go create mode 100644 drivers/netease_music/types.go create mode 100644 drivers/netease_music/upload.go create mode 100644 drivers/netease_music/util.go diff --git a/drivers/all.go b/drivers/all.go index 08d8f1cbd42..d1d5f84a989 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -32,6 +32,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/mediatrack" _ "github.com/alist-org/alist/v3/drivers/mega" _ "github.com/alist-org/alist/v3/drivers/mopan" + _ "github.com/alist-org/alist/v3/drivers/netease_music" _ "github.com/alist-org/alist/v3/drivers/onedrive" _ "github.com/alist-org/alist/v3/drivers/onedrive_app" _ "github.com/alist-org/alist/v3/drivers/pikpak" diff --git a/drivers/netease_music/crypto.go b/drivers/netease_music/crypto.go new file mode 100644 index 00000000000..76ff65486ac --- /dev/null +++ b/drivers/netease_music/crypto.go @@ -0,0 +1,135 @@ +package netease_music + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "math/big" + "strings" + + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" +) + +var ( + linuxapiKey = []byte("rFgB&h#%2?^eDg:Q") + eapiKey = []byte("e82ckenh8dichen8") + iv = []byte("0102030405060708") + presetKey = []byte("0CoJUm6Qyw8W8jud") + publicKey = []byte("-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgtQn2JZ34ZC28NWYpAUd98iZ37BUrX/aKzmFbt7clFSs6sXqHauqKWqdtLkF2KexO40H1YTX8z2lSgBBOAxLsvaklV8k4cBFK9snQXE9/DDaFt6Rr7iVZMldczhC0JNgTz+SHXT6CBHuX3e9SdB1Ua44oncaTWz7OBGLbCiK45wIDAQAB\n-----END PUBLIC KEY-----") + stdChars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") +) + +func aesKeyPending(key []byte) []byte { + k := len(key) + count := 0 + switch true { + case k <= 16: + count = 16 - k + case k <= 24: + count = 24 - k + case k <= 32: + count = 32 - k + default: + return key[:32] + } + if count == 0 { + return key + } + + return append(key, bytes.Repeat([]byte{0}, count)...) +} + +func pkcs7Padding(src []byte, blockSize int) []byte { + padding := blockSize - len(src)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(src, padtext...) +} + +func aesCBCEncrypt(src, key, iv []byte) []byte { + block, _ := aes.NewCipher(aesKeyPending(key)) + src = pkcs7Padding(src, block.BlockSize()) + dst := make([]byte, len(src)) + + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(dst, src) + + return dst +} + +func aesECBEncrypt(src, key []byte) []byte { + block, _ := aes.NewCipher(aesKeyPending(key)) + + src = pkcs7Padding(src, block.BlockSize()) + dst := make([]byte, len(src)) + + ecbCryptBlocks(block, dst, src) + + return dst +} + +func ecbCryptBlocks(block cipher.Block, dst, src []byte) { + bs := block.BlockSize() + + for len(src) > 0 { + block.Encrypt(dst, src[:bs]) + src = src[bs:] + dst = dst[bs:] + } +} + +func rsaEncrypt(buffer, key []byte) []byte { + buffers := make([]byte, 128-16, 128) + buffers = append(buffers, buffer...) + block, _ := pem.Decode(key) + pubInterface, _ := x509.ParsePKIXPublicKey(block.Bytes) + pub := pubInterface.(*rsa.PublicKey) + c := new(big.Int).SetBytes([]byte(buffers)) + return c.Exp(c, big.NewInt(int64(pub.E)), pub.N).Bytes() +} + +func getSecretKey() ([]byte, []byte) { + key := make([]byte, 16) + reversed := make([]byte, 16) + for i := 0; i < 16; i++ { + result := stdChars[random.RangeInt64(0, 62)] + key[i] = result + reversed[15-i] = result + } + return key, reversed +} + +func weapi(data map[string]string) map[string]string { + text, _ := utils.Json.Marshal(data) + secretKey, reversedKey := getSecretKey() + params := []byte(base64.StdEncoding.EncodeToString(aesCBCEncrypt(text, presetKey, iv))) + return map[string]string{ + "params": base64.StdEncoding.EncodeToString(aesCBCEncrypt(params, reversedKey, iv)), + "encSecKey": hex.EncodeToString(rsaEncrypt(secretKey, publicKey)), + } +} + +func eapi(url string, data map[string]interface{}) map[string]string { + text, _ := utils.Json.Marshal(data) + msg := "nobody" + url + "use" + string(text) + "md5forencrypt" + h := md5.New() + h.Write([]byte(msg)) + digest := hex.EncodeToString(h.Sum(nil)) + params := []byte(url + "-36cd479b6b5-" + string(text) + "-36cd479b6b5-" + digest) + return map[string]string{ + "params": hex.EncodeToString(aesECBEncrypt(params, eapiKey)), + } +} + +func linuxapi(data map[string]interface{}) map[string]string { + text, _ := utils.Json.Marshal(data) + return map[string]string{ + "eparams": strings.ToUpper(hex.EncodeToString(aesECBEncrypt(text, linuxapiKey))), + } +} diff --git a/drivers/netease_music/driver.go b/drivers/netease_music/driver.go new file mode 100644 index 00000000000..c0d103de0d9 --- /dev/null +++ b/drivers/netease_music/driver.go @@ -0,0 +1,110 @@ +package netease_music + +import ( + "context" + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + _ "golang.org/x/image/webp" +) + +type NeteaseMusic struct { + model.Storage + Addition + + csrfToken string + musicU string + fileMapByName map[string]model.Obj +} + +func (d *NeteaseMusic) Config() driver.Config { + return config +} + +func (d *NeteaseMusic) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *NeteaseMusic) Init(ctx context.Context) error { + d.csrfToken = d.Addition.getCookie("__csrf") + d.musicU = d.Addition.getCookie("MUSIC_U") + + if d.csrfToken == "" || d.musicU == "" { + return errs.EmptyToken + } + + return nil +} + +func (d *NeteaseMusic) Drop(ctx context.Context) error { + return nil +} + +func (d *NeteaseMusic) Get(ctx context.Context, path string) (model.Obj, error) { + if path == "/" { + return &model.Object{ + IsFolder: true, + Path: path, + }, nil + } + + fragments := strings.Split(path, "/") + if len(fragments) > 1 { + fileName := fragments[1] + if strings.HasSuffix(fileName, ".lrc") { + lrc := d.fileMapByName[fileName] + return d.getLyricObj(lrc) + } + if song, ok := d.fileMapByName[fileName]; ok { + return song, nil + } else { + return nil, errs.ObjectNotFound + } + } + + return nil, errs.ObjectNotFound +} + +func (d *NeteaseMusic) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.getSongObjs(args) +} + +func (d *NeteaseMusic) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if lrc, ok := file.(*LyricObj); ok { + if args.Type == "parsed" { + return lrc.getLyricLink(), nil + } else { + return lrc.getProxyLink(args), nil + } + } + + return d.getSongLink(file) +} + +func (d *NeteaseMusic) Remove(ctx context.Context, obj model.Obj) error { + return d.removeSongObj(obj) +} + +func (d *NeteaseMusic) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + return d.putSongStream(stream) +} + +func (d *NeteaseMusic) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *NeteaseMusic) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *NeteaseMusic) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return errs.NotSupport +} + +func (d *NeteaseMusic) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return errs.NotSupport +} + +var _ driver.Driver = (*NeteaseMusic)(nil) diff --git a/drivers/netease_music/meta.go b/drivers/netease_music/meta.go new file mode 100644 index 00000000000..8ddfd728178 --- /dev/null +++ b/drivers/netease_music/meta.go @@ -0,0 +1,32 @@ +package netease_music + +import ( + "regexp" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + Cookie string `json:"cookie" type:"text" required:"true" help:""` + SongLimit uint64 `json:"song_limit" default:"200" type:"number" help:"only get 200 songs by default"` +} + +func (ad *Addition) getCookie(name string) string { + re := regexp.MustCompile(name + "=([^(;|$)]+)") + matches := re.FindStringSubmatch(ad.Cookie) + if len(matches) < 2 { + return "" + } + return matches[1] +} + +var config = driver.Config{ + Name: "NeteaseMusic", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &NeteaseMusic{} + }) +} diff --git a/drivers/netease_music/types.go b/drivers/netease_music/types.go new file mode 100644 index 00000000000..edbd40eed59 --- /dev/null +++ b/drivers/netease_music/types.go @@ -0,0 +1,116 @@ +package netease_music + +import ( + "context" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/server/common" +) + +type HostsResp struct { + Upload []string `json:"upload"` +} + +type SongResp struct { + Data []struct { + Url string `json:"url"` + } `json:"data"` +} + +type ListResp struct { + Size string `json:"size"` + MaxSize string `json:"maxSize"` + Data []struct { + AddTime int64 `json:"addTime"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + SongId int64 `json:"songId"` + SimpleSong struct { + Al struct { + PicUrl string `json:"picUrl"` + } `json:"al"` + } `json:"simpleSong"` + } `json:"data"` +} + +type LyricObj struct { + model.Object + lyric string +} + +func (lrc *LyricObj) getProxyLink(args model.LinkArgs) *model.Link { + rawURL := common.GetApiUrl(args.HttpReq) + "/p" + lrc.Path + rawURL = utils.EncodePath(rawURL, true) + "?type=parsed&sign=" + sign.Sign(lrc.Path) + return &model.Link{URL: rawURL} +} + +func (lrc *LyricObj) getLyricLink() *model.Link { + reader := strings.NewReader(lrc.lyric) + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + if httpRange.Length < 0 { + return io.NopCloser(reader), nil + } + sr := io.NewSectionReader(reader, httpRange.Start, httpRange.Length) + return io.NopCloser(sr), nil + }, + Closers: utils.EmptyClosers(), + }, + } +} + +type ReqOption struct { + crypto string + stream model.FileStreamer + data map[string]string + headers map[string]string + cookies []*http.Cookie + url string +} + +type Characteristic map[string]string + +func (ch *Characteristic) fromDriver(d *NeteaseMusic) *Characteristic { + *ch = map[string]string{ + "osver": "", + "deviceId": "", + "mobilename": "", + "appver": "6.1.1", + "versioncode": "140", + "buildver": strconv.FormatInt(time.Now().Unix(), 10), + "resolution": "1920x1080", + "os": "android", + "channel": "", + "requestId": strconv.FormatInt(time.Now().Unix()*1000, 10) + strconv.Itoa(int(random.RangeInt64(0, 1000))), + "MUSIC_U": d.musicU, + } + return ch +} + +func (ch Characteristic) toCookies() []*http.Cookie { + cookies := make([]*http.Cookie, 0) + for k, v := range ch { + cookies = append(cookies, &http.Cookie{Name: k, Value: v}) + } + return cookies +} + +func (ch *Characteristic) merge(data map[string]string) map[string]interface{} { + body := map[string]interface{}{ + "header": ch, + } + for k, v := range data { + body[k] = v + } + return body +} diff --git a/drivers/netease_music/upload.go b/drivers/netease_music/upload.go new file mode 100644 index 00000000000..ece496b36da --- /dev/null +++ b/drivers/netease_music/upload.go @@ -0,0 +1,208 @@ +package netease_music + +import ( + "crypto/md5" + "encoding/hex" + "io" + "net/http" + "strconv" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/dhowden/tag" +) + +type token struct { + resourceId string + objectKey string + token string +} + +type songmeta struct { + needUpload bool + songId string + name string + artist string + album string +} + +type uploader struct { + driver *NeteaseMusic + file model.File + meta songmeta + md5 string + ext string + size string + filename string +} + +func (u *uploader) init(stream model.FileStreamer) error { + u.filename = stream.GetName() + u.size = strconv.FormatInt(stream.GetSize(), 10) + + u.ext = "mp3" + if strings.HasSuffix(stream.GetMimetype(), "flac") { + u.ext = "flac" + } + + h := md5.New() + io.Copy(h, stream) + u.md5 = hex.EncodeToString(h.Sum(nil)) + _, err := u.file.Seek(0, io.SeekStart) + if err != nil { + return err + } + + if m, err := tag.ReadFrom(u.file); err != nil { + u.meta = songmeta{} + } else { + u.meta = songmeta{ + name: m.Title(), + artist: m.Artist(), + album: m.Album(), + } + } + if u.meta.name == "" { + u.meta.name = u.filename + } + if u.meta.album == "" { + u.meta.album = "未知专辑" + } + if u.meta.artist == "" { + u.meta.artist = "未知艺术家" + } + _, err = u.file.Seek(0, io.SeekStart) + if err != nil { + return err + } + + return nil +} + +func (u *uploader) checkIfExisted() error { + body, err := u.driver.request("https://interface.music.163.com/api/cloud/upload/check", http.MethodPost, + ReqOption{ + crypto: "weapi", + data: map[string]string{ + "ext": "", + "songId": "0", + "version": "1", + "bitrate": "999000", + "length": u.size, + "md5": u.md5, + }, + cookies: []*http.Cookie{ + {Name: "os", Value: "pc"}, + {Name: "appver", Value: "2.9.7"}, + }, + }, + ) + if err != nil { + return err + } + + u.meta.songId = utils.Json.Get(body, "songId").ToString() + u.meta.needUpload = utils.Json.Get(body, "needUpload").ToBool() + + return nil +} + +func (u *uploader) allocToken(bucket ...string) (token, error) { + if len(bucket) == 0 { + bucket = []string{""} + } + + body, err := u.driver.request("https://music.163.com/weapi/nos/token/alloc", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "bucket": bucket[0], + "local": "false", + "type": "audio", + "nos_product": "3", + "filename": u.filename, + "md5": u.md5, + "ext": u.ext, + }, + }) + if err != nil { + return token{}, err + } + + return token{ + resourceId: utils.Json.Get(body, "result", "resourceId").ToString(), + objectKey: utils.Json.Get(body, "result", "objectKey").ToString(), + token: utils.Json.Get(body, "result", "token").ToString(), + }, nil +} + +func (u *uploader) publishInfo(resourceId string) error { + body, err := u.driver.request("https://music.163.com/api/upload/cloud/info/v2", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "md5": u.md5, + "filename": u.filename, + "song": u.meta.name, + "album": u.meta.album, + "artist": u.meta.artist, + "songid": u.meta.songId, + "resourceId": resourceId, + "bitrate": "999000", + }, + }) + if err != nil { + return err + } + + _, err = u.driver.request("https://interface.music.163.com/api/cloud/pub/v2", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "songid": utils.Json.Get(body, "songId").ToString(), + }, + }) + if err != nil { + return err + } + + return nil +} + +func (u *uploader) upload(stream model.FileStreamer) error { + bucket := "jd-musicrep-privatecloud-audio-public" + token, err := u.allocToken(bucket) + if err != nil { + return err + } + + body, err := u.driver.request("https://wanproxy.127.net/lbs?version=1.0&bucketname="+bucket, http.MethodGet, + ReqOption{}, + ) + if err != nil { + return err + } + var resp HostsResp + err = utils.Json.Unmarshal(body, &resp) + if err != nil { + return err + } + + objectKey := strings.ReplaceAll(token.objectKey, "/", "%2F") + _, err = u.driver.request( + resp.Upload[0]+"/"+bucket+"/"+objectKey+"?offset=0&complete=true&version=1.0", + http.MethodPost, + ReqOption{ + stream: stream, + headers: map[string]string{ + "x-nos-token": token.token, + "Content-Type": "audio/mpeg", + "Content-Length": u.size, + "Content-MD5": u.md5, + }, + }, + ) + if err != nil { + return err + } + + return nil +} diff --git a/drivers/netease_music/util.go b/drivers/netease_music/util.go new file mode 100644 index 00000000000..4d0696eb82b --- /dev/null +++ b/drivers/netease_music/util.go @@ -0,0 +1,246 @@ +package netease_music + +import ( + "io" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *NeteaseMusic) request(url, method string, opt ReqOption) ([]byte, error) { + req := base.RestyClient.R() + + req.SetHeader("Cookie", d.Addition.Cookie) + + if strings.Contains(url, "music.163.com") { + req.SetHeader("Referer", "https://music.163.com") + } + + if opt.cookies != nil { + for _, cookie := range opt.cookies { + req.SetCookie(cookie) + } + } + + if opt.headers != nil { + for header, value := range opt.headers { + req.SetHeader(header, value) + } + } + + data := opt.data + if opt.crypto == "weapi" { + data = weapi(data) + re, _ := regexp.Compile(`/\w*api/`) + url = re.ReplaceAllString(url, "/weapi/") + } else if opt.crypto == "eapi" { + ch := new(Characteristic).fromDriver(d) + req.SetCookies(ch.toCookies()) + data = eapi(opt.url, ch.merge(data)) + re, _ := regexp.Compile(`/\w*api/`) + url = re.ReplaceAllString(url, "/eapi/") + } else if opt.crypto == "linuxapi" { + re, _ := regexp.Compile(`/\w*api/`) + data = linuxapi(map[string]interface{}{ + "url": re.ReplaceAllString(url, "/api/"), + "method": method, + "params": data, + }) + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36") + url = "https://music.163.com/api/linux/forward" + } + + if method == http.MethodPost { + if opt.stream != nil { + req.SetContentLength(true) + req.SetBody(io.ReadCloser(opt.stream)) + } else { + req.SetFormData(data) + } + res, err := req.Post(url) + return res.Body(), err + } + + if method == http.MethodGet { + res, err := req.Get(url) + return res.Body(), err + } + + return nil, errs.NotImplement +} + +func (d *NeteaseMusic) getSongObjs(args model.ListArgs) ([]model.Obj, error) { + body, err := d.request("https://music.163.com/weapi/v1/cloud/get", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "limit": strconv.FormatUint(d.Addition.SongLimit, 10), + "offset": "0", + }, + cookies: []*http.Cookie{ + {Name: "os", Value: "pc"}, + }, + }) + if err != nil { + return nil, err + } + + var resp ListResp + err = utils.Json.Unmarshal(body, &resp) + if err != nil { + return nil, err + } + + d.fileMapByName = make(map[string]model.Obj) + files := make([]model.Obj, 0, len(resp.Data)) + for _, f := range resp.Data { + song := &model.ObjThumb{ + Object: model.Object{ + IsFolder: false, + Size: f.FileSize, + Name: f.FileName, + Modified: time.UnixMilli(f.AddTime), + ID: strconv.FormatInt(f.SongId, 10), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.SimpleSong.Al.PicUrl}, + } + d.fileMapByName[song.Name] = song + files = append(files, song) + + // map song id for lyric + lrcName := strings.Split(f.FileName, ".")[0] + ".lrc" + lrc := &model.Object{ + IsFolder: false, + Name: lrcName, + Path: path.Join(args.ReqPath, lrcName), + ID: strconv.FormatInt(f.SongId, 10), + } + d.fileMapByName[lrc.Name] = lrc + } + + return files, nil +} + +func (d *NeteaseMusic) getSongLink(file model.Obj) (*model.Link, error) { + body, err := d.request( + "https://music.163.com/api/song/enhance/player/url", http.MethodPost, ReqOption{ + crypto: "linuxapi", + data: map[string]string{ + "ids": "[" + file.GetID() + "]", + "br": "999000", + }, + cookies: []*http.Cookie{ + {Name: "os", Value: "pc"}, + }, + }, + ) + if err != nil { + return nil, err + } + + var resp SongResp + err = utils.Json.Unmarshal(body, &resp) + if err != nil { + return nil, err + } + + if len(resp.Data) < 1 { + return nil, errs.ObjectNotFound + } + + return &model.Link{URL: resp.Data[0].Url}, nil +} + +func (d *NeteaseMusic) getLyricObj(file model.Obj) (model.Obj, error) { + if lrc, ok := file.(*LyricObj); ok { + return lrc, nil + } + + body, err := d.request( + "https://music.163.com/api/song/lyric?_nmclfl=1", http.MethodPost, ReqOption{ + data: map[string]string{ + "id": file.GetID(), + "tv": "-1", + "lv": "-1", + "rv": "-1", + "kv": "-1", + }, + cookies: []*http.Cookie{ + {Name: "os", Value: "ios"}, + }, + }, + ) + if err != nil { + return nil, err + } + + lyric := utils.Json.Get(body, "lrc", "lyric").ToString() + + return &LyricObj{ + lyric: lyric, + Object: model.Object{ + IsFolder: false, + ID: file.GetID(), + Name: file.GetName(), + Path: file.GetPath(), + Size: int64(len(lyric)), + }, + }, nil +} + +func (d *NeteaseMusic) removeSongObj(file model.Obj) error { + _, err := d.request("http://music.163.com/weapi/cloud/del", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "songIds": "[" + file.GetID() + "]", + }, + }) + + return err +} + +func (d *NeteaseMusic) putSongStream(stream model.FileStreamer) error { + tmp, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + defer tmp.Close() + + u := uploader{driver: d, file: tmp} + + err = u.init(stream) + if err != nil { + return err + } + + err = u.checkIfExisted() + if err != nil { + return err + } + + token, err := u.allocToken() + if err != nil { + return err + } + + if u.meta.needUpload { + err = u.upload(stream) + if err != nil { + return err + } + } + + err = u.publishInfo(token.resourceId) + if err != nil { + return err + } + + return nil +} diff --git a/go.mod b/go.mod index 11a858a2ac9..a994d7688f7 100644 --- a/go.mod +++ b/go.mod @@ -108,6 +108,7 @@ require ( github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/geoffgarside/ber v1.1.0 // indirect diff --git a/go.sum b/go.sum index 05a9b3385a1..539151e660a 100644 --- a/go.sum +++ b/go.sum @@ -123,6 +123,8 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 h1:OtSeLS5y0Uy01jaKK4mA/WVIYtpzVm63vLVAPzJXigg= +github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8/go.mod h1:apkPC/CR3s48O2D7Y++n1XWEpgPNNCjXYga3PPbJe2E= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= From b57afd0a98638dd551dcd5c2c5f9deb6d2cd3fe8 Mon Sep 17 00:00:00 2001 From: George Chen <24411020+okcy1016@users.noreply.github.com> Date: Thu, 9 May 2024 14:53:25 +0800 Subject: [PATCH 182/659] fix(sftp): reconnect to server when connection was broken (#6416 close #6403) * fix(sftp): reconnect to server when conn was broken (close #6403) * fix(sftp): fix typo --------- Co-authored-by: George Chen --- drivers/sftp/driver.go | 24 +++++++++++++++++++++++- drivers/sftp/util.go | 18 ++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go index 77f5198457c..1f216598d2d 100644 --- a/drivers/sftp/driver.go +++ b/drivers/sftp/driver.go @@ -16,7 +16,8 @@ import ( type SFTP struct { model.Storage Addition - client *sftp.Client + client *sftp.Client + clientConnectionError error } func (d *SFTP) Config() driver.Config { @@ -39,6 +40,9 @@ func (d *SFTP) Drop(ctx context.Context) error { } func (d *SFTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.clientReconnectOnConnectionError(); err != nil { + return nil, err + } log.Debugf("[sftp] list dir: %s", dir.GetPath()) files, err := d.client.ReadDir(dir.GetPath()) if err != nil { @@ -51,6 +55,9 @@ func (d *SFTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] } func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.clientReconnectOnConnectionError(); err != nil { + return nil, err + } remoteFile, err := d.client.Open(file.GetPath()) if err != nil { return nil, err @@ -62,14 +69,23 @@ func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* } func (d *SFTP) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } return d.client.MkdirAll(path.Join(parentDir.GetPath(), dirName)) } func (d *SFTP) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } return d.client.Rename(srcObj.GetPath(), path.Join(dstDir.GetPath(), srcObj.GetName())) } func (d *SFTP) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } return d.client.Rename(srcObj.GetPath(), path.Join(path.Dir(srcObj.GetPath()), newName)) } @@ -78,10 +94,16 @@ func (d *SFTP) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *SFTP) Remove(ctx context.Context, obj model.Obj) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } return d.remove(obj.GetPath()) } func (d *SFTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } dstFile, err := d.client.Create(path.Join(dstDir.GetPath(), stream.GetName())) if err != nil { return err diff --git a/drivers/sftp/util.go b/drivers/sftp/util.go index 3deb8dcf94b..eaeeaff5814 100644 --- a/drivers/sftp/util.go +++ b/drivers/sftp/util.go @@ -4,6 +4,7 @@ import ( "path" "github.com/pkg/sftp" + log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" ) @@ -30,6 +31,23 @@ func (d *SFTP) initClient() error { return err } d.client, err = sftp.NewClient(conn) + if err == nil { + d.clientConnectionError = nil + go func(d *SFTP) { + d.clientConnectionError = d.client.Wait() + }(d) + } + return err +} + +func (d *SFTP) clientReconnectOnConnectionError() error { + err := d.clientConnectionError + if err == nil { + return nil + } + log.Debugf("[sftp] discarding closed sftp connection: %v", err) + _ = d.client.Close() + err = d.initClient() return err } From 8bf93562eb76971426b18cb463b21b05bac622b2 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 9 May 2024 14:54:53 +0800 Subject: [PATCH 183/659] fix(baidu): unknown type for custom upload part size (close #6435) --- drivers/baidu_netdisk/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 7e01767b2ed..bf2aed5a2b5 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -17,7 +17,7 @@ type Addition struct { AccessToken string UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` - CustomUploadPartSize int64 `json:"custom_upload_part_size" default:"0" help:"0 for auto"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` } var config = driver.Config{ From 78a9676c7cdd282b68f27acdd2a095df427aa228 Mon Sep 17 00:00:00 2001 From: Sakura-Byte <42319937+Sakura-Byte@users.noreply.github.com> Date: Sun, 12 May 2024 17:34:36 +0800 Subject: [PATCH 184/659] feat(alist_v3): Optional pass UA to upstream remote (#6443) * fix(115): Support 115 302 redirect while getting link under (nested) alist_v3 remote * chore: simplify logic * chore: simplify logic * use internal UA * add option to set if the user want their ua be passed to upstream --- drivers/alist_v3/driver.go | 10 +++++++++- drivers/alist_v3/meta.go | 11 ++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index f46e68d07a4..53fb93caa1f 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -109,11 +109,19 @@ func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp common.Resp[FsGetResp] + // if PassUAToUpsteam is true, then pass the user-agent to the upstream + userAgent := base.UserAgent + if d.PassUAToUpsteam { + userAgent = args.Header.Get("user-agent") + if userAgent == "" { + userAgent = base.UserAgent + } + } _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(FsGetReq{ Path: file.GetPath(), Password: d.MetaPassword, - }) + }).SetHeader("user-agent", userAgent) }) if err != nil { return nil, err diff --git a/drivers/alist_v3/meta.go b/drivers/alist_v3/meta.go index bb3d35aea22..c04c737b149 100644 --- a/drivers/alist_v3/meta.go +++ b/drivers/alist_v3/meta.go @@ -7,11 +7,12 @@ import ( type Addition struct { driver.RootPath - Address string `json:"url" required:"true"` - MetaPassword string `json:"meta_password"` - Username string `json:"username"` - Password string `json:"password"` - Token string `json:"token"` + Address string `json:"url" required:"true"` + MetaPassword string `json:"meta_password"` + Username string `json:"username"` + Password string `json:"password"` + Token string `json:"token"` + PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` } var config = driver.Config{ From bbe3d4e19f6171154b14a5b65fe527007e655607 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Tue, 21 May 2024 23:24:28 +0800 Subject: [PATCH 185/659] feat: add supports for thunderX driver (#6464) --- drivers/all.go | 1 + drivers/thunderx/driver.go | 527 +++++++++++++++++++++++++++++++++++++ drivers/thunderx/meta.go | 103 ++++++++ drivers/thunderx/types.go | 206 +++++++++++++++ drivers/thunderx/util.go | 202 ++++++++++++++ 5 files changed, 1039 insertions(+) create mode 100644 drivers/thunderx/driver.go create mode 100644 drivers/thunderx/meta.go create mode 100644 drivers/thunderx/types.go create mode 100644 drivers/thunderx/util.go diff --git a/drivers/all.go b/drivers/all.go index d1d5f84a989..bae72b3175a 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -46,6 +46,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" + _ "github.com/alist-org/alist/v3/drivers/thunderx" _ "github.com/alist-org/alist/v3/drivers/trainbit" _ "github.com/alist-org/alist/v3/drivers/url_tree" _ "github.com/alist-org/alist/v3/drivers/uss" diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go new file mode 100644 index 00000000000..17835d913f2 --- /dev/null +++ b/drivers/thunderx/driver.go @@ -0,0 +1,527 @@ +package thunderx + +import ( + "context" + "fmt" + "github.com/go-resty/resty/v2" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" +) + +type ThunderX struct { + *XunLeiXCommon + model.Storage + Addition + + identity string +} + +func (x *ThunderX) Config() driver.Config { + return config +} + +func (x *ThunderX) GetAddition() driver.Additional { + return &x.Addition +} + +func (x *ThunderX) Init(ctx context.Context) (err error) { + // 初始化所需参数 + if x.XunLeiXCommon == nil { + x.XunLeiXCommon = &XunLeiXCommon{ + Common: &Common{ + client: base.NewRestyClient(), + Algorithms: []string{ + "lHwINjLeqssT28Ym99p5MvR", + "xvFcxvtqPKCa9Ajf", + "2ywOP8spKHzfuhZMUYZ9IpsViq0t8vT0", + "FTBrJism20SHKQ2m2", + "BHrWJsPwjnr5VeLtOUr2191X9uXhWmt", + "yu0QgHEjNmDoPNwXN17so2hQlDT83T", + "OcaMfLMCGZ7oYlvZGIbTqb4U7cCY", + "jBGGu0GzXOjtCXYwkOBb+c6TZ/Nymv", + "YLWRjVor2rOuYEL", + "94wjoPazejyNC+gRpOj+JOm1XXvxa", + }, + DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), + ClientID: "ZQL_zwA4qhHcoe_2", + ClientSecret: "Og9Vr1L8Ee6bh0olFxFDRg", + ClientVersion: "1.05.0.2115", + PackageName: "com.thunder.downloader", + UserAgent: "ANDROID-com.thunder.downloader/1.05.0.2115 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", + DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", + UseVideoUrl: x.UseVideoUrl, + + refreshCTokenCk: func(token string) { + x.CaptchaToken = token + op.MustSaveDriverStorage(x) + }, + }, + refreshTokenFunc: func() error { + // 通过RefreshToken刷新 + token, err := x.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + // 重新登录 + token, err = x.Login(x.Username, x.Password) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + op.MustSaveDriverStorage(x) + } + } + x.SetTokenResp(token) + return err + }, + } + } + + // 自定义验证码token + ctoekn := strings.TrimSpace(x.CaptchaToken) + if ctoekn != "" { + x.SetCaptchaToken(ctoekn) + } + x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl + x.Addition.RootFolderID = x.RootFolderID + // 防止重复登录 + identity := x.GetIdentity() + if x.identity != identity || !x.IsLogin() { + x.identity = identity + // 登录 + token, err := x.Login(x.Username, x.Password) + if err != nil { + return err + } + x.SetTokenResp(token) + } + return nil +} + +func (x *ThunderX) Drop(ctx context.Context) error { + return nil +} + +type ThunderXExpert struct { + *XunLeiXCommon + model.Storage + ExpertAddition + + identity string +} + +func (x *ThunderXExpert) Config() driver.Config { + return configExpert +} + +func (x *ThunderXExpert) GetAddition() driver.Additional { + return &x.ExpertAddition +} + +func (x *ThunderXExpert) Init(ctx context.Context) (err error) { + // 防止重复登录 + identity := x.GetIdentity() + if identity != x.identity || !x.IsLogin() { + x.identity = identity + x.XunLeiXCommon = &XunLeiXCommon{ + Common: &Common{ + client: base.NewRestyClient(), + + DeviceID: func() string { + if len(x.DeviceID) != 32 { + return utils.GetMD5EncodeStr(x.DeviceID) + } + return x.DeviceID + }(), + ClientID: x.ClientID, + ClientSecret: x.ClientSecret, + ClientVersion: x.ClientVersion, + PackageName: x.PackageName, + UserAgent: x.UserAgent, + DownloadUserAgent: x.DownloadUserAgent, + UseVideoUrl: x.UseVideoUrl, + + refreshCTokenCk: func(token string) { + x.CaptchaToken = token + op.MustSaveDriverStorage(x) + }, + }, + } + + if x.CaptchaToken != "" { + x.SetCaptchaToken(x.CaptchaToken) + } + x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl + x.ExpertAddition.RootFolderID = x.RootFolderID + // 签名方法 + if x.SignType == "captcha_sign" { + x.Common.Timestamp = x.Timestamp + x.Common.CaptchaSign = x.CaptchaSign + } else { + x.Common.Algorithms = strings.Split(x.Algorithms, ",") + } + + // 登录方式 + if x.LoginType == "refresh_token" { + // 通过RefreshToken登录 + token, err := x.XunLeiXCommon.RefreshToken(x.ExpertAddition.RefreshToken) + if err != nil { + return err + } + x.SetTokenResp(token) + + // 刷新token方法 + x.SetRefreshTokenFunc(func() error { + token, err := x.XunLeiXCommon.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + } + x.SetTokenResp(token) + op.MustSaveDriverStorage(x) + return err + }) + } else { + // 通过用户密码登录 + token, err := x.Login(x.Username, x.Password) + if err != nil { + return err + } + x.SetTokenResp(token) + x.SetRefreshTokenFunc(func() error { + token, err := x.XunLeiXCommon.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + token, err = x.Login(x.Username, x.Password) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + } + } + x.SetTokenResp(token) + op.MustSaveDriverStorage(x) + return err + }) + } + } else { + // 仅修改验证码token + if x.CaptchaToken != "" { + x.SetCaptchaToken(x.CaptchaToken) + } + x.XunLeiXCommon.UserAgent = x.UserAgent + x.XunLeiXCommon.DownloadUserAgent = x.DownloadUserAgent + x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl + x.ExpertAddition.RootFolderID = x.RootFolderID + } + return nil +} + +func (x *ThunderXExpert) Drop(ctx context.Context) error { + return nil +} + +func (x *ThunderXExpert) SetTokenResp(token *TokenResp) { + x.XunLeiXCommon.SetTokenResp(token) + if token != nil { + x.ExpertAddition.RefreshToken = token.RefreshToken + } +} + +type XunLeiXCommon struct { + *Common + *TokenResp // 登录信息 + + refreshTokenFunc func() error +} + +func (xc *XunLeiXCommon) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return xc.getFiles(ctx, dir.GetID()) +} + +func (xc *XunLeiXCommon) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var lFile Files + _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", file.GetID()) + //r.SetQueryParam("space", "") + }, &lFile) + if err != nil { + return nil, err + } + link := &model.Link{ + URL: lFile.WebContentLink, + Header: http.Header{ + "User-Agent": {xc.DownloadUserAgent}, + }, + } + + if xc.UseVideoUrl { + for _, media := range lFile.Medias { + if media.Link.URL != "" { + link.URL = media.Link.URL + break + } + } + } + + /* + strs := regexp.MustCompile(`e=([0-9]*)`).FindStringSubmatch(lFile.WebContentLink) + if len(strs) == 2 { + timestamp, err := strconv.ParseInt(strs[1], 10, 64) + if err == nil { + expired := time.Duration(timestamp-time.Now().Unix()) * time.Second + link.Expiration = &expired + } + } + */ + return link, nil +} + +func (xc *XunLeiXCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FOLDER, + "name": dirName, + "parent_id": parentDir.GetID(), + }) + }, nil) + return err +} + +func (xc *XunLeiXCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + }) + }, nil) + return err +} + +func (xc *XunLeiXCommon) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", srcObj.GetID()) + r.SetBody(&base.Json{"name": newName}) + }, nil) + return err +} + +func (xc *XunLeiXCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + }) + }, nil) + return err +} + +func (xc *XunLeiXCommon) Remove(ctx context.Context, obj model.Obj) error { + _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", obj.GetID()) + r.SetBody("{}") + }, nil) + return err +} + +func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + hi := stream.GetHash() + gcid := hi.GetHash(hash_extend.GCID) + if len(gcid) < hash_extend.GCID.Width { + tFile, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + + gcid, err = utils.HashFile(hash_extend.GCID, tFile, stream.GetSize()) + if err != nil { + return err + } + } + + var resp UploadTaskResponse + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": stream.GetName(), + "size": stream.GetSize(), + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + }) + }, &resp) + if err != nil { + return err + } + + param := resp.Resumable.Params + if resp.UploadType == UPLOAD_TYPE_RESUMABLE { + param.Endpoint = strings.TrimLeft(param.Endpoint, param.Bucket+".") + s, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(param.AccessKeyID, param.AccessKeySecret, param.SecurityToken), + Region: aws.String("xunlei"), + Endpoint: aws.String(param.Endpoint), + }) + if err != nil { + return err + } + uploader := s3manager.NewUploader(s) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } + _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(param.Bucket), + Key: aws.String(param.Key), + Expires: aws.Time(param.Expiration), + Body: stream, + }) + return err + } + return nil +} + +func (xc *XunLeiXCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { + files := make([]model.Obj, 0) + var pageToken string + for { + var fileList FileList + _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + r.SetQueryParams(map[string]string{ + "space": "", + "__type": "drive", + "refresh": "true", + "__sync": "true", + "parent_id": folderId, + "page_token": pageToken, + "with_audit": "true", + "limit": "100", + "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, + }) + }, &fileList) + if err != nil { + return nil, err + } + + for i := 0; i < len(fileList.Files); i++ { + files = append(files, &fileList.Files[i]) + } + + if fileList.NextPageToken == "" { + break + } + pageToken = fileList.NextPageToken + } + return files, nil +} + +// 设置刷新Token的方法 +func (xc *XunLeiXCommon) SetRefreshTokenFunc(fn func() error) { + xc.refreshTokenFunc = fn +} + +// 设置Token +func (xc *XunLeiXCommon) SetTokenResp(tr *TokenResp) { + xc.TokenResp = tr +} + +// 携带Authorization和CaptchaToken的请求 +func (xc *XunLeiXCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + data, err := xc.Common.Request(url, method, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Authorization": xc.Token(), + "X-Captcha-Token": xc.GetCaptchaToken(), + }) + if callback != nil { + callback(req) + } + }, resp) + + errResp, ok := err.(*ErrResp) + if !ok { + return nil, err + } + + switch errResp.ErrorCode { + case 0: + return data, nil + case 4122, 4121, 10, 16: + if xc.refreshTokenFunc != nil { + if err = xc.refreshTokenFunc(); err == nil { + break + } + } + return nil, err + case 9: // 验证码token过期 + if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.UserID); err != nil { + return nil, err + } + default: + return nil, err + } + return xc.Request(url, method, callback, resp) +} + +// 刷新Token +func (xc *XunLeiXCommon) RefreshToken(refreshToken string) (*TokenResp, error) { + var resp TokenResp + _, err := xc.Common.Request(XLUSER_API_URL+"/auth/token", http.MethodPost, func(req *resty.Request) { + req.SetBody(&base.Json{ + "grant_type": "refresh_token", + "refresh_token": refreshToken, + "client_id": xc.ClientID, + "client_secret": xc.ClientSecret, + }) + }, &resp) + if err != nil { + return nil, err + } + + if resp.RefreshToken == "" { + return nil, errs.EmptyToken + } + return &resp, nil +} + +// 登录 +func (xc *XunLeiXCommon) Login(username, password string) (*TokenResp, error) { + url := XLUSER_API_URL + "/auth/signin" + err := xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username) + if err != nil { + return nil, err + } + + var resp TokenResp + _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(&SignInRequest{ + CaptchaToken: xc.GetCaptchaToken(), + ClientID: xc.ClientID, + ClientSecret: xc.ClientSecret, + Username: username, + Password: password, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (xc *XunLeiXCommon) IsLogin() bool { + if xc.TokenResp == nil { + return false + } + _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) + return err == nil +} diff --git a/drivers/thunderx/meta.go b/drivers/thunderx/meta.go new file mode 100644 index 00000000000..2c114c0f284 --- /dev/null +++ b/drivers/thunderx/meta.go @@ -0,0 +1,103 @@ +package thunderx + +import ( + "crypto/md5" + "encoding/hex" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// 高级设置 +type ExpertAddition struct { + driver.RootID + + LoginType string `json:"login_type" type:"select" options:"user,refresh_token" default:"user"` + SignType string `json:"sign_type" type:"select" options:"algorithms,captcha_sign" default:"algorithms"` + + // 登录方式1 + Username string `json:"username" required:"true" help:"login type is user,this is required"` + Password string `json:"password" required:"true" help:"login type is user,this is required"` + // 登录方式2 + RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` + + // 签名方法1 + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"lHwINjLeqssT28Ym99p5MvR,xvFcxvtqPKCa9Ajf,2ywOP8spKHzfuhZMUYZ9IpsViq0t8vT0,FTBrJism20SHKQ2m2,BHrWJsPwjnr5VeLtOUr2191X9uXhWmt,yu0QgHEjNmDoPNwXN17so2hQlDT83T,OcaMfLMCGZ7oYlvZGIbTqb4U7cCY,jBGGu0GzXOjtCXYwkOBb+c6TZ/Nymv,YLWRjVor2rOuYEL,94wjoPazejyNC+gRpOj+JOm1XXvxa"` + // 签名方法2 + CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` + Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` + + // 验证码 + CaptchaToken string `json:"captcha_token"` + + // 必要且影响登录,由签名决定 + DeviceID string `json:"device_id" required:"true" default:"9aa5c268e7bcfc197a9ad88e2fb330e5"` + ClientID string `json:"client_id" required:"true" default:"ZQL_zwA4qhHcoe_2"` + ClientSecret string `json:"client_secret" required:"true" default:"Og9Vr1L8Ee6bh0olFxFDRg"` + ClientVersion string `json:"client_version" required:"true" default:"1.05.0.2115"` + PackageName string `json:"package_name" required:"true" default:"com.thunder.downloader"` + + //不影响登录,影响下载速度 + UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.thunder.downloader/1.05.0.2115 netWorkType/4G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gdcf98eab238b) (JAVA 0)"` + DownloadUserAgent string `json:"download_user_agent" required:"true" default:"Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)"` + + //优先使用视频链接代替下载链接 + UseVideoUrl bool `json:"use_video_url"` +} + +// 登录特征,用于判断是否重新登录 +func (i *ExpertAddition) GetIdentity() string { + hash := md5.New() + if i.LoginType == "refresh_token" { + hash.Write([]byte(i.RefreshToken)) + } else { + hash.Write([]byte(i.Username + i.Password)) + } + + if i.SignType == "captcha_sign" { + hash.Write([]byte(i.CaptchaSign + i.Timestamp)) + } else { + hash.Write([]byte(i.Algorithms)) + } + + hash.Write([]byte(i.DeviceID)) + hash.Write([]byte(i.ClientID)) + hash.Write([]byte(i.ClientSecret)) + hash.Write([]byte(i.ClientVersion)) + hash.Write([]byte(i.PackageName)) + return hex.EncodeToString(hash.Sum(nil)) +} + +type Addition struct { + driver.RootID + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + CaptchaToken string `json:"captcha_token"` + UseVideoUrl bool `json:"use_video_url" default:"true"` +} + +// 登录特征,用于判断是否重新登录 +func (i *Addition) GetIdentity() string { + return utils.GetMD5EncodeStr(i.Username + i.Password) +} + +var config = driver.Config{ + Name: "ThunderX", + LocalSort: true, + OnlyProxy: true, +} + +var configExpert = driver.Config{ + Name: "ThunderXExpert", + LocalSort: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ThunderX{} + }) + op.RegisterDriver(func() driver.Driver { + return &ThunderXExpert{} + }) +} diff --git a/drivers/thunderx/types.go b/drivers/thunderx/types.go new file mode 100644 index 00000000000..77cfa0f2415 --- /dev/null +++ b/drivers/thunderx/types.go @@ -0,0 +1,206 @@ +package thunderx + +import ( + "fmt" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" +) + +type ErrResp struct { + ErrorCode int64 `json:"error_code"` + ErrorMsg string `json:"error"` + ErrorDescription string `json:"error_description"` + // ErrorDetails interface{} `json:"error_details"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) +} + +/* +* 验证码Token +**/ +type CaptchaTokenRequest struct { + Action string `json:"action"` + CaptchaToken string `json:"captcha_token"` + ClientID string `json:"client_id"` + DeviceID string `json:"device_id"` + Meta map[string]string `json:"meta"` + RedirectUri string `json:"redirect_uri"` +} + +type CaptchaTokenResponse struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Url string `json:"url"` +} + +/* +* 登录 +**/ +type TokenResp struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int64 `json:"expires_in"` + + Sub string `json:"sub"` + UserID string `json:"user_id"` +} + +func (t *TokenResp) Token() string { + return fmt.Sprint(t.TokenType, " ", t.AccessToken) +} + +type SignInRequest struct { + CaptchaToken string `json:"captcha_token"` + + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + + Username string `json:"username"` + Password string `json:"password"` +} + +/* +* 文件 +**/ +type FileList struct { + Kind string `json:"kind"` + NextPageToken string `json:"next_page_token"` + Files []Files `json:"files"` + Version string `json:"version"` + VersionOutdated bool `json:"version_outdated"` +} + +type Link struct { + URL string `json:"url"` + Token string `json:"token"` + Expire time.Time `json:"expire"` + Type string `json:"type"` +} + +var _ model.Obj = (*Files)(nil) + +type Files struct { + Kind string `json:"kind"` + ID string `json:"id"` + ParentID string `json:"parent_id"` + Name string `json:"name"` + //UserID string `json:"user_id"` + Size string `json:"size"` + //Revision string `json:"revision"` + //FileExtension string `json:"file_extension"` + //MimeType string `json:"mime_type"` + //Starred bool `json:"starred"` + WebContentLink string `json:"web_content_link"` + CreatedTime time.Time `json:"created_time"` + ModifiedTime time.Time `json:"modified_time"` + IconLink string `json:"icon_link"` + ThumbnailLink string `json:"thumbnail_link"` + // Md5Checksum string `json:"md5_checksum"` + Hash string `json:"hash"` + // Links map[string]Link `json:"links"` + // Phase string `json:"phase"` + // Audit struct { + // Status string `json:"status"` + // Message string `json:"message"` + // Title string `json:"title"` + // } `json:"audit"` + Medias []struct { + //Category string `json:"category"` + //IconLink string `json:"icon_link"` + //IsDefault bool `json:"is_default"` + //IsOrigin bool `json:"is_origin"` + //IsVisible bool `json:"is_visible"` + Link Link `json:"link"` + //MediaID string `json:"media_id"` + //MediaName string `json:"media_name"` + //NeedMoreQuota bool `json:"need_more_quota"` + //Priority int `json:"priority"` + //RedirectLink string `json:"redirect_link"` + //ResolutionName string `json:"resolution_name"` + // Video struct { + // AudioCodec string `json:"audio_codec"` + // BitRate int `json:"bit_rate"` + // Duration int `json:"duration"` + // FrameRate int `json:"frame_rate"` + // Height int `json:"height"` + // VideoCodec string `json:"video_codec"` + // VideoType string `json:"video_type"` + // Width int `json:"width"` + // } `json:"video"` + // VipTypes []string `json:"vip_types"` + } `json:"medias"` + Trashed bool `json:"trashed"` + DeleteTime string `json:"delete_time"` + OriginalURL string `json:"original_url"` + //Params struct{} `json:"params"` + //OriginalFileIndex int `json:"original_file_index"` + //Space string `json:"space"` + //Apps []interface{} `json:"apps"` + //Writable bool `json:"writable"` + //FolderType string `json:"folder_type"` + //Collection interface{} `json:"collection"` +} + +func (c *Files) GetHash() utils.HashInfo { + return utils.NewHashInfo(hash_extend.GCID, c.Hash) +} + +func (c *Files) GetSize() int64 { size, _ := strconv.ParseInt(c.Size, 10, 64); return size } +func (c *Files) GetName() string { return c.Name } +func (c *Files) CreateTime() time.Time { return c.CreatedTime } +func (c *Files) ModTime() time.Time { return c.ModifiedTime } +func (c *Files) IsDir() bool { return c.Kind == FOLDER } +func (c *Files) GetID() string { return c.ID } +func (c *Files) GetPath() string { return "" } +func (c *Files) Thumb() string { return c.ThumbnailLink } + +/* +* 上传 +**/ +type UploadTaskResponse struct { + UploadType string `json:"upload_type"` + + /*//UPLOAD_TYPE_FORM + Form struct { + //Headers struct{} `json:"headers"` + Kind string `json:"kind"` + Method string `json:"method"` + MultiParts struct { + OSSAccessKeyID string `json:"OSSAccessKeyId"` + Signature string `json:"Signature"` + Callback string `json:"callback"` + Key string `json:"key"` + Policy string `json:"policy"` + XUserData string `json:"x:user_data"` + } `json:"multi_parts"` + URL string `json:"url"` + } `json:"form"`*/ + + //UPLOAD_TYPE_RESUMABLE + Resumable struct { + Kind string `json:"kind"` + Params struct { + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + Bucket string `json:"bucket"` + Endpoint string `json:"endpoint"` + Expiration time.Time `json:"expiration"` + Key string `json:"key"` + SecurityToken string `json:"security_token"` + } `json:"params"` + Provider string `json:"provider"` + } `json:"resumable"` + + File Files `json:"file"` +} diff --git a/drivers/thunderx/util.go b/drivers/thunderx/util.go new file mode 100644 index 00000000000..6fa323ebc28 --- /dev/null +++ b/drivers/thunderx/util.go @@ -0,0 +1,202 @@ +package thunderx + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + "regexp" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + API_URL = "https://api-pan.xunleix.com/drive/v1" + FILE_API_URL = API_URL + "/files" + XLUSER_API_URL = "https://xluser-ssl.xunleix.com/v1" +) + +const ( + FOLDER = "drive#folder" + FILE = "drive#file" + RESUMABLE = "drive#resumable" +) + +const ( + UPLOAD_TYPE_UNKNOWN = "UPLOAD_TYPE_UNKNOWN" + //UPLOAD_TYPE_FORM = "UPLOAD_TYPE_FORM" + UPLOAD_TYPE_RESUMABLE = "UPLOAD_TYPE_RESUMABLE" + UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" +) + +func GetAction(method string, url string) string { + urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] + return method + ":" + urlpath +} + +type Common struct { + client *resty.Client + + captchaToken string + + // 签名相关,二选一 + Algorithms []string + Timestamp, CaptchaSign string + + // 必要值,签名相关 + DeviceID string + ClientID string + ClientSecret string + ClientVersion string + PackageName string + UserAgent string + DownloadUserAgent string + UseVideoUrl bool + + // 验证码token刷新成功回调 + refreshCTokenCk func(token string) +} + +func (c *Common) SetCaptchaToken(captchaToken string) { + c.captchaToken = captchaToken +} +func (c *Common) GetCaptchaToken() string { + return c.captchaToken +} + +// 刷新验证码token(登录后) +func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { + metas := map[string]string{ + "client_version": c.ClientVersion, + "package_name": c.PackageName, + "user_id": userID, + } + metas["timestamp"], metas["captcha_sign"] = c.GetCaptchaSign() + return c.refreshCaptchaToken(action, metas) +} + +// 刷新验证码token(登录时) +func (c *Common) RefreshCaptchaTokenInLogin(action, username string) error { + metas := make(map[string]string) + if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { + metas["email"] = username + } else if len(username) >= 11 && len(username) <= 18 { + metas["phone_number"] = username + } else { + metas["username"] = username + } + return c.refreshCaptchaToken(action, metas) +} + +// 获取验证码签名 +func (c *Common) GetCaptchaSign() (timestamp, sign string) { + if len(c.Algorithms) == 0 { + return c.Timestamp, c.CaptchaSign + } + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) + for _, algorithm := range c.Algorithms { + str = utils.GetMD5EncodeStr(str + algorithm) + } + sign = "1." + str + return +} + +// 刷新验证码token +func (c *Common) refreshCaptchaToken(action string, metas map[string]string) error { + param := CaptchaTokenRequest{ + Action: action, + CaptchaToken: c.captchaToken, + ClientID: c.ClientID, + DeviceID: c.DeviceID, + Meta: metas, + RedirectUri: "xlaccsdk01://xbase.cloud/callback?state=harbor", + } + var e ErrResp + var resp CaptchaTokenResponse + _, err := c.Request(XLUSER_API_URL+"/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + req.SetError(&e).SetBody(param) + }, &resp) + + if err != nil { + return err + } + + if e.IsError() { + return &e + } + + if resp.Url != "" { + return fmt.Errorf(`need verify: Click Here`, resp.Url) + } + + if resp.CaptchaToken == "" { + return fmt.Errorf("empty captchaToken") + } + + if c.refreshCTokenCk != nil { + c.refreshCTokenCk(resp.CaptchaToken) + } + c.SetCaptchaToken(resp.CaptchaToken) + return nil +} + +// 只有基础信息的请求 +func (c *Common) Request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := c.client.R().SetHeaders(map[string]string{ + "user-agent": c.UserAgent, + "accept": "application/json;charset=UTF-8", + "x-device-id": c.DeviceID, + "x-client-id": c.ClientID, + "x-client-version": c.ClientVersion, + }) + + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + + var erron ErrResp + utils.Json.Unmarshal(res.Body(), &erron) + if erron.IsError() { + return nil, &erron + } + + return res.Body(), nil +} + +// 计算文件Gcid +func getGcid(r io.Reader, size int64) (string, error) { + calcBlockSize := func(j int64) int64 { + var psize int64 = 0x40000 + for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { + psize = psize << 1 + } + return psize + } + + hash1 := sha1.New() + hash2 := sha1.New() + readSize := calcBlockSize(size) + for { + hash2.Reset() + if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { + if err != io.EOF { + return "", err + } + break + } + hash1.Write(hash2.Sum(nil)) + } + return hex.EncodeToString(hash1.Sum(nil)), nil +} From 037850bbd5096707c43e05b361b9b154d1a2d183 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Wed, 22 May 2024 09:27:48 +0800 Subject: [PATCH 186/659] feat(alias): support Rename and Remove (#6478) * feat(alias): support `Rename` and `Remove` * fix(alias): `autoFlatten` not updated after editing * feat(alias): add `protect_same_name` option --- drivers/alias/driver.go | 26 ++++++++++++++++++++++++++ drivers/alias/meta.go | 9 +++++++-- drivers/alias/util.go | 33 +++++++++++++++++++++++++++++++++ internal/errs/errors.go | 4 ++++ 4 files changed, 70 insertions(+), 2 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 271096b3e46..d9b290edc65 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -7,6 +7,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" ) @@ -45,6 +46,9 @@ func (d *Alias) Init(ctx context.Context) error { d.oneKey = k } d.autoFlatten = true + } else { + d.oneKey = "" + d.autoFlatten = false } return nil } @@ -111,4 +115,26 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, errs.ObjectNotFound } +func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + reqPath, err := d.getReqPath(ctx, srcObj) + if err == nil { + return fs.Rename(ctx, *reqPath, newName) + } + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be Rename") + } + return err +} + +func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { + reqPath, err := d.getReqPath(ctx, obj) + if err == nil { + return fs.Remove(ctx, *reqPath) + } + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be Delete") + } + return err +} + var _ driver.Driver = (*Alias)(nil) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 6611e1dc302..2bebe7b00a8 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -9,7 +9,8 @@ type Addition struct { // Usually one of two // driver.RootPath // define other - Paths string `json:"paths" required:"true" type:"text"` + Paths string `json:"paths" required:"true" type:"text"` + ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"` } var config = driver.Config{ @@ -22,6 +23,10 @@ var config = driver.Config{ func init() { op.RegisterDriver(func() driver.Driver { - return &Alias{} + return &Alias{ + Addition: Addition{ + ProtectSameName: true, + }, + } }) } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 4e3d6bf0f5c..803bd0736b9 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -6,6 +6,7 @@ import ( stdpath "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/sign" @@ -112,3 +113,35 @@ func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) link, _, err := fs.Link(ctx, reqPath, args) return link, err } + +func (d *Alias) getReqPath(ctx context.Context, obj model.Obj) (*string, error) { + root, sub := d.getRootAndPath(obj.GetPath()) + if sub == "" || sub == "/" { + return nil, errs.NotSupport + } + dsts, ok := d.pathMap[root] + if !ok { + return nil, errs.ObjectNotFound + } + var reqPath string + var err error + for _, dst := range dsts { + reqPath = stdpath.Join(dst, sub) + _, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) + if err == nil { + if d.ProtectSameName { + if ok { + ok = false + } else { + return nil, errs.NotImplement + } + } else { + break + } + } + } + if err != nil { + return nil, errs.ObjectNotFound + } + return &reqPath, nil +} diff --git a/internal/errs/errors.go b/internal/errs/errors.go index cd681e607b3..ecfe43e3dc0 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -3,6 +3,7 @@ package errs import ( "errors" "fmt" + pkgerr "github.com/pkg/errors" ) @@ -33,3 +34,6 @@ func IsNotFoundError(err error) bool { func IsNotSupportError(err error) bool { return errors.Is(pkgerr.Cause(err), NotSupport) } +func IsNotImplement(err error) bool { + return errors.Is(pkgerr.Cause(err), NotImplement) +} From 9eec87263726fbfbe7b786e3a7d142b8d1a8dbc4 Mon Sep 17 00:00:00 2001 From: Kuingsmile Date: Wed, 22 May 2024 23:28:14 +0800 Subject: [PATCH 187/659] feat(mega): add 2FA support (#6473) * feat(mega): add support for two-factor authentication in Mega driver #6226 * feat(mega): remove debug print statement in Mega driver Init function * feat(mega): add help message for new field --- drivers/mega/driver.go | 11 ++++++++++- drivers/mega/meta.go | 6 ++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index 9fa1d0eeafa..162aeef37e0 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -8,6 +8,7 @@ import ( "time" "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/pquerna/otp/totp" "github.com/rclone/rclone/lib/readers" "github.com/alist-org/alist/v3/internal/driver" @@ -33,8 +34,16 @@ func (d *Mega) GetAddition() driver.Additional { } func (d *Mega) Init(ctx context.Context) error { + var twoFACode = d.TwoFACode d.c = mega.New() - return d.c.Login(d.Email, d.Password) + if d.TwoFASecret != "" { + code, err := totp.GenerateCode(d.TwoFASecret, time.Now()) + if err != nil { + return fmt.Errorf("generate totp code failed: %w", err) + } + twoFACode = code + } + return d.c.MultiFactorLogin(d.Email, d.Password, twoFACode) } func (d *Mega) Drop(ctx context.Context) error { diff --git a/drivers/mega/meta.go b/drivers/mega/meta.go index 77e768f0542..d0758637bb3 100644 --- a/drivers/mega/meta.go +++ b/drivers/mega/meta.go @@ -9,8 +9,10 @@ type Addition struct { // Usually one of two //driver.RootPath //driver.RootID - Email string `json:"email" required:"true"` - Password string `json:"password" required:"true"` + Email string `json:"email" required:"true"` + Password string `json:"password" required:"true"` + TwoFACode string `json:"two_fa_code" required:"false" help:"2FA 6-digit code, filling in the 2FA code alone will not support reloading driver"` + TwoFASecret string `json:"two_fa_secret" required:"false" help:"2FA secret"` } var config = driver.Config{ From 7013d1b7b86705488e714e8a8dcbf3729938e01e Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Wed, 22 May 2024 23:29:29 +0800 Subject: [PATCH 188/659] fix: pikpak captcha_required (#6497) * fix: pikpak captcha_required * fix(pikpak_share): video download --- drivers/pikpak/driver.go | 33 +++++++++-- drivers/pikpak/meta.go | 2 + drivers/pikpak/util.go | 101 +++------------------------------ drivers/pikpak_share/driver.go | 36 ++++++++++-- drivers/pikpak_share/meta.go | 10 ++-- drivers/pikpak_share/util.go | 70 ++--------------------- 6 files changed, 81 insertions(+), 171 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index f3676b8291b..e27263ddbb7 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -17,13 +17,14 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" ) type PikPak struct { model.Storage Addition - RefreshToken string - AccessToken string + + oauth2Token oauth2.TokenSource } func (d *PikPak) Config() driver.Config { @@ -34,8 +35,32 @@ func (d *PikPak) GetAddition() driver.Additional { return &d.Addition } -func (d *PikPak) Init(ctx context.Context) error { - return d.login() +func (d *PikPak) Init(ctx context.Context) (err error) { + if d.ClientID == "" || d.ClientSecret == "" { + d.ClientID = "YNxT9w7GMdWvEOKa" + d.ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + } + + withClient := func(ctx context.Context) context.Context { + return context.WithValue(ctx, oauth2.HTTPClient, base.HttpClient) + } + + oauth2Config := &oauth2.Config{ + ClientID: d.ClientID, + ClientSecret: d.ClientSecret, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://user.mypikpak.com/v1/auth/signin", + TokenURL: "https://user.mypikpak.com/v1/auth/token", + AuthStyle: oauth2.AuthStyleInParams, + }, + } + + oauth2Token, err := oauth2Config.PasswordCredentialsToken(withClient(ctx), d.Username, d.Password) + if err != nil { + return err + } + d.oauth2Token = oauth2Config.TokenSource(withClient(context.Background()), oauth2Token) + return nil } func (d *PikPak) Drop(ctx context.Context) error { diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index 3512c44b93b..c462ed13d6b 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -9,6 +9,8 @@ type Addition struct { driver.RootID Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` + ClientID string `json:"client_id" required:"true" default:"YNxT9w7GMdWvEOKa"` + ClientSecret string `json:"client_secret" required:"true" default:"dbw2OtmVEeuUvIptb1Coyg"` DisableMediaLink bool `json:"disable_media_link"` } diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 71ad1dca8a3..0edfc384eba 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -1,78 +1,24 @@ package pikpak import ( - "crypto/sha1" - "encoding/hex" "errors" - "github.com/alist-org/alist/v3/pkg/utils" - "io" "net/http" "github.com/alist-org/alist/v3/drivers/base" - "github.com/alist-org/alist/v3/internal/op" "github.com/go-resty/resty/v2" - jsoniter "github.com/json-iterator/go" ) // do others that not defined in Driver interface -func (d *PikPak) login() error { - url := "https://user.mypikpak.com/v1/auth/signin" - var e RespErr - res, err := base.RestyClient.R().SetError(&e).SetBody(base.Json{ - "captcha_token": "", - "client_id": "YNxT9w7GMdWvEOKa", - "client_secret": "dbw2OtmVEeuUvIptb1Coyg", - "username": d.Username, - "password": d.Password, - }).Post(url) - if err != nil { - return err - } - if e.ErrorCode != 0 { - return errors.New(e.Error) - } - data := res.Body() - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() - return nil -} +func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() -func (d *PikPak) refreshToken() error { - url := "https://user.mypikpak.com/v1/auth/token" - var e RespErr - res, err := base.RestyClient.R().SetError(&e). - SetHeader("user-agent", "").SetBody(base.Json{ - "client_id": "YNxT9w7GMdWvEOKa", - "client_secret": "dbw2OtmVEeuUvIptb1Coyg", - "grant_type": "refresh_token", - "refresh_token": d.RefreshToken, - }).Post(url) + token, err := d.oauth2Token.Token() if err != nil { - d.Status = err.Error() - op.MustSaveDriverStorage(d) - return err - } - if e.ErrorCode != 0 { - if e.ErrorCode == 4126 { - // refresh_token invalid, re-login - return d.login() - } - d.Status = e.Error - op.MustSaveDriverStorage(d) - return errors.New(e.Error) + return nil, err } - data := res.Body() - d.Status = "work" - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() - op.MustSaveDriverStorage(d) - return nil -} + req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) -func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - req := base.RestyClient.R() - req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { callback(req) } @@ -85,17 +31,9 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r if err != nil { return nil, err } + if e.ErrorCode != 0 { - if e.ErrorCode == 16 { - // login / refresh token - err = d.refreshToken() - if err != nil { - return nil, err - } - return d.request(url, method, callback, resp) - } else { - return nil, errors.New(e.Error) - } + return nil, errors.New(e.Error) } return res.Body(), nil } @@ -127,28 +65,3 @@ func (d *PikPak) getFiles(id string) ([]File, error) { } return res, nil } - -func getGcid(r io.Reader, size int64) (string, error) { - calcBlockSize := func(j int64) int64 { - var psize int64 = 0x40000 - for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { - psize = psize << 1 - } - return psize - } - - hash1 := sha1.New() - hash2 := sha1.New() - readSize := calcBlockSize(size) - for { - hash2.Reset() - if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { - if err != io.EOF { - return "", err - } - break - } - hash1.Write(hash2.Sum(nil)) - } - return hex.EncodeToString(hash1.Sum(nil)), nil -} diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index 3003ff48860..58c2c8c4b55 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -4,17 +4,18 @@ import ( "context" "net/http" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + "golang.org/x/oauth2" ) type PikPakShare struct { model.Storage Addition - RefreshToken string - AccessToken string + oauth2Token oauth2.TokenSource PassCodeToken string } @@ -27,10 +28,31 @@ func (d *PikPakShare) GetAddition() driver.Additional { } func (d *PikPakShare) Init(ctx context.Context) error { - err := d.login() + if d.ClientID == "" || d.ClientSecret == "" { + d.ClientID = "YNxT9w7GMdWvEOKa" + d.ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + } + + withClient := func(ctx context.Context) context.Context { + return context.WithValue(ctx, oauth2.HTTPClient, base.HttpClient) + } + + oauth2Config := &oauth2.Config{ + ClientID: d.ClientID, + ClientSecret: d.ClientSecret, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://user.mypikpak.com/v1/auth/signin", + TokenURL: "https://user.mypikpak.com/v1/auth/token", + AuthStyle: oauth2.AuthStyleInParams, + }, + } + + oauth2Token, err := oauth2Config.PasswordCredentialsToken(withClient(ctx), d.Username, d.Password) if err != nil { return err } + d.oauth2Token = oauth2Config.TokenSource(withClient(context.Background()), oauth2Token) + if d.SharePwd != "" { err = d.getSharePassToken() if err != nil { @@ -67,8 +89,14 @@ func (d *PikPakShare) Link(ctx context.Context, file model.Obj, args model.LinkA if err != nil { return nil, err } + + downloadUrl := resp.FileInfo.WebContentLink + if downloadUrl == "" && len(resp.FileInfo.Medias) > 0 { + downloadUrl = resp.FileInfo.Medias[0].Link.Url + } + link := model.Link{ - URL: resp.FileInfo.WebContentLink, + URL: downloadUrl, } return &link, nil } diff --git a/drivers/pikpak_share/meta.go b/drivers/pikpak_share/meta.go index bf77e22b3cf..5d05badb590 100644 --- a/drivers/pikpak_share/meta.go +++ b/drivers/pikpak_share/meta.go @@ -7,10 +7,12 @@ import ( type Addition struct { driver.RootID - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - ShareId string `json:"share_id" required:"true"` - SharePwd string `json:"share_pwd"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + ShareId string `json:"share_id" required:"true"` + SharePwd string `json:"share_pwd"` + ClientID string `json:"client_id" required:"true" default:"YNxT9w7GMdWvEOKa"` + ClientSecret string `json:"client_secret" required:"true" default:"dbw2OtmVEeuUvIptb1Coyg"` } var config = driver.Config{ diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index e62f17842c6..41bb30d4aa6 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -5,70 +5,18 @@ import ( "net/http" "github.com/alist-org/alist/v3/drivers/base" - "github.com/alist-org/alist/v3/internal/op" "github.com/go-resty/resty/v2" - jsoniter "github.com/json-iterator/go" ) -// do others that not defined in Driver interface - -func (d *PikPakShare) login() error { - url := "https://user.mypikpak.com/v1/auth/signin" - var e RespErr - res, err := base.RestyClient.R().SetError(&e).SetBody(base.Json{ - "captcha_token": "", - "client_id": "YNxT9w7GMdWvEOKa", - "client_secret": "dbw2OtmVEeuUvIptb1Coyg", - "username": d.Username, - "password": d.Password, - }).Post(url) - if err != nil { - return err - } - if e.ErrorCode != 0 { - return errors.New(e.Error) - } - data := res.Body() - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() - return nil -} +func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() -func (d *PikPakShare) refreshToken() error { - url := "https://user.mypikpak.com/v1/auth/token" - var e RespErr - res, err := base.RestyClient.R().SetError(&e). - SetHeader("user-agent", "").SetBody(base.Json{ - "client_id": "YNxT9w7GMdWvEOKa", - "client_secret": "dbw2OtmVEeuUvIptb1Coyg", - "grant_type": "refresh_token", - "refresh_token": d.RefreshToken, - }).Post(url) + token, err := d.oauth2Token.Token() if err != nil { - d.Status = err.Error() - op.MustSaveDriverStorage(d) - return err - } - if e.ErrorCode != 0 { - if e.ErrorCode == 4126 { - // refresh_token invalid, re-login - return d.login() - } - d.Status = e.Error - op.MustSaveDriverStorage(d) - return errors.New(e.Error) + return nil, err } - data := res.Body() - d.Status = "work" - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() - op.MustSaveDriverStorage(d) - return nil -} + req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) -func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - req := base.RestyClient.R() - req.SetHeader("Authorization", "Bearer "+d.AccessToken) if callback != nil { callback(req) } @@ -82,14 +30,6 @@ func (d *PikPakShare) request(url string, method string, callback base.ReqCallba return nil, err } if e.ErrorCode != 0 { - if e.ErrorCode == 16 { - // login / refresh token - err = d.refreshToken() - if err != nil { - return nil, err - } - return d.request(url, method, callback, resp) - } return nil, errors.New(e.Error) } return res.Body(), nil From 5f60b51cf8abd48265904b82cb0a7ff0f21a099f Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Wed, 22 May 2024 23:31:42 +0800 Subject: [PATCH 189/659] feat: add `proxy_range` option for `139Yun` `Alias` `AList V3` (#6496) --- drivers/139/meta.go | 9 ++++++--- drivers/alias/meta.go | 11 ++++++----- drivers/alias/util.go | 8 ++++++-- drivers/alist_v3/meta.go | 9 +++++---- internal/driver/config.go | 1 + internal/model/storage.go | 1 + internal/op/driver.go | 11 +++++++++++ server/common/proxy.go | 20 ++++++++++++++++++++ server/handles/down.go | 3 +++ server/webdav/webdav.go | 3 +++ 10 files changed, 62 insertions(+), 14 deletions(-) diff --git a/drivers/139/meta.go b/drivers/139/meta.go index 416e63a796c..56a4c1df96b 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -14,12 +14,15 @@ type Addition struct { } var config = driver.Config{ - Name: "139Yun", - LocalSort: true, + Name: "139Yun", + LocalSort: true, + ProxyRangeOption: true, } func init() { op.RegisterDriver(func() driver.Driver { - return &Yun139{} + d := &Yun139{} + d.ProxyRange = true + return d }) } diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 2bebe7b00a8..45b885753d0 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -14,11 +14,12 @@ type Addition struct { } var config = driver.Config{ - Name: "Alias", - LocalSort: true, - NoCache: true, - NoUpload: true, - DefaultRoot: "/", + Name: "Alias", + LocalSort: true, + NoCache: true, + NoUpload: true, + DefaultRoot: "/", + ProxyRangeOption: true, } func init() { diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 803bd0736b9..ba1f7e72649 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -103,12 +103,16 @@ func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) return nil, err } if common.ShouldProxy(storage, stdpath.Base(sub)) { - return &model.Link{ + link := &model.Link{ URL: fmt.Sprintf("%s/p%s?sign=%s", common.GetApiUrl(args.HttpReq), utils.EncodePath(reqPath, true), sign.Sign(reqPath)), - }, nil + } + if args.HttpReq != nil && d.ProxyRange { + link.RangeReadCloser = common.NoProxyRange + } + return link, nil } link, _, err := fs.Link(ctx, reqPath, args) return link, err diff --git a/drivers/alist_v3/meta.go b/drivers/alist_v3/meta.go index c04c737b149..cc5f2189395 100644 --- a/drivers/alist_v3/meta.go +++ b/drivers/alist_v3/meta.go @@ -16,10 +16,11 @@ type Addition struct { } var config = driver.Config{ - Name: "AList V3", - LocalSort: true, - DefaultRoot: "/", - CheckStatus: true, + Name: "AList V3", + LocalSort: true, + DefaultRoot: "/", + CheckStatus: true, + ProxyRangeOption: true, } func init() { diff --git a/internal/driver/config.go b/internal/driver/config.go index c9e3f949af0..6068143cb71 100644 --- a/internal/driver/config.go +++ b/internal/driver/config.go @@ -12,6 +12,7 @@ type Config struct { CheckStatus bool `json:"-"` Alert string `json:"alert"` //info,success,warning,danger NoOverwriteUpload bool `json:"-"` // whether to support overwrite upload + ProxyRangeOption bool `json:"-"` } func (c Config) MustProxy() bool { diff --git a/internal/model/storage.go b/internal/model/storage.go index 1045a00767b..14bcf45f6e2 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -27,6 +27,7 @@ type Sort struct { type Proxy struct { WebProxy bool `json:"web_proxy"` WebdavPolicy string `json:"webdav_policy"` + ProxyRange bool `json:"proxy_range"` DownProxyUrl string `json:"down_proxy_url"` } diff --git a/internal/op/driver.go b/internal/op/driver.go index 66d1ae3ce9c..4f10e8e23c6 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -93,6 +93,17 @@ func getMainItems(config driver.Config) []driver.Item { Required: true, }, }...) + if config.ProxyRangeOption { + item := driver.Item{ + Name: "proxy_range", + Type: conf.TypeBool, + Help: "Need to enable proxy", + } + if config.Name == "139Yun" { + item.Default = "true" + } + items = append(items, item) + } } else { items = append(items, driver.Item{ Name: "webdav_policy", diff --git a/server/common/proxy.go b/server/common/proxy.go index 4ca4ba7f6ab..10923613ede 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -9,8 +9,10 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" + log "github.com/sirupsen/logrus" ) func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { @@ -82,3 +84,21 @@ func attachFileName(w http.ResponseWriter, file model.Obj) { w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, fileName, url.PathEscape(fileName))) w.Header().Set("Content-Type", utils.GetMimeType(fileName)) } + +var NoProxyRange = &model.RangeReadCloser{} + +func ProxyRange(link *model.Link, size int64) { + if link.MFile != nil { + return + } + if link.RangeReadCloser == nil { + var rrc, err = stream.GetRangeReadCloserFromLink(size, link) + if err != nil { + log.Warnf("ProxyRange error: %s", err) + return + } + link.RangeReadCloser = rrc + } else if link.RangeReadCloser == NoProxyRange { + link.RangeReadCloser = nil + } +} diff --git a/server/handles/down.go b/server/handles/down.go index d3d41e85a2b..0020ed1453e 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -106,6 +106,9 @@ func Proxy(c *gin.Context) { return } } + if storage.GetStorage().ProxyRange { + common.ProxyRange(link, file.GetSize()) + } err = common.Proxy(c.Writer, c.Request, link, file) if err != nil { common.ErrorResp(c, err, 500, true) diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 6054991a0c2..b84e65b06b7 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -247,6 +247,9 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta if err != nil { return http.StatusInternalServerError, err } + if storage.GetStorage().ProxyRange { + common.ProxyRange(link, fi.GetSize()) + } err = common.Proxy(w, r, link, fi) if err != nil { log.Errorf("webdav proxy error: %+v", err) From 85d743c5d26236d4f48ee1a909083364973d03cb Mon Sep 17 00:00:00 2001 From: WintBit Date: Wed, 22 May 2024 23:31:58 +0800 Subject: [PATCH 190/659] feat: add support for lark driver (#6475) * feat: lark storage driver * feat: external view mode * limit lark targets * fix: missing package --------- Co-authored-by: Andy Hsu --- drivers/lark.go | 8 + drivers/lark/driver.go | 396 +++++++++++++++++++++++++++++++++++++++++ drivers/lark/meta.go | 36 ++++ drivers/lark/types.go | 32 ++++ drivers/lark/util.go | 66 +++++++ go.mod | 3 +- go.sum | 18 ++ 7 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 drivers/lark.go create mode 100644 drivers/lark/driver.go create mode 100644 drivers/lark/meta.go create mode 100644 drivers/lark/types.go create mode 100644 drivers/lark/util.go diff --git a/drivers/lark.go b/drivers/lark.go new file mode 100644 index 00000000000..0fbbaddc09b --- /dev/null +++ b/drivers/lark.go @@ -0,0 +1,8 @@ +// +build linux darwin +// +build amd64 arm64 + +package drivers + +import ( + _ "github.com/alist-org/alist/v3/drivers/lark" +) \ No newline at end of file diff --git a/drivers/lark/driver.go b/drivers/lark/driver.go new file mode 100644 index 00000000000..6783aa5241d --- /dev/null +++ b/drivers/lark/driver.go @@ -0,0 +1,396 @@ +package lark + +import ( + "context" + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/ipfs/boxo/path" + lark "github.com/larksuite/oapi-sdk-go/v3" + larkcore "github.com/larksuite/oapi-sdk-go/v3/core" + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + "golang.org/x/time/rate" + "io" + "net/http" + "strconv" + "time" +) + +type Lark struct { + model.Storage + Addition + + client *lark.Client + rootFolderToken string +} + +func (c *Lark) Config() driver.Config { + return config +} + +func (c *Lark) GetAddition() driver.Additional { + return &c.Addition +} + +func (c *Lark) Init(ctx context.Context) error { + c.client = lark.NewClient(c.AppId, c.AppSecret, lark.WithTokenCache(newTokenCache())) + + paths := path.SplitList(c.RootFolderPath) + token := "" + + var ok bool + var file *larkdrive.File + for _, p := range paths { + if p == "" { + token = "" + continue + } + + resp, err := c.client.Drive.File.ListByIterator(ctx, larkdrive.NewListFileReqBuilder().FolderToken(token).Build()) + if err != nil { + return err + } + + for { + ok, file, err = resp.Next() + if !ok { + return errs.ObjectNotFound + } + + if err != nil { + return err + } + + if *file.Type == "folder" && *file.Name == p { + token = *file.Token + break + } + } + } + + c.rootFolderToken = token + + return nil +} + +func (c *Lark) Drop(ctx context.Context) error { + return nil +} + +func (c *Lark) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + token, ok := c.getObjToken(ctx, dir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + if token == emptyFolderToken { + return nil, nil + } + + resp, err := c.client.Drive.File.ListByIterator(ctx, larkdrive.NewListFileReqBuilder().FolderToken(token).Build()) + if err != nil { + return nil, err + } + + ok = false + var file *larkdrive.File + var res []model.Obj + + for { + ok, file, err = resp.Next() + if !ok { + break + } + + if err != nil { + return nil, err + } + + modifiedUnix, _ := strconv.ParseInt(*file.ModifiedTime, 10, 64) + createdUnix, _ := strconv.ParseInt(*file.CreatedTime, 10, 64) + + f := model.Object{ + ID: *file.Token, + Path: path.Join([]string{c.RootFolderPath, dir.GetPath(), *file.Name}), + Name: *file.Name, + Size: 0, + Modified: time.Unix(modifiedUnix, 0), + Ctime: time.Unix(createdUnix, 0), + IsFolder: *file.Type == "folder", + } + res = append(res, &f) + } + + return res, nil +} + +func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + token, ok := c.getObjToken(ctx, file.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + resp, err := c.client.GetTenantAccessTokenBySelfBuiltApp(ctx, &larkcore.SelfBuiltTenantAccessTokenReq{ + AppID: c.AppId, + AppSecret: c.AppSecret, + }) + + if err != nil { + return nil, err + } + + if !c.ExternalMode { + accessToken := resp.TenantAccessToken + + url := fmt.Sprintf("https://open.feishu.cn/open-apis/drive/v1/files/%s/download", token) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + req.Header.Set("Range", "bytes=0-1") + + ar, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + if ar.StatusCode != http.StatusPartialContent { + return nil, errors.New("failed to get download link") + } + + return &model.Link{ + URL: url, + Header: http.Header{ + "Authorization": []string{fmt.Sprintf("Bearer %s", accessToken)}, + }, + }, nil + } else { + url := path.Join([]string{c.TenantUrlPrefix, "file", token}) + + return &model.Link{ + URL: url, + }, nil + } +} + +func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + token, ok := c.getObjToken(ctx, parentDir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + body, err := larkdrive.NewCreateFolderFilePathReqBodyBuilder().FolderToken(token).Name(dirName).Build() + if err != nil { + return nil, err + } + + resp, err := c.client.Drive.File.CreateFolder(ctx, + larkdrive.NewCreateFolderFileReqBuilder().Body(body).Build()) + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + return &model.Object{ + ID: *resp.Data.Token, + Path: path.Join([]string{c.RootFolderPath, parentDir.GetPath(), dirName}), + Name: dirName, + Size: 0, + IsFolder: true, + }, nil +} + +func (c *Lark) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcToken, ok := c.getObjToken(ctx, srcObj.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + dstDirToken, ok := c.getObjToken(ctx, dstDir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + req := larkdrive.NewMoveFileReqBuilder(). + Body(larkdrive.NewMoveFileReqBodyBuilder(). + Type("file"). + FolderToken(dstDirToken). + Build()).FileToken(srcToken). + Build() + + // 发起请求 + resp, err := c.client.Drive.File.Move(ctx, req) + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + return nil, nil +} + +func (c *Lark) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + // TODO rename obj, optional + return nil, errs.NotImplement +} + +func (c *Lark) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcToken, ok := c.getObjToken(ctx, srcObj.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + dstDirToken, ok := c.getObjToken(ctx, dstDir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + req := larkdrive.NewCopyFileReqBuilder(). + Body(larkdrive.NewCopyFileReqBodyBuilder(). + Name(srcObj.GetName()). + Type("file"). + FolderToken(dstDirToken). + Build()).FileToken(srcToken). + Build() + + // 发起请求 + resp, err := c.client.Drive.File.Copy(ctx, req) + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + return nil, nil +} + +func (c *Lark) Remove(ctx context.Context, obj model.Obj) error { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return errs.ObjectNotFound + } + + req := larkdrive.NewDeleteFileReqBuilder(). + FileToken(token). + Type("file"). + Build() + + // 发起请求 + resp, err := c.client.Drive.File.Delete(ctx, req) + if err != nil { + return err + } + + if !resp.Success() { + return errors.New(resp.Error()) + } + + return nil +} + +var uploadLimit = rate.NewLimiter(rate.Every(time.Second), 5) + +func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + token, ok := c.getObjToken(ctx, dstDir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + // prepare + req := larkdrive.NewUploadPrepareFileReqBuilder(). + FileUploadInfo(larkdrive.NewFileUploadInfoBuilder(). + FileName(stream.GetName()). + ParentType(`explorer`). + ParentNode(token). + Size(int(stream.GetSize())). + Build()). + Build() + + // 发起请求 + uploadLimit.Wait(ctx) + resp, err := c.client.Drive.File.UploadPrepare(ctx, req) + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + uploadId := *resp.Data.UploadId + blockSize := *resp.Data.BlockSize + blockCount := *resp.Data.BlockNum + + // upload + for i := 0; i < blockCount; i++ { + length := int64(blockSize) + if i == blockCount-1 { + length = stream.GetSize() - int64(i*blockSize) + } + + reader := io.LimitReader(stream, length) + + req := larkdrive.NewUploadPartFileReqBuilder(). + Body(larkdrive.NewUploadPartFileReqBodyBuilder(). + UploadId(uploadId). + Seq(i). + Size(int(length)). + File(reader). + Build()). + Build() + + // 发起请求 + uploadLimit.Wait(ctx) + resp, err := c.client.Drive.File.UploadPart(ctx, req) + + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + up(float64(i) / float64(blockCount)) + } + + //close + closeReq := larkdrive.NewUploadFinishFileReqBuilder(). + Body(larkdrive.NewUploadFinishFileReqBodyBuilder(). + UploadId(uploadId). + BlockNum(blockCount). + Build()). + Build() + + // 发起请求 + closeResp, err := c.client.Drive.File.UploadFinish(ctx, closeReq) + if err != nil { + return nil, err + } + + if !closeResp.Success() { + return nil, errors.New(closeResp.Error()) + } + + return &model.Object{ + ID: *closeResp.Data.FileToken, + }, nil +} + +//func (d *Lark) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Lark)(nil) diff --git a/drivers/lark/meta.go b/drivers/lark/meta.go new file mode 100644 index 00000000000..221345e222c --- /dev/null +++ b/drivers/lark/meta.go @@ -0,0 +1,36 @@ +package lark + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // define other + AppId string `json:"app_id" type:"text" help:"app id"` + AppSecret string `json:"app_secret" type:"text" help:"app secret"` + ExternalMode bool `json:"external_mode" type:"bool" help:"external mode"` + TenantUrlPrefix string `json:"tenant_url_prefix" type:"text" help:"tenant url prefix"` +} + +var config = driver.Config{ + Name: "Lark", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Lark{} + }) +} diff --git a/drivers/lark/types.go b/drivers/lark/types.go new file mode 100644 index 00000000000..3ebefd556dc --- /dev/null +++ b/drivers/lark/types.go @@ -0,0 +1,32 @@ +package lark + +import ( + "context" + "github.com/Xhofe/go-cache" + "time" +) + +type TokenCache struct { + cache.ICache[string] +} + +func (t *TokenCache) Set(_ context.Context, key string, value string, expireTime time.Duration) error { + t.ICache.Set(key, value, cache.WithEx[string](expireTime)) + + return nil +} + +func (t *TokenCache) Get(_ context.Context, key string) (string, error) { + v, ok := t.ICache.Get(key) + if ok { + return v, nil + } + + return "", nil +} + +func newTokenCache() *TokenCache { + c := cache.NewMemCache[string]() + + return &TokenCache{c} +} diff --git a/drivers/lark/util.go b/drivers/lark/util.go new file mode 100644 index 00000000000..8c6828bd176 --- /dev/null +++ b/drivers/lark/util.go @@ -0,0 +1,66 @@ +package lark + +import ( + "context" + "github.com/Xhofe/go-cache" + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + log "github.com/sirupsen/logrus" + "path" + "time" +) + +const objTokenCacheDuration = 5 * time.Minute +const emptyFolderToken = "empty" + +var objTokenCache = cache.NewMemCache[string]() +var exOpts = cache.WithEx[string](objTokenCacheDuration) + +func (c *Lark) getObjToken(ctx context.Context, folderPath string) (string, bool) { + if token, ok := objTokenCache.Get(folderPath); ok { + return token, true + } + + dir, name := path.Split(folderPath) + // strip the last slash of dir if it exists + if len(dir) > 0 && dir[len(dir)-1] == '/' { + dir = dir[:len(dir)-1] + } + if name == "" { + return c.rootFolderToken, true + } + + var parentToken string + var found bool + parentToken, found = c.getObjToken(ctx, dir) + if !found { + return emptyFolderToken, false + } + + req := larkdrive.NewListFileReqBuilder().FolderToken(parentToken).Build() + resp, err := c.client.Drive.File.ListByIterator(ctx, req) + + if err != nil { + log.WithError(err).Error("failed to list files") + return emptyFolderToken, false + } + + var file *larkdrive.File + for { + found, file, err = resp.Next() + if !found { + break + } + + if err != nil { + log.WithError(err).Error("failed to get next file") + break + } + + if *file.Name == name { + objTokenCache.Set(folderPath, *file.Token, exOpts) + return *file.Token, true + } + } + + return emptyFolderToken, false +} diff --git a/go.mod b/go.mod index a994d7688f7..118c5a35001 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/charmbracelet/lipgloss v0.9.1 github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.6.0 + github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 github.com/disintegration/imaging v1.6.2 github.com/djherbis/times v1.6.0 github.com/dlclark/regexp2 v1.10.0 @@ -36,6 +37,7 @@ require ( github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 + github.com/larksuite/oapi-sdk-go/v3 v3.2.5 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.26.1 github.com/minio/sio v0.3.0 @@ -108,7 +110,6 @@ require ( github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/geoffgarside/ber v1.1.0 // indirect diff --git a/go.sum b/go.sum index 539151e660a..d8bc53dbb9a 100644 --- a/go.sum +++ b/go.sum @@ -184,6 +184,7 @@ github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGF github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= @@ -215,6 +216,7 @@ github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvki github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -261,6 +263,7 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= @@ -281,6 +284,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/larksuite/oapi-sdk-go/v3 v3.2.5 h1:MkmkfCHzvmi35EId9SeFPJMZ8bUsijnxwneAWHnnk0k= +github.com/larksuite/oapi-sdk-go/v3 v3.2.5/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= @@ -483,6 +488,8 @@ github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3K github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.1 h1:O5QY4cVjIGELx3UGh6LbVAc18MWGXgRNQjMt72x6w/8= github.com/xhofe/tache v0.1.1/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= @@ -497,6 +504,7 @@ golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -514,10 +522,14 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -534,6 +546,8 @@ golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= @@ -596,12 +610,16 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= From d781f7127a356ac389512ce1a70ae00f2dcd9023 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 23 May 2024 11:52:37 +0800 Subject: [PATCH 191/659] fix: add lark to windows target --- drivers/lark.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/lark.go b/drivers/lark.go index 0fbbaddc09b..d5070078651 100644 --- a/drivers/lark.go +++ b/drivers/lark.go @@ -1,8 +1,8 @@ -// +build linux darwin +// +build linux darwin windows // +build amd64 arm64 package drivers import ( _ "github.com/alist-org/alist/v3/drivers/lark" -) \ No newline at end of file +) From 0a8d710e01cdf161d4bf6d52a869ff0a2259885e Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Thu, 23 May 2024 18:56:17 +0800 Subject: [PATCH 192/659] fix(mopan): upgrade version (#6500) --- go.mod | 3 ++- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 118c5a35001..9dd012c9667 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/djherbis/times v1.6.0 github.com/dlclark/regexp2 v1.10.0 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 - github.com/foxxorcat/mopan-sdk-go v0.1.5 + github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gaoyb7/115drive-webdav v0.1.8 github.com/gin-contrib/cors v1.5.0 @@ -129,6 +129,7 @@ require ( github.com/google/go-tpm v0.9.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect diff --git a/go.sum b/go.sum index d8bc53dbb9a..c4f6832bcf1 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,8 @@ github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= -github.com/foxxorcat/mopan-sdk-go v0.1.5 h1:N3LqOvk2aWWxszsFIkArP5udIv74uTei/bH2jM3tfSc= -github.com/foxxorcat/mopan-sdk-go v0.1.5/go.mod h1:iWHA2JFhzmKR28ySp1ON0g6DjLaYtvb5jhTqPVTDW9A= +github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= +github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -224,6 +224,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= From 8e2b9c681a47ce0cf75c619000c8bc62cf5968b1 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 23 May 2024 20:05:00 +0800 Subject: [PATCH 193/659] fix(ilanzou): upgrade devVersion --- drivers/ilanzou/driver.go | 10 ++++++++-- drivers/ilanzou/meta.go | 4 ++-- drivers/ilanzou/util.go | 5 +++-- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 1d8e5d36b09..63d86363962 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -147,7 +147,8 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) // get the url after redirect res, err := base.NoRedirectClient.R().SetHeaders(map[string]string{ //"Origin": d.conf.site, - "Referer": d.conf.site + "/", + "Referer": d.conf.site + "/", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", }).Get(realURL) if err != nil { return nil, err @@ -155,7 +156,12 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if res.StatusCode() == 302 { realURL = res.Header().Get("location") } else { - return nil, fmt.Errorf("redirect failed, status: %d", res.StatusCode()) + contentLengthStr := res.Header().Get("Content-Length") + contentLength, err := strconv.Atoi(contentLengthStr) + if err != nil || contentLength == 0 || contentLength > 1024*10 { + return nil, fmt.Errorf("redirect failed, status: %d", res.StatusCode()) + } + return nil, fmt.Errorf("redirect failed, content: %s", res.String()) } link := model.Link{URL: realURL} return &link, nil diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go index ed5b2edb52e..f15fc01a492 100644 --- a/drivers/ilanzou/meta.go +++ b/drivers/ilanzou/meta.go @@ -46,7 +46,7 @@ func init() { bucket: "wpanstore-lanzou", unproved: "unproved", proved: "proved", - devVersion: "122", + devVersion: "125", site: "https://www.ilanzou.com", }, } @@ -72,7 +72,7 @@ func init() { bucket: "wpanstore", unproved: "ws", proved: "app", - devVersion: "121", + devVersion: "125", site: "https://www.feijipan.com", }, } diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index 37e111ad53e..c9a30765b7f 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -59,8 +59,9 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr "extra": "2", }) req.SetHeaders(map[string]string{ - "Origin": d.conf.site, - "Referer": d.conf.site + "/", + "Origin": d.conf.site, + "Referer": d.conf.site + "/", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", }) if proved { req.SetQueryParam("appToken", d.Token) From 163af0515fee7c9ee27dda6e599fbf18fe8dc662 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Mon, 27 May 2024 21:31:59 +0800 Subject: [PATCH 194/659] fix(pikpak): refresh_token contention (#6501 close #6511) --- drivers/pikpak/driver.go | 16 +++++++--------- drivers/pikpak_share/driver.go | 21 ++++++++------------- pkg/utils/oauth2.go | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 22 deletions(-) create mode 100644 pkg/utils/oauth2.go diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index e27263ddbb7..2dab2a9b066 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -41,10 +41,6 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" } - withClient := func(ctx context.Context) context.Context { - return context.WithValue(ctx, oauth2.HTTPClient, base.HttpClient) - } - oauth2Config := &oauth2.Config{ ClientID: d.ClientID, ClientSecret: d.ClientSecret, @@ -55,11 +51,13 @@ func (d *PikPak) Init(ctx context.Context) (err error) { }, } - oauth2Token, err := oauth2Config.PasswordCredentialsToken(withClient(ctx), d.Username, d.Password) - if err != nil { - return err - } - d.oauth2Token = oauth2Config.TokenSource(withClient(context.Background()), oauth2Token) + d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { + return oauth2Config.PasswordCredentialsToken( + context.WithValue(context.Background(), oauth2.HTTPClient, base.HttpClient), + d.Username, + d.Password, + ) + })) return nil } diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index 58c2c8c4b55..1862db06bf4 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -33,10 +33,6 @@ func (d *PikPakShare) Init(ctx context.Context) error { d.ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" } - withClient := func(ctx context.Context) context.Context { - return context.WithValue(ctx, oauth2.HTTPClient, base.HttpClient) - } - oauth2Config := &oauth2.Config{ ClientID: d.ClientID, ClientSecret: d.ClientSecret, @@ -47,17 +43,16 @@ func (d *PikPakShare) Init(ctx context.Context) error { }, } - oauth2Token, err := oauth2Config.PasswordCredentialsToken(withClient(ctx), d.Username, d.Password) - if err != nil { - return err - } - d.oauth2Token = oauth2Config.TokenSource(withClient(context.Background()), oauth2Token) + d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { + return oauth2Config.PasswordCredentialsToken( + context.WithValue(context.Background(), oauth2.HTTPClient, base.HttpClient), + d.Username, + d.Password, + ) + })) if d.SharePwd != "" { - err = d.getSharePassToken() - if err != nil { - return err - } + return d.getSharePassToken() } return nil } diff --git a/pkg/utils/oauth2.go b/pkg/utils/oauth2.go new file mode 100644 index 00000000000..c1ad161245f --- /dev/null +++ b/pkg/utils/oauth2.go @@ -0,0 +1,15 @@ +package utils + +import "golang.org/x/oauth2" + +type tokenSource struct { + fn func() (*oauth2.Token, error) +} + +func (t *tokenSource) Token() (*oauth2.Token, error) { + return t.fn() +} + +func TokenSource(fn func() (*oauth2.Token, error)) oauth2.TokenSource { + return &tokenSource{fn} +} From 639b7817bf6a3d6ac937a281495c36522e9601e3 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Mon, 27 May 2024 21:34:26 +0800 Subject: [PATCH 195/659] feat: add supports for thunder_browser driver (#6529 close #6526) * feat: add supports for thunderX driver * fix: Fix the bug where UserID is not passed correctly * feat: add support for thunder_browser driver --- drivers/all.go | 1 + drivers/thunder_browser/driver.go | 813 ++++++++++++++++++++++++++++++ drivers/thunder_browser/meta.go | 108 ++++ drivers/thunder_browser/types.go | 223 ++++++++ drivers/thunder_browser/util.go | 249 +++++++++ drivers/thunderx/driver.go | 1 + 6 files changed, 1395 insertions(+) create mode 100644 drivers/thunder_browser/driver.go create mode 100644 drivers/thunder_browser/meta.go create mode 100644 drivers/thunder_browser/types.go create mode 100644 drivers/thunder_browser/util.go diff --git a/drivers/all.go b/drivers/all.go index bae72b3175a..df8a1ffc72a 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -46,6 +46,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" + _ "github.com/alist-org/alist/v3/drivers/thunder_browser" _ "github.com/alist-org/alist/v3/drivers/thunderx" _ "github.com/alist-org/alist/v3/drivers/trainbit" _ "github.com/alist-org/alist/v3/drivers/url_tree" diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go new file mode 100644 index 00000000000..f3a08f93d54 --- /dev/null +++ b/drivers/thunder_browser/driver.go @@ -0,0 +1,813 @@ +package thunder_browser + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/go-resty/resty/v2" + "net/http" + "regexp" + "strings" +) + +type ThunderBrowser struct { + *XunLeiBrowserCommon + model.Storage + Addition + + identity string +} + +func (x *ThunderBrowser) Config() driver.Config { + return config +} + +func (x *ThunderBrowser) GetAddition() driver.Additional { + return &x.Addition +} + +func (x *ThunderBrowser) Init(ctx context.Context) (err error) { + + spaceTokenFunc := func() error { + // 如果用户未设置 "超级保险柜" 密码 则直接返回 + if x.SafePassword == "" { + return nil + } + // 通过 GetSafeAccessToken 获取 + token, err := x.GetSafeAccessToken(x.SafePassword) + x.SetSpaceTokenResp(token) + return err + } + + // 初始化所需参数 + if x.XunLeiBrowserCommon == nil { + x.XunLeiBrowserCommon = &XunLeiBrowserCommon{ + Common: &Common{ + client: base.NewRestyClient(), + Algorithms: []string{ + "x+I5XiTByg", + "6QU1x5DqGAV3JKg6h", + "VI1vL1WXr7st0es", + "n+/3yhlrnKs4ewhLgZhZ5ITpt554", + "UOip2PE7BLIEov/ZX6VOnsz", + "Q70h9lpViNCOC8sGVkar9o22LhBTjfP", + "IVHFuB1JcMlaZHnW", + "bKE", + "HZRbwxOiQx+diNopi6Nu", + "fwyasXgYL3rP314331b", + "LWxXAiSW4", + "UlWIjv1HGrC6Ngmt4Nohx", + "FOa+Lc0bxTDpTwIh2", + "0+RY", + "xmRVMqokHHpvsiH0", + }, + DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), + ClientID: "ZUBzD9J_XPXfn7f7", + ClientSecret: "yESVmHecEe6F0aou69vl-g", + ClientVersion: "1.0.7.1938", + PackageName: "com.xunlei.browser", + UserAgent: "ANDROID-com.xunlei.browser/1.0.7.1938 netWorkType/5G appid/22062 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/233100 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", + DownloadUserAgent: "AndroidDownloadManager/12 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", + UseVideoUrl: x.UseVideoUrl, + + refreshCTokenCk: func(token string) { + x.CaptchaToken = token + op.MustSaveDriverStorage(x) + }, + }, + refreshTokenFunc: func() error { + // 通过RefreshToken刷新 + token, err := x.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + // 重新登录 + token, err = x.Login(x.Username, x.Password) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + op.MustSaveDriverStorage(x) + } + } + x.SetTokenResp(token) + return err + }, + } + } + + // 自定义验证码token + ctoekn := strings.TrimSpace(x.CaptchaToken) + if ctoekn != "" { + x.SetCaptchaToken(ctoekn) + } + x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl + x.Addition.RootFolderID = x.RootFolderID + // 防止重复登录 + identity := x.GetIdentity() + if x.identity != identity || !x.IsLogin() { + x.identity = identity + // 登录 + token, err := x.Login(x.Username, x.Password) + if err != nil { + return err + } + x.SetTokenResp(token) + } + + // 获取 spaceToken + err = spaceTokenFunc() + if err != nil { + return err + } + + return nil +} + +func (x *ThunderBrowser) Drop(ctx context.Context) error { + return nil +} + +type ThunderBrowserExpert struct { + *XunLeiBrowserCommon + model.Storage + ExpertAddition + + identity string +} + +func (x *ThunderBrowserExpert) Config() driver.Config { + return configExpert +} + +func (x *ThunderBrowserExpert) GetAddition() driver.Additional { + return &x.ExpertAddition +} + +func (x *ThunderBrowserExpert) Init(ctx context.Context) (err error) { + + spaceTokenFunc := func() error { + // 如果用户未设置 "超级保险柜" 密码 则直接返回 + if x.SafePassword == "" { + return nil + } + // 通过 GetSafeAccessToken 获取 + token, err := x.GetSafeAccessToken(x.SafePassword) + x.SetSpaceTokenResp(token) + return err + } + + // 防止重复登录 + identity := x.GetIdentity() + if identity != x.identity || !x.IsLogin() { + x.identity = identity + x.XunLeiBrowserCommon = &XunLeiBrowserCommon{ + Common: &Common{ + client: base.NewRestyClient(), + + DeviceID: func() string { + if len(x.DeviceID) != 32 { + return utils.GetMD5EncodeStr(x.DeviceID) + } + return x.DeviceID + }(), + ClientID: x.ClientID, + ClientSecret: x.ClientSecret, + ClientVersion: x.ClientVersion, + PackageName: x.PackageName, + UserAgent: x.UserAgent, + DownloadUserAgent: x.DownloadUserAgent, + UseVideoUrl: x.UseVideoUrl, + + refreshCTokenCk: func(token string) { + x.CaptchaToken = token + op.MustSaveDriverStorage(x) + }, + }, + } + + if x.CaptchaToken != "" { + x.SetCaptchaToken(x.CaptchaToken) + } + x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl + x.ExpertAddition.RootFolderID = x.RootFolderID + // 签名方法 + if x.SignType == "captcha_sign" { + x.Common.Timestamp = x.Timestamp + x.Common.CaptchaSign = x.CaptchaSign + } else { + x.Common.Algorithms = strings.Split(x.Algorithms, ",") + } + + // 登录方式 + if x.LoginType == "refresh_token" { + // 通过RefreshToken登录 + token, err := x.XunLeiBrowserCommon.RefreshToken(x.ExpertAddition.RefreshToken) + if err != nil { + return err + } + x.SetTokenResp(token) + + // 刷新token方法 + x.SetRefreshTokenFunc(func() error { + token, err := x.XunLeiBrowserCommon.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + } + x.SetTokenResp(token) + op.MustSaveDriverStorage(x) + return err + }) + + err = spaceTokenFunc() + if err != nil { + return err + } + + } else { + // 通过用户密码登录 + token, err := x.Login(x.Username, x.Password) + if err != nil { + return err + } + x.SetTokenResp(token) + x.SetRefreshTokenFunc(func() error { + token, err := x.XunLeiBrowserCommon.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + token, err = x.Login(x.Username, x.Password) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + } + } + x.SetTokenResp(token) + op.MustSaveDriverStorage(x) + return err + }) + + err = spaceTokenFunc() + if err != nil { + return err + } + } + } else { + // 仅修改验证码token + if x.CaptchaToken != "" { + x.SetCaptchaToken(x.CaptchaToken) + } + + err = spaceTokenFunc() + if err != nil { + return err + } + + x.XunLeiBrowserCommon.UserAgent = x.UserAgent + x.XunLeiBrowserCommon.DownloadUserAgent = x.DownloadUserAgent + x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl + x.ExpertAddition.RootFolderID = x.RootFolderID + } + + return nil +} + +func (x *ThunderBrowserExpert) Drop(ctx context.Context) error { + return nil +} + +func (x *ThunderBrowserExpert) SetTokenResp(token *TokenResp) { + x.XunLeiBrowserCommon.SetTokenResp(token) + if token != nil { + x.ExpertAddition.RefreshToken = token.RefreshToken + } +} + +type XunLeiBrowserCommon struct { + *Common + *TokenResp // 登录信息 + + refreshTokenFunc func() error +} + +func (xc *XunLeiBrowserCommon) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return xc.getFiles(ctx, dir.GetID(), args.ReqPath) +} + +func (xc *XunLeiBrowserCommon) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var lFile Files + + params := map[string]string{ + "_magic": "2021", + "space": "SPACE_BROWSER", + "thumbnail_size": "SIZE_LARGE", + "with": "url", + } + // 对 "迅雷云盘" 内的文件 特殊处理 + if file.GetPath() == ThunderDriveFileID { + params = map[string]string{} + } else if file.GetPath() == ThunderBrowserDriveSafeFileID { + // 对 "超级保险箱" 内的文件 特殊处理 + params["space"] = "SPACE_BROWSER_SAFE" + } + + _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", file.GetID()) + r.SetQueryParams(params) + //r.SetQueryParam("space", "") + }, &lFile) + if err != nil { + return nil, err + } + link := &model.Link{ + URL: lFile.WebContentLink, + Header: http.Header{ + "User-Agent": {xc.DownloadUserAgent}, + }, + } + + if xc.UseVideoUrl { + for _, media := range lFile.Medias { + if media.Link.URL != "" { + link.URL = media.Link.URL + break + } + } + } + return link, nil +} + +func (xc *XunLeiBrowserCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + js := base.Json{ + "kind": FOLDER, + "name": dirName, + "parent_id": parentDir.GetID(), + "space": "SPACE_BROWSER", + } + if parentDir.GetPath() == ThunderDriveFileID { + js = base.Json{ + "kind": FOLDER, + "name": dirName, + "parent_id": parentDir.GetID(), + } + } else if parentDir.GetPath() == ThunderBrowserDriveSafeFileID { + js = base.Json{ + "kind": FOLDER, + "name": dirName, + "parent_id": parentDir.GetID(), + "space": "SPACE_BROWSER_SAFE", + } + } + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err +} + +func (xc *XunLeiBrowserCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + + srcSpace := "SPACE_BROWSER" + dstSpace := "SPACE_BROWSER" + + // 对 "超级保险箱" 内的文件 特殊处理 + if srcObj.GetPath() == ThunderBrowserDriveSafeFileID { + srcSpace = "SPACE_BROWSER_SAFE" + } + if dstDir.GetPath() == ThunderBrowserDriveSafeFileID { + dstSpace = "SPACE_BROWSER_SAFE" + } + + params := map[string]string{ + "_from": dstSpace, + } + js := base.Json{ + "to": base.Json{"parent_id": dstDir.GetID(), "space": dstSpace}, + "space": srcSpace, + "ids": []string{srcObj.GetID()}, + } + // 对 "迅雷云盘" 内的文件 特殊处理 + if srcObj.GetPath() == ThunderDriveFileID { + params = map[string]string{} + js = base.Json{ + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + } + } + + _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + r.SetQueryParams(params) + }, nil) + return err +} + +func (xc *XunLeiBrowserCommon) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + + params := map[string]string{ + "space": "SPACE_BROWSER", + } + // 对 "迅雷云盘" 内的文件 特殊处理 + if srcObj.GetPath() == ThunderDriveFileID { + params = map[string]string{} + } else if srcObj.GetPath() == ThunderBrowserDriveSafeFileID { + // 对 "超级保险箱" 内的文件 特殊处理 + params = map[string]string{ + "space": "SPACE_BROWSER_SAFE", + } + } + + _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", srcObj.GetID()) + r.SetBody(&base.Json{"name": newName}) + r.SetQueryParams(params) + }, nil) + return err +} + +func (xc *XunLeiBrowserCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + + srcSpace := "SPACE_BROWSER" + dstSpace := "SPACE_BROWSER" + + // 对 "超级保险箱" 内的文件 特殊处理 + if srcObj.GetPath() == ThunderBrowserDriveSafeFileID { + srcSpace = "SPACE_BROWSER_SAFE" + } + if dstDir.GetPath() == ThunderBrowserDriveSafeFileID { + dstSpace = "SPACE_BROWSER_SAFE" + } + + params := map[string]string{ + "_from": dstSpace, + } + js := base.Json{ + "to": base.Json{"parent_id": dstDir.GetID(), "space": dstSpace}, + "space": srcSpace, + "ids": []string{srcObj.GetID()}, + } + // 对 "迅雷云盘" 内的文件 特殊处理 + if srcObj.GetPath() == ThunderDriveFileID { + params = map[string]string{} + js = base.Json{ + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + } + } + + _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + r.SetQueryParams(params) + }, nil) + return err +} + +func (xc *XunLeiBrowserCommon) Remove(ctx context.Context, obj model.Obj) error { + + js := base.Json{ + "ids": []string{obj.GetID()}, + "space": "SPACE_BROWSER", + } + // 对 "迅雷云盘" 内的文件 特殊处理 + if obj.GetPath() == ThunderDriveFileID { + js = base.Json{ + "ids": []string{obj.GetID()}, + } + } else if obj.GetPath() == ThunderBrowserDriveSafeFileID { + // 对 "超级保险箱" 内的文件 特殊处理 + js = base.Json{ + "ids": []string{obj.GetID()}, + "space": "SPACE_BROWSER_SAFE", + } + } + + if xc.RemoveWay == "delete" && obj.GetPath() == ThunderDriveFileID { + _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", obj.GetID()) + r.SetBody("{}") + }, nil) + return err + } else if obj.GetPath() == ThunderBrowserDriveSafeFileID { + _, err := xc.Request(FILE_API_URL+":batchDelete", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err + } + + _, err := xc.Request(FILE_API_URL+":batchTrash", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err + +} + +func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + hi := stream.GetHash() + gcid := hi.GetHash(hash_extend.GCID) + if len(gcid) < hash_extend.GCID.Width { + tFile, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + + gcid, err = utils.HashFile(hash_extend.GCID, tFile, stream.GetSize()) + if err != nil { + return err + } + } + + js := base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": stream.GetName(), + "size": stream.GetSize(), + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + "space": "SPACE_BROWSER", + } + // 对 "迅雷云盘" 内的文件 特殊处理 + if dstDir.GetPath() == ThunderDriveFileID { + js = base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": stream.GetName(), + "size": stream.GetSize(), + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + } + } else if dstDir.GetPath() == ThunderBrowserDriveSafeFileID { + // 对 "超级保险箱" 内的文件 特殊处理 + js = base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": stream.GetName(), + "size": stream.GetSize(), + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + "space": "SPACE_BROWSER_SAFE", + } + } + + var resp UploadTaskResponse + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, &resp) + if err != nil { + return err + } + + param := resp.Resumable.Params + if resp.UploadType == UPLOAD_TYPE_RESUMABLE { + param.Endpoint = strings.TrimLeft(param.Endpoint, param.Bucket+".") + s, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(param.AccessKeyID, param.AccessKeySecret, param.SecurityToken), + Region: aws.String("xunlei"), + Endpoint: aws.String(param.Endpoint), + }) + if err != nil { + return err + } + uploader := s3manager.NewUploader(s) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } + _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(param.Bucket), + Key: aws.String(param.Key), + Expires: aws.Time(param.Expiration), + Body: stream, + }) + return err + } + return nil +} + +func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, folderId string, path string) ([]model.Obj, error) { + files := make([]model.Obj, 0) + var pageToken string + for { + var fileList FileList + folderSpace := "SPACE_BROWSER" + params := map[string]string{ + "parent_id": folderId, + "page_token": pageToken, + "space": folderSpace, + "filters": `{"trashed":{"eq":false}}`, + "with_audit": "true", + "thumbnail_size": "SIZE_LARGE", + } + var fileType int8 + // 处理特殊目录 “迅雷云盘” 设置特殊的 params 以便正常访问 + pattern1 := fmt.Sprintf(`^/.*/%s(/.*)?$`, ThunderDriveFolderName) + thunderDriveMatch, _ := regexp.MatchString(pattern1, path) + // 处理特殊目录 “超级保险箱” 设置特殊的 params 以便正常访问 + pattern2 := fmt.Sprintf(`^/.*/%s(/.*)?$`, ThunderBrowserDriveSafeFolderName) + thunderBrowserDriveSafeMatch, _ := regexp.MatchString(pattern2, path) + + // 如果是 "迅雷云盘" 内的 + if folderId == ThunderDriveFileID || thunderDriveMatch { + params = map[string]string{ + "space": "", + "__type": "drive", + "refresh": "true", + "__sync": "true", + "parent_id": folderId, + "page_token": pageToken, + "with_audit": "true", + "limit": "100", + "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, + } + // 如果不是 "迅雷云盘"的"根目录" + if folderId == ThunderDriveFileID { + params["parent_id"] = "" + } + fileType = ThunderDriveType + } else if thunderBrowserDriveSafeMatch { + // 如果是 "超级保险箱" 内的 + fileType = ThunderBrowserDriveSafeType + params["space"] = "SPACE_BROWSER_SAFE" + } + + _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + r.SetQueryParams(params) + }, &fileList) + if err != nil { + return nil, err + } + // 对文件夹也进行处理 + fileList.FolderType = fileType + + for i := 0; i < len(fileList.Files); i++ { + file := &fileList.Files[i] + // 标记 文件夹内的文件 + file.FileType = fileList.FolderType + // 解决 "迅雷云盘" 重复出现问题————迅雷后端发送错误 + if file.Name == ThunderDriveFolderName && file.ID == "" && file.FolderType == ThunderDriveFolderType && folderId != "" { + continue + } + // 处理特殊目录 “迅雷云盘” 设置特殊的文件夹ID + if file.Name == ThunderDriveFolderName && file.ID == "" && file.FolderType == ThunderDriveFolderType { + file.ID = ThunderDriveFileID + } else if file.Name == ThunderBrowserDriveSafeFolderName && file.FolderType == ThunderBrowserDriveSafeFolderType { + file.FileType = ThunderBrowserDriveSafeType + } + files = append(files, file) + } + + if fileList.NextPageToken == "" { + break + } + pageToken = fileList.NextPageToken + } + return files, nil +} + +// SetRefreshTokenFunc 设置刷新Token的方法 +func (xc *XunLeiBrowserCommon) SetRefreshTokenFunc(fn func() error) { + xc.refreshTokenFunc = fn +} + +// SetTokenResp 设置Token +func (xc *XunLeiBrowserCommon) SetTokenResp(tr *TokenResp) { + xc.TokenResp = tr +} + +// SetSpaceTokenResp 设置Token +func (xc *XunLeiBrowserCommon) SetSpaceTokenResp(spaceToken string) { + xc.TokenResp.Token = spaceToken +} + +// Request 携带Authorization和CaptchaToken的请求 +func (xc *XunLeiBrowserCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + data, err := xc.Common.Request(url, method, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Authorization": xc.GetToken(), + "X-Captcha-Token": xc.GetCaptchaToken(), + "X-Space-Authorization": xc.GetSpaceToken(), + }) + if callback != nil { + callback(req) + } + }, resp) + + errResp, ok := err.(*ErrResp) + if !ok { + return nil, err + } + + switch errResp.ErrorCode { + case 0: + return data, nil + case 4122, 4121, 10, 16: + if xc.refreshTokenFunc != nil { + if err = xc.refreshTokenFunc(); err == nil { + break + } + } + return nil, err + case 9: + // space_token 获取失败 + if errResp.ErrorMsg == "space_token_invalid" { + if token, err := xc.GetSafeAccessToken(xc.Token); err != nil { + return nil, err + } else { + xc.SetSpaceTokenResp(token) + } + + } + if errResp.ErrorMsg == "captcha_invalid" { + // 验证码token过期 + if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.UserID); err != nil { + return nil, err + } + } + return nil, err + default: + return nil, err + } + return xc.Request(url, method, callback, resp) +} + +// RefreshToken 刷新Token +func (xc *XunLeiBrowserCommon) RefreshToken(refreshToken string) (*TokenResp, error) { + var resp TokenResp + _, err := xc.Common.Request(XLUSER_API_URL+"/auth/token", http.MethodPost, func(req *resty.Request) { + req.SetBody(&base.Json{ + "grant_type": "refresh_token", + "refresh_token": refreshToken, + "client_id": xc.ClientID, + "client_secret": xc.ClientSecret, + }) + }, &resp) + if err != nil { + return nil, err + } + + if resp.RefreshToken == "" { + return nil, errs.EmptyToken + } + return &resp, nil +} + +// GetSafeAccessToken 获取 超级保险柜 AccessToken +func (xc *XunLeiBrowserCommon) GetSafeAccessToken(safePassword string) (string, error) { + var resp TokenResp + _, err := xc.Request(XLUSER_API_URL+"/password/check", http.MethodPost, func(req *resty.Request) { + req.SetBody(&base.Json{ + "scene": "box", + "password": EncryptPassword(safePassword), + }) + }, &resp) + if err != nil { + return "", err + } + + if resp.Token == "" { + return "", errs.EmptyToken + } + return resp.Token, nil +} + +// Login 登录 +func (xc *XunLeiBrowserCommon) Login(username, password string) (*TokenResp, error) { + url := XLUSER_API_URL + "/auth/signin" + err := xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username) + if err != nil { + return nil, err + } + + var resp TokenResp + _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(&SignInRequest{ + CaptchaToken: xc.GetCaptchaToken(), + ClientID: xc.ClientID, + ClientSecret: xc.ClientSecret, + Username: username, + Password: password, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (xc *XunLeiBrowserCommon) IsLogin() bool { + if xc.TokenResp == nil { + return false + } + _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) + return err == nil +} diff --git a/drivers/thunder_browser/meta.go b/drivers/thunder_browser/meta.go new file mode 100644 index 00000000000..9d16cd78e5c --- /dev/null +++ b/drivers/thunder_browser/meta.go @@ -0,0 +1,108 @@ +package thunder_browser + +import ( + "crypto/md5" + "encoding/hex" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// ExpertAddition 高级设置 +type ExpertAddition struct { + driver.RootID + + LoginType string `json:"login_type" type:"select" options:"user,refresh_token" default:"user"` + SignType string `json:"sign_type" type:"select" options:"algorithms,captcha_sign" default:"algorithms"` + + // 登录方式1 + Username string `json:"username" required:"true" help:"login type is user,this is required"` + Password string `json:"password" required:"true" help:"login type is user,this is required"` + SafePassword string `json:"safe_password" required:"false" help:"login type is user,this is required"` // 超级保险箱密码 + // 登录方式2 + RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` + + // 签名方法1 + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"x+I5XiTByg,6QU1x5DqGAV3JKg6h,VI1vL1WXr7st0es,n+/3yhlrnKs4ewhLgZhZ5ITpt554,UOip2PE7BLIEov/ZX6VOnsz,Q70h9lpViNCOC8sGVkar9o22LhBTjfP,IVHFuB1JcMlaZHnW,bKE,HZRbwxOiQx+diNopi6Nu,fwyasXgYL3rP314331b,LWxXAiSW4,UlWIjv1HGrC6Ngmt4Nohx,FOa+Lc0bxTDpTwIh2,0+RY,xmRVMqokHHpvsiH0"` + // 签名方法2 + CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` + Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` + + // 验证码 + CaptchaToken string `json:"captcha_token"` + + // 必要且影响登录,由签名决定 + DeviceID string `json:"device_id" required:"true" default:"9aa5c268e7bcfc197a9ad88e2fb330e5"` + ClientID string `json:"client_id" required:"true" default:"ZUBzD9J_XPXfn7f7"` + ClientSecret string `json:"client_secret" required:"true" default:"yESVmHecEe6F0aou69vl-g"` + ClientVersion string `json:"client_version" required:"true" default:"1.0.7.1938"` + PackageName string `json:"package_name" required:"true" default:"com.xunlei.browser"` + + // 不影响登录,影响下载速度 + UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.xunlei.browser/1.0.7.1938 netWorkType/5G appid/22062 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/233100 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)"` + DownloadUserAgent string `json:"download_user_agent" required:"true" default:"AndroidDownloadManager/12 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)"` + + // 优先使用视频链接代替下载链接 + UseVideoUrl bool `json:"use_video_url"` + // 移除方式 + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` +} + +// GetIdentity 登录特征,用于判断是否重新登录 +func (i *ExpertAddition) GetIdentity() string { + hash := md5.New() + if i.LoginType == "refresh_token" { + hash.Write([]byte(i.RefreshToken)) + } else { + hash.Write([]byte(i.Username + i.Password)) + } + + if i.SignType == "captcha_sign" { + hash.Write([]byte(i.CaptchaSign + i.Timestamp)) + } else { + hash.Write([]byte(i.Algorithms)) + } + + hash.Write([]byte(i.DeviceID)) + hash.Write([]byte(i.ClientID)) + hash.Write([]byte(i.ClientSecret)) + hash.Write([]byte(i.ClientVersion)) + hash.Write([]byte(i.PackageName)) + return hex.EncodeToString(hash.Sum(nil)) +} + +type Addition struct { + driver.RootID + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + SafePassword string `json:"safe_password" required:"false"` // 超级保险箱密码 + CaptchaToken string `json:"captcha_token"` + UseVideoUrl bool `json:"use_video_url" default:"false"` + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` +} + +// GetIdentity 登录特征,用于判断是否重新登录 +func (i *Addition) GetIdentity() string { + return utils.GetMD5EncodeStr(i.Username + i.Password) +} + +var config = driver.Config{ + Name: "ThunderBrowser", + LocalSort: true, + OnlyProxy: true, +} + +var configExpert = driver.Config{ + Name: "ThunderBrowserExpert", + LocalSort: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ThunderBrowser{} + }) + op.RegisterDriver(func() driver.Driver { + return &ThunderBrowserExpert{} + }) +} diff --git a/drivers/thunder_browser/types.go b/drivers/thunder_browser/types.go new file mode 100644 index 00000000000..774b34bb287 --- /dev/null +++ b/drivers/thunder_browser/types.go @@ -0,0 +1,223 @@ +package thunder_browser + +import ( + "fmt" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" +) + +type ErrResp struct { + ErrorCode int64 `json:"error_code"` + ErrorMsg string `json:"error"` + ErrorDescription string `json:"error_description"` + // ErrorDetails interface{} `json:"error_details"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) +} + +/* +* 验证码Token +**/ +type CaptchaTokenRequest struct { + Action string `json:"action"` + CaptchaToken string `json:"captcha_token"` + ClientID string `json:"client_id"` + DeviceID string `json:"device_id"` + Meta map[string]string `json:"meta"` + RedirectUri string `json:"redirect_uri"` +} + +type CaptchaTokenResponse struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Url string `json:"url"` +} + +/* +* 登录 +**/ +type TokenResp struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int64 `json:"expires_in"` + + Sub string `json:"sub"` + UserID string `json:"user_id"` + + Token string `json:"token"` // "超级保险箱" 访问Token +} + +func (t *TokenResp) GetToken() string { + return fmt.Sprint(t.TokenType, " ", t.AccessToken) +} + +// GetSpaceToken 获取"超级保险箱" 访问Token +func (t *TokenResp) GetSpaceToken() string { + return t.Token +} + +type SignInRequest struct { + CaptchaToken string `json:"captcha_token"` + + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + + Username string `json:"username"` + Password string `json:"password"` +} + +/* +* 文件 +**/ +type FileList struct { + Kind string `json:"kind"` + NextPageToken string `json:"next_page_token"` + Files []Files `json:"files"` + Version string `json:"version"` + VersionOutdated bool `json:"version_outdated"` + FolderType int8 +} + +type Link struct { + URL string `json:"url"` + Token string `json:"token"` + Expire time.Time `json:"expire"` + Type string `json:"type"` +} + +var _ model.Obj = (*Files)(nil) + +type Files struct { + Kind string `json:"kind"` + ID string `json:"id"` + ParentID string `json:"parent_id"` + Name string `json:"name"` + //UserID string `json:"user_id"` + Size string `json:"size"` + //Revision string `json:"revision"` + //FileExtension string `json:"file_extension"` + //MimeType string `json:"mime_type"` + //Starred bool `json:"starred"` + WebContentLink string `json:"web_content_link"` + CreatedTime CustomTime `json:"created_time"` + ModifiedTime CustomTime `json:"modified_time"` + IconLink string `json:"icon_link"` + ThumbnailLink string `json:"thumbnail_link"` + // Md5Checksum string `json:"md5_checksum"` + Hash string `json:"hash"` + // Links map[string]Link `json:"links"` + // Phase string `json:"phase"` + // Audit struct { + // Status string `json:"status"` + // Message string `json:"message"` + // Title string `json:"title"` + // } `json:"audit"` + Medias []struct { + //Category string `json:"category"` + //IconLink string `json:"icon_link"` + //IsDefault bool `json:"is_default"` + //IsOrigin bool `json:"is_origin"` + //IsVisible bool `json:"is_visible"` + Link Link `json:"link"` + //MediaID string `json:"media_id"` + //MediaName string `json:"media_name"` + //NeedMoreQuota bool `json:"need_more_quota"` + //Priority int `json:"priority"` + //RedirectLink string `json:"redirect_link"` + //ResolutionName string `json:"resolution_name"` + // Video struct { + // AudioCodec string `json:"audio_codec"` + // BitRate int `json:"bit_rate"` + // Duration int `json:"duration"` + // FrameRate int `json:"frame_rate"` + // Height int `json:"height"` + // VideoCodec string `json:"video_codec"` + // VideoType string `json:"video_type"` + // Width int `json:"width"` + // } `json:"video"` + // VipTypes []string `json:"vip_types"` + } `json:"medias"` + Trashed bool `json:"trashed"` + DeleteTime string `json:"delete_time"` + OriginalURL string `json:"original_url"` + //Params struct{} `json:"params"` + //OriginalFileIndex int `json:"original_file_index"` + //Space string `json:"space"` + //Apps []interface{} `json:"apps"` + //Writable bool `json:"writable"` + FolderType string `json:"folder_type"` + //Collection interface{} `json:"collection"` + FileType int8 +} + +func (c *Files) GetHash() utils.HashInfo { + return utils.NewHashInfo(hash_extend.GCID, c.Hash) +} + +func (c *Files) GetSize() int64 { size, _ := strconv.ParseInt(c.Size, 10, 64); return size } +func (c *Files) GetName() string { return c.Name } +func (c *Files) CreateTime() time.Time { return c.CreatedTime.Time } +func (c *Files) ModTime() time.Time { return c.ModifiedTime.Time } +func (c *Files) IsDir() bool { return c.Kind == FOLDER } +func (c *Files) GetID() string { return c.ID } +func (c *Files) GetPath() string { + // 对特殊文件进行特殊处理 + if c.FileType == ThunderDriveType { + return ThunderDriveFileID + } else if c.FileType == ThunderBrowserDriveSafeType { + return ThunderBrowserDriveSafeFileID + } + return "" +} +func (c *Files) Thumb() string { return c.ThumbnailLink } + +/* +* 上传 +**/ +type UploadTaskResponse struct { + UploadType string `json:"upload_type"` + + /*//UPLOAD_TYPE_FORM + Form struct { + //Headers struct{} `json:"headers"` + Kind string `json:"kind"` + Method string `json:"method"` + MultiParts struct { + OSSAccessKeyID string `json:"OSSAccessKeyId"` + Signature string `json:"Signature"` + Callback string `json:"callback"` + Key string `json:"key"` + Policy string `json:"policy"` + XUserData string `json:"x:user_data"` + } `json:"multi_parts"` + URL string `json:"url"` + } `json:"form"`*/ + + //UPLOAD_TYPE_RESUMABLE + Resumable struct { + Kind string `json:"kind"` + Params struct { + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + Bucket string `json:"bucket"` + Endpoint string `json:"endpoint"` + Expiration time.Time `json:"expiration"` + Key string `json:"key"` + SecurityToken string `json:"security_token"` + } `json:"params"` + Provider string `json:"provider"` + } `json:"resumable"` + + File Files `json:"file"` +} diff --git a/drivers/thunder_browser/util.go b/drivers/thunder_browser/util.go new file mode 100644 index 00000000000..fd8a4047b1b --- /dev/null +++ b/drivers/thunder_browser/util.go @@ -0,0 +1,249 @@ +package thunder_browser + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + "regexp" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + API_URL = "https://x-api-pan.xunlei.com/drive/v1" + FILE_API_URL = API_URL + "/files" + XLUSER_API_URL = "https://xluser-ssl.xunlei.com/v1" +) + +const ( + FOLDER = "drive#folder" + FILE = "drive#file" + RESUMABLE = "drive#resumable" +) + +const ( + UPLOAD_TYPE_UNKNOWN = "UPLOAD_TYPE_UNKNOWN" + //UPLOAD_TYPE_FORM = "UPLOAD_TYPE_FORM" + UPLOAD_TYPE_RESUMABLE = "UPLOAD_TYPE_RESUMABLE" + UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" +) + +const ( + ThunderDriveFileID = "XXXXXXXXXXXXXXXXXXXXXXXXXX" + ThunderBrowserDriveSafeFileID = "YYYYYYYYYYYYYYYYYYYYYYYYYY" + ThunderDriveFolderName = "迅雷云盘" + ThunderBrowserDriveSafeFolderName = "超级保险箱" + ThunderDriveType = 1 + ThunderBrowserDriveSafeType = 2 + ThunderDriveFolderType = "DEFAULT_ROOT" + ThunderBrowserDriveSafeFolderType = "BROWSER_SAFE" +) + +func GetAction(method string, url string) string { + urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] + return method + ":" + urlpath +} + +type Common struct { + client *resty.Client + + captchaToken string + + // 签名相关,二选一 + Algorithms []string + Timestamp, CaptchaSign string + + // 必要值,签名相关 + DeviceID string + ClientID string + ClientSecret string + ClientVersion string + PackageName string + UserAgent string + DownloadUserAgent string + UseVideoUrl bool + RemoveWay string + + // 验证码token刷新成功回调 + refreshCTokenCk func(token string) +} + +func (c *Common) SetCaptchaToken(captchaToken string) { + c.captchaToken = captchaToken +} +func (c *Common) GetCaptchaToken() string { + return c.captchaToken +} + +// RefreshCaptchaTokenAtLogin 刷新验证码token(登录后) +func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { + metas := map[string]string{ + "client_version": c.ClientVersion, + "package_name": c.PackageName, + "user_id": userID, + } + metas["timestamp"], metas["captcha_sign"] = c.GetCaptchaSign() + return c.refreshCaptchaToken(action, metas) +} + +// RefreshCaptchaTokenInLogin 刷新验证码token(登录时) +func (c *Common) RefreshCaptchaTokenInLogin(action, username string) error { + metas := make(map[string]string) + if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { + metas["email"] = username + } else if len(username) >= 11 && len(username) <= 18 { + metas["phone_number"] = username + } else { + metas["username"] = username + } + return c.refreshCaptchaToken(action, metas) +} + +// GetCaptchaSign 获取验证码签名 +func (c *Common) GetCaptchaSign() (timestamp, sign string) { + if len(c.Algorithms) == 0 { + return c.Timestamp, c.CaptchaSign + } + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) + for _, algorithm := range c.Algorithms { + str = utils.GetMD5EncodeStr(str + algorithm) + } + sign = "1." + str + return +} + +// 刷新验证码token +func (c *Common) refreshCaptchaToken(action string, metas map[string]string) error { + param := CaptchaTokenRequest{ + Action: action, + CaptchaToken: c.captchaToken, + ClientID: c.ClientID, + DeviceID: c.DeviceID, + Meta: metas, + RedirectUri: "xlaccsdk01://xunlei.com/callback?state=harbor", + } + var e ErrResp + var resp CaptchaTokenResponse + _, err := c.Request(XLUSER_API_URL+"/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + req.SetError(&e).SetBody(param) + }, &resp) + + if err != nil { + return err + } + + if e.IsError() { + return &e + } + + if resp.Url != "" { + return fmt.Errorf(`need verify: Click Here`, resp.Url) + } + + if resp.CaptchaToken == "" { + return fmt.Errorf("empty captchaToken") + } + + if c.refreshCTokenCk != nil { + c.refreshCTokenCk(resp.CaptchaToken) + } + c.SetCaptchaToken(resp.CaptchaToken) + return nil +} + +// Request 只有基础信息的请求 +func (c *Common) Request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := c.client.R().SetHeaders(map[string]string{ + "user-agent": c.UserAgent, + "accept": "application/json;charset=UTF-8", + "x-device-id": c.DeviceID, + "x-client-id": c.ClientID, + "x-client-version": c.ClientVersion, + }) + + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + + var erron ErrResp + utils.Json.Unmarshal(res.Body(), &erron) + if erron.IsError() { + return nil, &erron + } + + return res.Body(), nil +} + +// 计算文件Gcid +func getGcid(r io.Reader, size int64) (string, error) { + calcBlockSize := func(j int64) int64 { + var psize int64 = 0x40000 + for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { + psize = psize << 1 + } + return psize + } + + hash1 := sha1.New() + hash2 := sha1.New() + readSize := calcBlockSize(size) + for { + hash2.Reset() + if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { + if err != io.EOF { + return "", err + } + break + } + hash1.Write(hash2.Sum(nil)) + } + return hex.EncodeToString(hash1.Sum(nil)), nil +} + +type CustomTime struct { + time.Time +} + +const timeFormat = time.RFC3339 + +func (ct *CustomTime) UnmarshalJSON(b []byte) error { + str := string(b) + if str == `""` { + *ct = CustomTime{Time: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)} + return nil + } + + t, err := time.Parse(`"`+timeFormat+`"`, str) + if err != nil { + return err + } + *ct = CustomTime{Time: t} + return nil +} + +// EncryptPassword 超级保险箱 加密 +func EncryptPassword(password string) string { + if password == "" { + return "" + } + // 将字符串转换为字节数组 + byteData := []byte(password) + // 计算MD5哈希值 + hash := md5.Sum(byteData) + // 将哈希值转换为十六进制字符串 + return hex.EncodeToString(hash[:]) +} diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index 17835d913f2..7b5daf414ac 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -515,6 +515,7 @@ func (xc *XunLeiXCommon) Login(username, password string) (*TokenResp, error) { if err != nil { return nil, err } + resp.UserID = resp.Sub return &resp, nil } From 1b14d33b9f8c68e2a8652ee4337bea1d8c301b04 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sun, 16 Jun 2024 16:55:49 +0800 Subject: [PATCH 196/659] fix(alist_v3): use `net/http` for uploading (#6616 close #6613) --- drivers/alist_v3/driver.go | 48 +++++++++++++++++++++++++++++--------- drivers/alist_v3/util.go | 34 +++------------------------ 2 files changed, 40 insertions(+), 42 deletions(-) diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index 53fb93caa1f..d078c5fb421 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -6,9 +6,7 @@ import ( "io" "net/http" "path" - "strconv" "strings" - "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" @@ -17,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) type AListV3 struct { @@ -42,7 +41,7 @@ func (d *AListV3) Init(ctx context.Context) error { return err } // if the username is not empty and the username is not the same as the current username, then login again - if d.Username != "" && d.Username != resp.Data.Username { + if d.Username != resp.Data.Username { err = d.login() if err != nil { return err @@ -183,14 +182,41 @@ func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { } func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - _, err := d.requestWithTimeout("/fs/put", http.MethodPut, func(req *resty.Request) { - req.SetHeader("File-Path", path.Join(dstDir.GetPath(), stream.GetName())). - SetHeader("Password", d.MetaPassword). - SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). - SetContentLength(true). - SetBody(io.ReadCloser(stream)) - }, time.Hour*6) - return err + req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", stream) + if err != nil { + return err + } + req.Header.Set("Authorization", d.Token) + req.Header.Set("File-Path", path.Join(dstDir.GetPath(), stream.GetName())) + req.Header.Set("Password", d.MetaPassword) + + req.ContentLength = stream.GetSize() + // client := base.NewHttpClient() + // client.Timeout = time.Hour * 6 + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + + bytes, err := io.ReadAll(res.Body) + if err != nil { + return err + } + log.Debugf("[alist_v3] response body: %s", string(bytes)) + if res.StatusCode >= 400 { + return fmt.Errorf("request failed, status: %s", res.Status) + } + code := utils.Json.Get(bytes, "code").ToInt() + if code != 200 { + if code == 401 || code == 403 { + err = d.login() + if err != nil { + return err + } + } + return fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(bytes, "message").ToString()) + } + return nil } //func (d *AList) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/alist_v3/util.go b/drivers/alist_v3/util.go index 978f3ac0568..5ede285af5b 100644 --- a/drivers/alist_v3/util.go +++ b/drivers/alist_v3/util.go @@ -3,7 +3,6 @@ package alist_v3 import ( "fmt" "net/http" - "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/op" @@ -14,6 +13,9 @@ import ( ) func (d *AListV3) login() error { + if d.Username == "" { + return nil + } var resp common.Resp[LoginResp] _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(base.Json{ @@ -57,33 +59,3 @@ func (d *AListV3) request(api, method string, callback base.ReqCallback, retry . } return res.Body(), nil } - -func (d *AListV3) requestWithTimeout(api, method string, callback base.ReqCallback, timeout time.Duration, retry ...bool) ([]byte, error) { - url := d.Address + "/api" + api - client := base.NewRestyClient().SetTimeout(timeout) - req := client.R() - req.SetHeader("Authorization", d.Token) - if callback != nil { - callback(req) - } - res, err := req.Execute(method, url) - if err != nil { - return nil, err - } - log.Debugf("[alist_v3] response body: %s", res.String()) - if res.StatusCode() >= 400 { - return nil, fmt.Errorf("request failed, status: %s", res.Status()) - } - code := utils.Json.Get(res.Body(), "code").ToInt() - if code != 200 { - if (code == 401 || code == 403) && !utils.IsBool(retry...) { - err = d.login() - if err != nil { - return nil, err - } - return d.requestWithTimeout(api, method, callback, timeout, true) - } - return nil, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) - } - return res.Body(), nil -} From 3a996a1a3a6cc76237b410b90eee808cb73b6722 Mon Sep 17 00:00:00 2001 From: Mmx Date: Sun, 16 Jun 2024 16:56:45 +0800 Subject: [PATCH 197/659] build: update sqlite driver (#6599) * build: update sqlite driver * build: remove docker build sqlite-compatible commands --- build.sh | 9 +-------- go.mod | 8 ++++---- go.sum | 13 ++++++------- 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/build.sh b/build.sh index 368d2d2cfb0..9d0f4174432 100644 --- a/build.sh +++ b/build.sh @@ -85,14 +85,7 @@ BuildDev() { cat md5.txt } -PrepareBuildDocker() { - echo "replace github.com/mattn/go-sqlite3 => github.com/leso-kn/go-sqlite3 v0.0.0-20230710125852-03158dc838ed" >>go.mod - go get gorm.io/driver/sqlite@v1.4.4 - go mod download -} - BuildDocker() { - PrepareBuildDocker go build -o ./bin/alist -ldflags="$ldflags" -tags=jsoniter . } @@ -110,7 +103,7 @@ PrepareBuildDockerMusl() { } BuildDockerMultiplatform() { - PrepareBuildDocker + go mod download # run PrepareBuildDockerMusl before build export PATH=$PATH:$PWD/build/musl-libs/bin diff --git a/go.mod b/go.mod index 9dd012c9667..cbe21348d0e 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/google/uuid v1.5.0 github.com/gorilla/websocket v1.5.1 github.com/hirochachacha/go-smb2 v1.1.0 + github.com/ipfs/boxo v0.12.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 @@ -66,8 +67,8 @@ require ( gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 - gorm.io/driver/sqlite v1.4.4 - gorm.io/gorm v1.24.5 + gorm.io/driver/sqlite v1.5.5 + gorm.io/gorm v1.25.10 ) require ( @@ -131,7 +132,6 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ipfs/boxo v0.12.0 // indirect github.com/ipfs/go-cid v0.4.1 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect @@ -156,7 +156,7 @@ require ( github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/mattn/go-sqlite3 v1.14.15 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/go.sum b/go.sum index c4f6832bcf1..f84330a873f 100644 --- a/go.sum +++ b/go.sum @@ -317,8 +317,8 @@ github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+Ei github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/meilisearch/meilisearch-go v0.26.1 h1:3bmo2uLijX7kvBmiZ9LupVfC95TFcRJDgrRTzbOoE4A= @@ -661,13 +661,12 @@ gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= gorm.io/driver/postgres v1.4.8 h1:NDWizaclb7Q2aupT0jkwK8jx1HVCNzt+PQ8v/VnxviA= gorm.io/driver/postgres v1.4.8/go.mod h1:O9MruWGNLUBUWVYfWuBClpf3HeGjOoybY0SNmCs3wsw= -gorm.io/driver/sqlite v1.4.4 h1:gIufGoR0dQzjkyqDyYSCvsYR6fba1Gw5YKDqKeChxFc= -gorm.io/driver/sqlite v1.4.4/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= +gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= +gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= -gorm.io/gorm v1.24.5 h1:g6OPREKqqlWq4kh/3MCQbZKImeB9e6Xgc4zD+JgNZGE= -gorm.io/gorm v1.24.5/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= +gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= From fcf2683112ecf4df1f672b68b64b6d7c1efbd31c Mon Sep 17 00:00:00 2001 From: Toby Shi Date: Sun, 16 Jun 2024 16:58:02 +0800 Subject: [PATCH 198/659] feat(ftp): custom encoding (#6528 close #1260) --- drivers/ftp/driver.go | 26 +++++++++++++++++--------- drivers/ftp/meta.go | 18 ++++++++++++++++++ go.mod | 1 + go.sum | 2 ++ 4 files changed, 38 insertions(+), 9 deletions(-) diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index 70fbabdcdcd..05b9e49a91d 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -39,7 +39,7 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m if err := d.login(); err != nil { return nil, err } - entries, err := d.conn.List(dir.GetPath()) + entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) if err != nil { return nil, err } @@ -49,7 +49,7 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m continue } f := model.Object{ - Name: entry.Name, + Name: decode(entry.Name, d.Encoding), Size: int64(entry.Size), Modified: entry.Time, IsFolder: entry.Type == ftp.EntryTypeFolder, @@ -64,7 +64,7 @@ func (d *FTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*m return nil, err } - r := NewFileReader(d.conn, file.GetPath(), file.GetSize()) + r := NewFileReader(d.conn, encode(file.GetPath(), d.Encoding), file.GetSize()) link := &model.Link{ MFile: r, } @@ -75,21 +75,27 @@ func (d *FTP) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) if err := d.login(); err != nil { return err } - return d.conn.MakeDir(stdpath.Join(parentDir.GetPath(), dirName)) + return d.conn.MakeDir(encode(stdpath.Join(parentDir.GetPath(), dirName), d.Encoding)) } func (d *FTP) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if err := d.login(); err != nil { return err } - return d.conn.Rename(srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName())) + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(dstDir.GetPath(), srcObj.GetName()), d.Encoding), + ) } func (d *FTP) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if err := d.login(); err != nil { return err } - return d.conn.Rename(srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName)) + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), d.Encoding), + ) } func (d *FTP) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -100,10 +106,11 @@ func (d *FTP) Remove(ctx context.Context, obj model.Obj) error { if err := d.login(); err != nil { return err } + path := encode(obj.GetPath(), d.Encoding) if obj.IsDir() { - return d.conn.RemoveDirRecur(obj.GetPath()) + return d.conn.RemoveDirRecur(path) } else { - return d.conn.Delete(obj.GetPath()) + return d.conn.Delete(path) } } @@ -112,7 +119,8 @@ func (d *FTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStream return err } // TODO: support cancel - return d.conn.Stor(stdpath.Join(dstDir.GetPath(), stream.GetName()), stream) + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) + return d.conn.Stor(encode(path, d.Encoding), stream) } var _ driver.Driver = (*FTP)(nil) diff --git a/drivers/ftp/meta.go b/drivers/ftp/meta.go index 61d9d4a824e..5652c12e184 100644 --- a/drivers/ftp/meta.go +++ b/drivers/ftp/meta.go @@ -3,10 +3,28 @@ package ftp import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/op" + "github.com/axgle/mahonia" ) +func encode(str string, encoding string) string { + if encoding == "" { + return str + } + encoder := mahonia.NewEncoder(encoding) + return encoder.ConvertString(str) +} + +func decode(str string, encoding string) string { + if encoding == "" { + return str + } + decoder := mahonia.NewDecoder(encoding) + return decoder.ConvertString(str) +} + type Addition struct { Address string `json:"address" required:"true"` + Encoding string `json:"encoding" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootPath diff --git a/go.mod b/go.mod index cbe21348d0e..b18ff9373fb 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( github.com/aead/ecdh v0.2.0 // indirect github.com/andreburgaud/crypt2go v1.2.0 // indirect github.com/andybalholm/brotli v1.0.4 // indirect + github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index f84330a873f..de6005d153e 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.50.24 h1:3o2Pg7mOoVL0jv54vWtuafoZqAeEXLhm1tltWA2GcEw= github.com/aws/aws-sdk-go v1.50.24/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= +github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= From 29fe49fb872561df7c73dc89b29ce3e2b359a631 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sun, 16 Jun 2024 16:59:10 +0800 Subject: [PATCH 199/659] fix(alias): Support forced refresh of file list (#6562) --- drivers/alias/driver.go | 3 ++- drivers/alias/util.go | 40 ++++++++++++++++++++-------------------- internal/fs/list.go | 3 ++- internal/model/args.go | 1 + internal/op/fs.go | 4 ++-- 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index d9b290edc65..1b439a2c9d9 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -91,8 +91,9 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ return nil, errs.ObjectNotFound } var objs []model.Obj + fsArgs := &fs.ListArgs{NoLog: true, Refresh: args.Refresh} for _, dst := range dsts { - tmp, err := d.list(ctx, dst, sub) + tmp, err := d.list(ctx, dst, sub, fsArgs) if err == nil { objs = append(objs, tmp...) } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index ba1f7e72649..c0e9081b0fc 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -16,7 +16,7 @@ import ( func (d *Alias) listRoot() []model.Obj { var objs []model.Obj - for k, _ := range d.pathMap { + for k := range d.pathMap { obj := model.Object{ Name: k, IsFolder: true, @@ -65,8 +65,8 @@ func (d *Alias) get(ctx context.Context, path string, dst, sub string) (model.Ob }, nil } -func (d *Alias) list(ctx context.Context, dst, sub string) ([]model.Obj, error) { - objs, err := fs.List(ctx, stdpath.Join(dst, sub), &fs.ListArgs{NoLog: true}) +func (d *Alias) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]model.Obj, error) { + objs, err := fs.List(ctx, stdpath.Join(dst, sub), args) // the obj must implement the model.SetPath interface // return objs, err if err != nil { @@ -120,32 +120,32 @@ func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) func (d *Alias) getReqPath(ctx context.Context, obj model.Obj) (*string, error) { root, sub := d.getRootAndPath(obj.GetPath()) - if sub == "" || sub == "/" { + if sub == "" { return nil, errs.NotSupport } dsts, ok := d.pathMap[root] if !ok { return nil, errs.ObjectNotFound } - var reqPath string - var err error + var reqPath *string for _, dst := range dsts { - reqPath = stdpath.Join(dst, sub) - _, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) - if err == nil { - if d.ProtectSameName { - if ok { - ok = false - } else { - return nil, errs.NotImplement - } - } else { - break - } + path := stdpath.Join(dst, sub) + _, err := fs.Get(ctx, path, &fs.GetArgs{NoLog: true}) + if err != nil { + continue } + if !d.ProtectSameName { + return &path, nil + } + if ok { + ok = false + } else { + return nil, errs.NotImplement + } + reqPath = &path } - if err != nil { + if reqPath == nil { return nil, errs.ObjectNotFound } - return &reqPath, nil + return reqPath, nil } diff --git a/internal/fs/list.go b/internal/fs/list.go index 6e257cea6fa..d4f59cb829f 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -24,7 +24,8 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) if storage != nil { _objs, err = op.List(ctx, storage, actualPath, model.ListArgs{ ReqPath: path, - }, args.Refresh) + Refresh: args.Refresh, + }) if err != nil { if !args.NoLog { log.Errorf("fs/list: %+v", err) diff --git a/internal/model/args.go b/internal/model/args.go index ac3c1875bfa..613699b95b4 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -13,6 +13,7 @@ import ( type ListArgs struct { ReqPath string S3ShowPlaceholder bool + Refresh bool } type LinkArgs struct { diff --git a/internal/op/fs.go b/internal/op/fs.go index 4f0cbbdd3ae..5c9c9f3f138 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -100,14 +100,14 @@ func Key(storage driver.Driver, path string) string { } // List files in storage, not contains virtual file -func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) { +func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) log.Debugf("op.List %s", path) key := Key(storage, path) - if !utils.IsBool(refresh...) { + if !args.Refresh { if files, ok := listCache.Get(key); ok { log.Debugf("use cache when list %s", path) return files, nil From 453d7da62290664f3d4c762e29286a1bc516c4a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=AB=E4=B9=90=E7=9A=84=E8=80=81=E9=BC=A0=E5=AE=9D?= =?UTF-8?q?=E5=AE=9D?= Date: Fri, 28 Jun 2024 23:47:21 +0800 Subject: [PATCH 200/659] docs: change outdated repository link to alist-org (#6007) --- .github/ISSUE_TEMPLATE/config.yml | 2 +- README.md | 14 +++++++------- README_cn.md | 14 +++++++------- README_ja.md | 14 +++++++------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index be284ab6178..9012760c8f3 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: Questions & Discussions - url: https://github.com/Xhofe/alist/discussions + url: https://github.com/alist-org/alist/discussions about: Use GitHub discussions for message-board style questions and discussions. \ No newline at end of file diff --git a/README.md b/README.md index 702638e11c7..9f0b7ab8312 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,13 @@ latest version - + License - + Build status - + latest version @@ -19,13 +19,13 @@
- + discussions discussions - + Downloads @@ -106,7 +106,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Discussion -Please go to our [discussion forum](https://github.com/Xhofe/alist/discussions) for general questions, **issues are for bug reports and feature requests only.** +Please go to our [discussion forum](https://github.com/alist-org/alist/discussions) for general questions, **issues are for bug reports and feature requests only.** ## Sponsor @@ -138,4 +138,4 @@ The `AList` is open-source software licensed under the AGPL-3.0 license. --- -> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) +> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/alist-org) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) diff --git a/README_cn.md b/README_cn.md index f268d383c8b..ec45c6ef9bc 100644 --- a/README_cn.md +++ b/README_cn.md @@ -5,13 +5,13 @@ latest version - + License - + Build status - + latest version @@ -19,13 +19,13 @@
- + discussions discussions - + Downloads @@ -105,7 +105,7 @@ ## 讨论 -一般问题请到[讨论论坛](https://github.com/Xhofe/alist/discussions) ，**issue仅针对错误报告和功能请求。** +一般问题请到[讨论论坛](https://github.com/alist-org/alist/discussions) ，**issue仅针对错误报告和功能请求。** ## 赞助 @@ -136,4 +136,4 @@ Thanks goes to these wonderful people: --- -> [@博客](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@Telegram群](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) +> [@博客](https://nn.ci/) · [@GitHub](https://github.com/alist-org) · [@Telegram群](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) diff --git a/README_ja.md b/README_ja.md index 7cef979f75e..ef1351dfd8b 100644 --- a/README_ja.md +++ b/README_ja.md @@ -5,13 +5,13 @@ latest version - + License - + Build status - + latest version @@ -19,13 +19,13 @@
- + discussions discussions - + Downloads @@ -106,7 +106,7 @@ ## ディスカッション -一般的なご質問は[ディスカッションフォーラム](https://github.com/Xhofe/alist/discussions)をご利用ください。**問題はバグレポートと機能リクエストのみです。** +一般的なご質問は[ディスカッションフォーラム](https://github.com/alist-org/alist/discussions)をご利用ください。**問題はバグレポートと機能リクエストのみです。** ## スポンサー @@ -138,4 +138,4 @@ https://alist.nn.ci/guide/sponsor.html --- -> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) +> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/alist-org) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) From 227d034db85015f286b43091643ee8b42233dbdf Mon Sep 17 00:00:00 2001 From: XZB-1248 <28593573+XZB-1248@users.noreply.github.com> Date: Fri, 28 Jun 2024 23:50:00 +0800 Subject: [PATCH 201/659] feat(sftp): add suport for passphrase of private key (#6624 close #6592) Co-authored-by: XZB --- drivers/sftp/meta.go | 1 + drivers/sftp/util.go | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/drivers/sftp/meta.go b/drivers/sftp/meta.go index bdc3d827ff2..9b1665679cd 100644 --- a/drivers/sftp/meta.go +++ b/drivers/sftp/meta.go @@ -10,6 +10,7 @@ type Addition struct { Username string `json:"username" required:"true"` PrivateKey string `json:"private_key" type:"text"` Password string `json:"password"` + Passphrase string `json:"passphrase"` driver.RootPath IgnoreSymlinkError bool `json:"ignore_symlink_error" default:"false" info:"Ignore symlink error"` } diff --git a/drivers/sftp/util.go b/drivers/sftp/util.go index eaeeaff5814..53f9c379e04 100644 --- a/drivers/sftp/util.go +++ b/drivers/sftp/util.go @@ -12,8 +12,14 @@ import ( func (d *SFTP) initClient() error { var auth ssh.AuthMethod - if d.PrivateKey != "" { - signer, err := ssh.ParsePrivateKey([]byte(d.PrivateKey)) + if len(d.PrivateKey) > 0 { + var err error + var signer ssh.Signer + if len(d.Passphrase) > 0 { + signer, err = ssh.ParsePrivateKeyWithPassphrase([]byte(d.PrivateKey), []byte(d.Passphrase)) + } else { + signer, err = ssh.ParsePrivateKey([]byte(d.PrivateKey)) + } if err != nil { return err } From 432901db5af14891b210b6dd792c44ac9409ad88 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Tue, 2 Jul 2024 14:59:07 +0800 Subject: [PATCH 202/659] feat(thunderx): generate UserAgent automatically (#6664) --- drivers/thunderx/driver.go | 120 +++++++++++++++++++++++-------------- drivers/thunderx/meta.go | 14 ++--- drivers/thunderx/util.go | 99 +++++++++++++++++++++++++++++- 3 files changed, 179 insertions(+), 54 deletions(-) diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index 7b5daf414ac..b9ee668c2f9 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -3,10 +3,6 @@ package thunderx import ( "context" "fmt" - "github.com/go-resty/resty/v2" - "net/http" - "strings" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -18,6 +14,9 @@ import ( "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/go-resty/resty/v2" + "net/http" + "strings" ) type ThunderX struct { @@ -41,26 +40,15 @@ func (x *ThunderX) Init(ctx context.Context) (err error) { if x.XunLeiXCommon == nil { x.XunLeiXCommon = &XunLeiXCommon{ Common: &Common{ - client: base.NewRestyClient(), - Algorithms: []string{ - "lHwINjLeqssT28Ym99p5MvR", - "xvFcxvtqPKCa9Ajf", - "2ywOP8spKHzfuhZMUYZ9IpsViq0t8vT0", - "FTBrJism20SHKQ2m2", - "BHrWJsPwjnr5VeLtOUr2191X9uXhWmt", - "yu0QgHEjNmDoPNwXN17so2hQlDT83T", - "OcaMfLMCGZ7oYlvZGIbTqb4U7cCY", - "jBGGu0GzXOjtCXYwkOBb+c6TZ/Nymv", - "YLWRjVor2rOuYEL", - "94wjoPazejyNC+gRpOj+JOm1XXvxa", - }, + client: base.NewRestyClient(), + Algorithms: Algorithms, DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), - ClientID: "ZQL_zwA4qhHcoe_2", - ClientSecret: "Og9Vr1L8Ee6bh0olFxFDRg", - ClientVersion: "1.05.0.2115", - PackageName: "com.thunder.downloader", - UserAgent: "ANDROID-com.thunder.downloader/1.05.0.2115 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", - DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", + ClientID: ClientID, + ClientSecret: ClientSecret, + ClientVersion: ClientVersion, + PackageName: PackageName, + UserAgent: BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, ""), + DownloadUserAgent: DownloadUserAgent, UseVideoUrl: x.UseVideoUrl, refreshCTokenCk: func(token string) { @@ -76,6 +64,10 @@ func (x *ThunderX) Init(ctx context.Context) (err error) { token, err = x.Login(x.Username, x.Password) if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + if token.UserID != "" { + x.SetUserID(token.UserID) + x.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, token.UserID) + } op.MustSaveDriverStorage(x) } } @@ -86,10 +78,14 @@ func (x *ThunderX) Init(ctx context.Context) (err error) { } // 自定义验证码token - ctoekn := strings.TrimSpace(x.CaptchaToken) - if ctoekn != "" { - x.SetCaptchaToken(ctoekn) + ctoken := strings.TrimSpace(x.CaptchaToken) + if ctoken != "" { + x.SetCaptchaToken(ctoken) + } + if x.DeviceID == "" { + x.SetDeviceID(utils.GetMD5EncodeStr(x.Username + x.Password)) } + x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl x.Addition.RootFolderID = x.RootFolderID // 防止重复登录 @@ -102,6 +98,10 @@ func (x *ThunderX) Init(ctx context.Context) (err error) { return err } x.SetTokenResp(token) + if token.UserID != "" { + x.SetUserID(token.UserID) + x.UserAgent = BuildCustomUserAgent(x.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, token.UserID) + } } return nil } @@ -137,18 +137,33 @@ func (x *ThunderXExpert) Init(ctx context.Context) (err error) { DeviceID: func() string { if len(x.DeviceID) != 32 { - return utils.GetMD5EncodeStr(x.DeviceID) + if x.LoginType == "user" { + return utils.GetMD5EncodeStr(x.Username + x.Password) + } + return utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken) } return x.DeviceID }(), - ClientID: x.ClientID, - ClientSecret: x.ClientSecret, - ClientVersion: x.ClientVersion, - PackageName: x.PackageName, - UserAgent: x.UserAgent, - DownloadUserAgent: x.DownloadUserAgent, - UseVideoUrl: x.UseVideoUrl, - + ClientID: x.ClientID, + ClientSecret: x.ClientSecret, + ClientVersion: x.ClientVersion, + PackageName: x.PackageName, + UserAgent: func() string { + if x.ExpertAddition.UserAgent != "" { + return x.ExpertAddition.UserAgent + } + if x.LoginType == "user" { + return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, "") + } + return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, "") + }(), + DownloadUserAgent: func() string { + if x.ExpertAddition.DownloadUserAgent != "" { + return x.ExpertAddition.DownloadUserAgent + } + return DownloadUserAgent + }(), + UseVideoUrl: x.UseVideoUrl, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) @@ -156,8 +171,17 @@ func (x *ThunderXExpert) Init(ctx context.Context) (err error) { }, } - if x.CaptchaToken != "" { - x.SetCaptchaToken(x.CaptchaToken) + if x.ExpertAddition.CaptchaToken != "" { + x.SetCaptchaToken(x.ExpertAddition.CaptchaToken) + op.MustSaveDriverStorage(x) + } + if x.Common.DeviceID != "" { + x.ExpertAddition.DeviceID = x.Common.DeviceID + op.MustSaveDriverStorage(x) + } + if x.Common.DownloadUserAgent != "" { + x.ExpertAddition.DownloadUserAgent = x.Common.DownloadUserAgent + op.MustSaveDriverStorage(x) } x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl x.ExpertAddition.RootFolderID = x.RootFolderID @@ -177,7 +201,6 @@ func (x *ThunderXExpert) Init(ctx context.Context) (err error) { return err } x.SetTokenResp(token) - // 刷新token方法 x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiXCommon.RefreshToken(x.TokenResp.RefreshToken) @@ -208,13 +231,19 @@ func (x *ThunderXExpert) Init(ctx context.Context) (err error) { return err }) } + // 更新 UserAgent + if x.TokenResp.UserID != "" { + x.ExpertAddition.UserAgent = BuildCustomUserAgent(x.ExpertAddition.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, x.TokenResp.UserID) + x.SetUserAgent(x.ExpertAddition.UserAgent) + op.MustSaveDriverStorage(x) + } } else { // 仅修改验证码token if x.CaptchaToken != "" { x.SetCaptchaToken(x.CaptchaToken) } - x.XunLeiXCommon.UserAgent = x.UserAgent - x.XunLeiXCommon.DownloadUserAgent = x.DownloadUserAgent + x.XunLeiXCommon.UserAgent = x.ExpertAddition.UserAgent + x.XunLeiXCommon.DownloadUserAgent = x.ExpertAddition.UserAgent x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl x.ExpertAddition.RootFolderID = x.RootFolderID } @@ -426,17 +455,17 @@ func (xc *XunLeiXCommon) getFiles(ctx context.Context, folderId string) ([]model return files, nil } -// 设置刷新Token的方法 +// SetRefreshTokenFunc 设置刷新Token的方法 func (xc *XunLeiXCommon) SetRefreshTokenFunc(fn func() error) { xc.refreshTokenFunc = fn } -// 设置Token +// SetTokenResp 设置Token func (xc *XunLeiXCommon) SetTokenResp(tr *TokenResp) { xc.TokenResp = tr } -// 携带Authorization和CaptchaToken的请求 +// Request 携带Authorization和CaptchaToken的请求 func (xc *XunLeiXCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { data, err := xc.Common.Request(url, method, func(req *resty.Request) { req.SetHeaders(map[string]string{ @@ -473,7 +502,7 @@ func (xc *XunLeiXCommon) Request(url string, method string, callback base.ReqCal return xc.Request(url, method, callback, resp) } -// 刷新Token +// RefreshToken 刷新Token func (xc *XunLeiXCommon) RefreshToken(refreshToken string) (*TokenResp, error) { var resp TokenResp _, err := xc.Common.Request(XLUSER_API_URL+"/auth/token", http.MethodPost, func(req *resty.Request) { @@ -491,10 +520,11 @@ func (xc *XunLeiXCommon) RefreshToken(refreshToken string) (*TokenResp, error) { if resp.RefreshToken == "" { return nil, errs.EmptyToken } + resp.UserID = resp.Sub return &resp, nil } -// 登录 +// Login 登录 func (xc *XunLeiXCommon) Login(username, password string) (*TokenResp, error) { url := XLUSER_API_URL + "/auth/signin" err := xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username) diff --git a/drivers/thunderx/meta.go b/drivers/thunderx/meta.go index 2c114c0f284..fa60ebbdb0a 100644 --- a/drivers/thunderx/meta.go +++ b/drivers/thunderx/meta.go @@ -23,7 +23,7 @@ type ExpertAddition struct { RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` // 签名方法1 - Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"lHwINjLeqssT28Ym99p5MvR,xvFcxvtqPKCa9Ajf,2ywOP8spKHzfuhZMUYZ9IpsViq0t8vT0,FTBrJism20SHKQ2m2,BHrWJsPwjnr5VeLtOUr2191X9uXhWmt,yu0QgHEjNmDoPNwXN17so2hQlDT83T,OcaMfLMCGZ7oYlvZGIbTqb4U7cCY,jBGGu0GzXOjtCXYwkOBb+c6TZ/Nymv,YLWRjVor2rOuYEL,94wjoPazejyNC+gRpOj+JOm1XXvxa"` + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"kVy0WbPhiE4v6oxXZ88DvoA3Q,lON/AUoZKj8/nBtcE85mVbkOaVdVa,rLGffQrfBKH0BgwQ33yZofvO3Or,FO6HWqw,GbgvyA2,L1NU9QvIQIH7DTRt,y7llk4Y8WfYflt6,iuDp1WPbV3HRZudZtoXChxH4HNVBX5ZALe,8C28RTXmVcco0,X5Xh,7xe25YUgfGgD0xW3ezFS,,CKCR,8EmDjBo6h3eLaK7U6vU2Qys0NsMx,t2TeZBXKqbdP09Arh9C3"` // 签名方法2 CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` @@ -32,15 +32,15 @@ type ExpertAddition struct { CaptchaToken string `json:"captcha_token"` // 必要且影响登录,由签名决定 - DeviceID string `json:"device_id" required:"true" default:"9aa5c268e7bcfc197a9ad88e2fb330e5"` + DeviceID string `json:"device_id" required:"false" default:""` ClientID string `json:"client_id" required:"true" default:"ZQL_zwA4qhHcoe_2"` ClientSecret string `json:"client_secret" required:"true" default:"Og9Vr1L8Ee6bh0olFxFDRg"` - ClientVersion string `json:"client_version" required:"true" default:"1.05.0.2115"` + ClientVersion string `json:"client_version" required:"true" default:"1.06.0.2132"` PackageName string `json:"package_name" required:"true" default:"com.thunder.downloader"` - //不影响登录,影响下载速度 - UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.thunder.downloader/1.05.0.2115 netWorkType/4G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gdcf98eab238b) (JAVA 0)"` - DownloadUserAgent string `json:"download_user_agent" required:"true" default:"Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)"` + ////不影响登录,影响下载速度 + UserAgent string `json:"user_agent" required:"false" default:""` + DownloadUserAgent string `json:"download_user_agent" required:"false" default:""` //优先使用视频链接代替下载链接 UseVideoUrl bool `json:"use_video_url"` @@ -85,7 +85,7 @@ func (i *Addition) GetIdentity() string { var config = driver.Config{ Name: "ThunderX", LocalSort: true, - OnlyProxy: true, + OnlyProxy: false, } var configExpert = driver.Config{ diff --git a/drivers/thunderx/util.go b/drivers/thunderx/util.go index 6fa323ebc28..661da87e0b0 100644 --- a/drivers/thunderx/util.go +++ b/drivers/thunderx/util.go @@ -1,12 +1,14 @@ package thunderx import ( + "crypto/md5" "crypto/sha1" "encoding/hex" "fmt" "io" "net/http" "regexp" + "strings" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -20,6 +22,33 @@ const ( XLUSER_API_URL = "https://xluser-ssl.xunleix.com/v1" ) +var Algorithms = []string{ + "kVy0WbPhiE4v6oxXZ88DvoA3Q", + "lON/AUoZKj8/nBtcE85mVbkOaVdVa", + "rLGffQrfBKH0BgwQ33yZofvO3Or", + "FO6HWqw", + "GbgvyA2", + "L1NU9QvIQIH7DTRt", + "y7llk4Y8WfYflt6", + "iuDp1WPbV3HRZudZtoXChxH4HNVBX5ZALe", + "8C28RTXmVcco0", + "X5Xh", + "7xe25YUgfGgD0xW3ezFS", + "", + "CKCR", + "8EmDjBo6h3eLaK7U6vU2Qys0NsMx", + "t2TeZBXKqbdP09Arh9C3", +} + +const ( + ClientID = "ZQL_zwA4qhHcoe_2" + ClientSecret = "Og9Vr1L8Ee6bh0olFxFDRg" + ClientVersion = "1.06.0.2132" + PackageName = "com.thunder.downloader" + DownloadUserAgent = "Dalvik/2.1.0 (Linux; U; Android 13; M2004J7AC Build/SP1A.210812.016)" + SdkVersion = "2.0.3.203100 " +) + const ( FOLDER = "drive#folder" FILE = "drive#file" @@ -42,7 +71,7 @@ type Common struct { client *resty.Client captchaToken string - + userID string // 签名相关,二选一 Algorithms []string Timestamp, CaptchaSign string @@ -61,6 +90,18 @@ type Common struct { refreshCTokenCk func(token string) } +func (c *Common) SetDeviceID(deviceID string) { + c.DeviceID = deviceID +} + +func (c *Common) SetUserID(userID string) { + c.userID = userID +} + +func (c *Common) SetUserAgent(userAgent string) { + c.UserAgent = userAgent +} + func (c *Common) SetCaptchaToken(captchaToken string) { c.captchaToken = captchaToken } @@ -145,7 +186,7 @@ func (c *Common) refreshCaptchaToken(action string, metas map[string]string) err return nil } -// 只有基础信息的请求 +// Request 只有基础信息的请求 func (c *Common) Request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := c.client.R().SetHeaders(map[string]string{ "user-agent": c.UserAgent, @@ -200,3 +241,57 @@ func getGcid(r io.Reader, size int64) (string, error) { } return hex.EncodeToString(hash1.Sum(nil)), nil } + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} + +func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { + deviceSign := generateDeviceSign(deviceID, packageName) + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) + sb.WriteString("protocolVersion/200 ") + sb.WriteString("accesstype/ ") + sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) + sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) + sb.WriteString("action_type/ ") + sb.WriteString("networktype/WIFI ") + sb.WriteString("sessionid/ ") + sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) + sb.WriteString("providername/NONE ") + sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) + sb.WriteString("refresh_token/ ") + sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) + sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) + sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) + sb.WriteString(fmt.Sprintf("appname/%s ", appName)) + sb.WriteString(fmt.Sprintf("session_origin/ ")) + sb.WriteString(fmt.Sprintf("grant_type/ ")) + sb.WriteString(fmt.Sprintf("appid/ ")) + sb.WriteString(fmt.Sprintf("clientip/ ")) + sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) + sb.WriteString(fmt.Sprintf("osversion/13 ")) + sb.WriteString(fmt.Sprintf("platformversion/10 ")) + sb.WriteString(fmt.Sprintf("accessmode/ ")) + sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) + + return sb.String() +} From 270587723579f717d6da0c3e5b7131d72f919b1a Mon Sep 17 00:00:00 2001 From: lany Date: Tue, 2 Jul 2024 15:30:00 +0800 Subject: [PATCH 203/659] fix(iLanZou): resolve resource access issue (#6673) * fix(drivers/iLanZou): resolve resource access issue on iLanZou driver mount The driver failed to mount due to incorrect URL parameter ordering which the backend did not accept This commit reorders the parameters to meet the backend's expectations and ensures successful mounting of the iLanZou driver. Closes #6271, Closes #6415 * fix(drivers/iLanZou): Fixed the error ID number returned when creating a folder Closes #6610, Closes #6333 --------- Co-authored-by: maye174 <96584640+maye174@users.noreply.github.com> --- drivers/ilanzou/driver.go | 72 +++++++++++++++++++-------------------- drivers/ilanzou/util.go | 49 +++++++++++++++----------- 2 files changed, 64 insertions(+), 57 deletions(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 63d86363962..ab5ebe7ee5d 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -66,18 +66,18 @@ func (d *ILanZou) Drop(ctx context.Context) error { } func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - offset := 1 - limit := 60 var res []ListItem for { var resp ListResp _, err := d.proved("/record/file/list", http.MethodGet, func(req *resty.Request) { - req.SetQueryParams(map[string]string{ - "type": "0", - "folderId": dir.GetID(), - "offset": strconv.Itoa(offset), - "limit": strconv.Itoa(limit), - }).SetResult(&resp) + params := []string{ + "offset=1", + "limit=60", + "folderId=" + dir.GetID(), + "type=0", + } + queryString := strings.Join(params, "&") + req.SetQueryString(queryString).SetResult(&resp) }) if err != nil { return nil, err @@ -86,7 +86,6 @@ func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) if resp.TotalPage <= resp.Offset { break } - offset++ } return utils.SliceConvert(res, func(f ListItem) (model.Obj, error) { updTime, err := time.ParseInLocation("2006-01-02 15:04:05", f.UpdTime, time.Local) @@ -118,31 +117,33 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err != nil { return nil, err } - query := u.Query() - query.Set("uuid", d.UUID) - query.Set("devType", "6") - query.Set("devCode", d.UUID) - query.Set("devModel", "chrome") - query.Set("devVersion", d.conf.devVersion) - query.Set("appVersion", "") - ts, err := getTimestamp(d.conf.secret) - if err != nil { - return nil, err + ts, ts_str, err := getTimestamp(d.conf.secret) + + params := []string{ + "uuid=" + url.QueryEscape(d.UUID), + "devType=6", + "devCode=" + url.QueryEscape(d.UUID), + "devModel=chrome", + "devVersion=" + url.QueryEscape(d.conf.devVersion), + "appVersion=", + "timestamp=" + ts_str, + "appToken=" + url.QueryEscape(d.Token), + "enable=0", } - query.Set("timestamp", ts) - query.Set("appToken", d.Token) - query.Set("enable", "1") + downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) if err != nil { return nil, err } - query.Set("downloadId", hex.EncodeToString(downloadId)) - auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), time.Now().UnixMilli())), d.conf.secret) + params = append(params, "downloadId="+url.QueryEscape(hex.EncodeToString(downloadId))) + + auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), ts)), d.conf.secret) if err != nil { return nil, err } - query.Set("auth", hex.EncodeToString(auth)) - u.RawQuery = query.Encode() + params = append(params, "auth="+url.QueryEscape(hex.EncodeToString(auth))) + + u.RawQuery = strings.Join(params, "&") realURL := u.String() // get the url after redirect res, err := base.NoRedirectClient.R().SetHeaders(map[string]string{ @@ -156,12 +157,7 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if res.StatusCode() == 302 { realURL = res.Header().Get("location") } else { - contentLengthStr := res.Header().Get("Content-Length") - contentLength, err := strconv.Atoi(contentLengthStr) - if err != nil || contentLength == 0 || contentLength > 1024*10 { - return nil, fmt.Errorf("redirect failed, status: %d", res.StatusCode()) - } - return nil, fmt.Errorf("redirect failed, content: %s", res.String()) + return nil, fmt.Errorf("redirect failed, status: %d, msg: %s", res.StatusCode(), utils.Json.Get(res.Body(), "msg").ToString()) } link := model.Link{URL: realURL} return &link, nil @@ -179,7 +175,7 @@ func (d *ILanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri return nil, err } return &model.Object{ - ID: utils.Json.Get(res, "list", "0", "id").ToString(), + ID: utils.Json.Get(res, "list", 0, "id").ToString(), //Path: "", Name: dirName, Size: 0, @@ -348,10 +344,12 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt var resp UploadResultResp for i := 0; i < 10; i++ { _, err = d.unproved("/7n/results", http.MethodPost, func(req *resty.Request) { - req.SetQueryParams(map[string]string{ - "tokenList": token, - "tokenTime": time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)"), - }).SetResult(&resp) + params := []string{ + "tokenList=" + token, + "tokenTime=" + time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)"), + } + queryString := strings.Join(params, "&") + req.SetQueryString(queryString).SetResult(&resp) }) if err != nil { return nil, err diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index c9a30765b7f..a57e2a4a6be 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -4,7 +4,9 @@ import ( "encoding/hex" "fmt" "net/http" + "net/url" "strconv" + "strings" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -31,45 +33,52 @@ func (d *ILanZou) login() error { return nil } -func getTimestamp(secret []byte) (string, error) { +func getTimestamp(secret []byte) (int64, string, error) { ts := time.Now().UnixMilli() tsStr := strconv.FormatInt(ts, 10) res, err := mopan.AesEncrypt([]byte(tsStr), secret) if err != nil { - return "", err + return 0, "", err } - return hex.EncodeToString(res), nil + return ts, hex.EncodeToString(res), nil } func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, proved bool, retry ...bool) ([]byte, error) { - req := base.RestyClient.R() - ts, err := getTimestamp(d.conf.secret) + _, ts_str, err := getTimestamp(d.conf.secret) if err != nil { return nil, err } - req.SetQueryParams(map[string]string{ - "uuid": d.UUID, - "devType": "6", - "devCode": d.UUID, - "devModel": "chrome", - "devVersion": d.conf.devVersion, - "appVersion": "", - "timestamp": ts, - //"appToken": d.Token, - "extra": "2", - }) + + params := []string{ + "uuid=" + url.QueryEscape(d.UUID), + "devType=6", + "devCode=" + url.QueryEscape(d.UUID), + "devModel=chrome", + "devVersion=" + url.QueryEscape(d.conf.devVersion), + "appVersion=", + "timestamp=" + ts_str, + } + + if proved { + params = append(params, "appToken="+url.QueryEscape(d.Token)) + } + + params = append(params, "extra=2") + + queryString := strings.Join(params, "&") + + req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Origin": d.conf.site, "Referer": d.conf.site + "/", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", }) - if proved { - req.SetQueryParam("appToken", d.Token) - } + if callback != nil { callback(req) } - res, err := req.Execute(method, d.conf.base+pathname) + + res, err := req.Execute(method, d.conf.base+pathname+"?"+queryString) if err != nil { if res != nil { log.Errorf("[iLanZou] request error: %s", res.String()) From 316f3569a5df2790093f49788aa2730317093cdd Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sun, 7 Jul 2024 13:19:19 +0800 Subject: [PATCH 204/659] feat(thunderBrowser): add automatically generate UserAgent (#6692) --- drivers/thunder_browser/driver.go | 118 ++++++++++++++++++------------ drivers/thunder_browser/meta.go | 20 ++--- drivers/thunder_browser/util.go | 69 +++++++++++++++++ 3 files changed, 150 insertions(+), 57 deletions(-) diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index f3a08f93d54..a389f6102fc 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -2,10 +2,10 @@ package thunder_browser import ( "context" + "errors" "fmt" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -53,33 +53,17 @@ func (x *ThunderBrowser) Init(ctx context.Context) (err error) { if x.XunLeiBrowserCommon == nil { x.XunLeiBrowserCommon = &XunLeiBrowserCommon{ Common: &Common{ - client: base.NewRestyClient(), - Algorithms: []string{ - "x+I5XiTByg", - "6QU1x5DqGAV3JKg6h", - "VI1vL1WXr7st0es", - "n+/3yhlrnKs4ewhLgZhZ5ITpt554", - "UOip2PE7BLIEov/ZX6VOnsz", - "Q70h9lpViNCOC8sGVkar9o22LhBTjfP", - "IVHFuB1JcMlaZHnW", - "bKE", - "HZRbwxOiQx+diNopi6Nu", - "fwyasXgYL3rP314331b", - "LWxXAiSW4", - "UlWIjv1HGrC6Ngmt4Nohx", - "FOa+Lc0bxTDpTwIh2", - "0+RY", - "xmRVMqokHHpvsiH0", - }, + client: base.NewRestyClient(), + Algorithms: Algorithms, DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), - ClientID: "ZUBzD9J_XPXfn7f7", - ClientSecret: "yESVmHecEe6F0aou69vl-g", - ClientVersion: "1.0.7.1938", - PackageName: "com.xunlei.browser", - UserAgent: "ANDROID-com.xunlei.browser/1.0.7.1938 netWorkType/5G appid/22062 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/233100 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", - DownloadUserAgent: "AndroidDownloadManager/12 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", + ClientID: ClientID, + ClientSecret: ClientSecret, + ClientVersion: ClientVersion, + PackageName: PackageName, + UserAgent: BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), PackageName, SdkVersion, ClientVersion, PackageName), + DownloadUserAgent: DownloadUserAgent, UseVideoUrl: x.UseVideoUrl, - + RemoveWay: x.Addition.RemoveWay, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) @@ -107,6 +91,9 @@ func (x *ThunderBrowser) Init(ctx context.Context) (err error) { if ctoekn != "" { x.SetCaptchaToken(ctoekn) } + if x.DeviceID == "" { + x.SetDeviceID(utils.GetMD5EncodeStr(x.Username + x.Password)) + } x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl x.Addition.RootFolderID = x.RootFolderID // 防止重复登录 @@ -170,21 +157,36 @@ func (x *ThunderBrowserExpert) Init(ctx context.Context) (err error) { x.XunLeiBrowserCommon = &XunLeiBrowserCommon{ Common: &Common{ client: base.NewRestyClient(), - DeviceID: func() string { if len(x.DeviceID) != 32 { - return utils.GetMD5EncodeStr(x.DeviceID) + if x.LoginType == "user" { + return utils.GetMD5EncodeStr(x.Username + x.Password) + } + return utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken) } return x.DeviceID }(), - ClientID: x.ClientID, - ClientSecret: x.ClientSecret, - ClientVersion: x.ClientVersion, - PackageName: x.PackageName, - UserAgent: x.UserAgent, - DownloadUserAgent: x.DownloadUserAgent, - UseVideoUrl: x.UseVideoUrl, - + ClientID: x.ClientID, + ClientSecret: x.ClientSecret, + ClientVersion: x.ClientVersion, + PackageName: x.PackageName, + UserAgent: func() string { + if x.ExpertAddition.UserAgent != "" { + return x.ExpertAddition.UserAgent + } + if x.LoginType == "user" { + return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), x.PackageName, SdkVersion, x.ClientVersion, x.PackageName) + } + return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken), x.PackageName, SdkVersion, x.ClientVersion, x.PackageName) + }(), + DownloadUserAgent: func() string { + if x.ExpertAddition.DownloadUserAgent != "" { + return x.ExpertAddition.DownloadUserAgent + } + return DownloadUserAgent + }(), + UseVideoUrl: x.UseVideoUrl, + RemoveWay: x.ExpertAddition.RemoveWay, refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) @@ -192,8 +194,21 @@ func (x *ThunderBrowserExpert) Init(ctx context.Context) (err error) { }, } - if x.CaptchaToken != "" { - x.SetCaptchaToken(x.CaptchaToken) + if x.ExpertAddition.CaptchaToken != "" { + x.SetCaptchaToken(x.ExpertAddition.CaptchaToken) + op.MustSaveDriverStorage(x) + } + if x.Common.DeviceID != "" { + x.ExpertAddition.DeviceID = x.Common.DeviceID + op.MustSaveDriverStorage(x) + } + if x.Common.UserAgent != "" { + x.ExpertAddition.UserAgent = x.Common.UserAgent + op.MustSaveDriverStorage(x) + } + if x.Common.DownloadUserAgent != "" { + x.ExpertAddition.DownloadUserAgent = x.Common.DownloadUserAgent + op.MustSaveDriverStorage(x) } x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl x.ExpertAddition.RootFolderID = x.RootFolderID @@ -488,7 +503,8 @@ func (xc *XunLeiBrowserCommon) Remove(ctx context.Context, obj model.Obj) error } } - if xc.RemoveWay == "delete" && obj.GetPath() == ThunderDriveFileID { + // 先判断是否是特殊情况 + if obj.GetPath() == ThunderDriveFileID { _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", obj.GetID()) @@ -503,12 +519,20 @@ func (xc *XunLeiBrowserCommon) Remove(ctx context.Context, obj model.Obj) error return err } - _, err := xc.Request(FILE_API_URL+":batchTrash", http.MethodPost, func(r *resty.Request) { - r.SetContext(ctx) - r.SetBody(&js) - }, nil) - return err - + // 根据用户选择的删除方式进行删除 + if xc.RemoveWay == "delete" { + _, err := xc.Request(FILE_API_URL+":batchDelete", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err + } else { + _, err := xc.Request(FILE_API_URL+":batchTrash", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err + } } func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { @@ -756,7 +780,7 @@ func (xc *XunLeiBrowserCommon) RefreshToken(refreshToken string) (*TokenResp, er } if resp.RefreshToken == "" { - return nil, errs.EmptyToken + return nil, errors.New("refresh token is empty") } return &resp, nil } @@ -775,7 +799,7 @@ func (xc *XunLeiBrowserCommon) GetSafeAccessToken(safePassword string) (string, } if resp.Token == "" { - return "", errs.EmptyToken + return "", errors.New("SafePassword is incorrect ") } return resp.Token, nil } diff --git a/drivers/thunder_browser/meta.go b/drivers/thunder_browser/meta.go index 9d16cd78e5c..247353b7b0f 100644 --- a/drivers/thunder_browser/meta.go +++ b/drivers/thunder_browser/meta.go @@ -17,14 +17,15 @@ type ExpertAddition struct { SignType string `json:"sign_type" type:"select" options:"algorithms,captcha_sign" default:"algorithms"` // 登录方式1 - Username string `json:"username" required:"true" help:"login type is user,this is required"` - Password string `json:"password" required:"true" help:"login type is user,this is required"` - SafePassword string `json:"safe_password" required:"false" help:"login type is user,this is required"` // 超级保险箱密码 + Username string `json:"username" required:"true" help:"login type is user,this is required"` + Password string `json:"password" required:"true" help:"login type is user,this is required"` // 登录方式2 RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` + SafePassword string `json:"safe_password" required:"true" help:"super safe password"` // 超级保险箱密码 + // 签名方法1 - Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"x+I5XiTByg,6QU1x5DqGAV3JKg6h,VI1vL1WXr7st0es,n+/3yhlrnKs4ewhLgZhZ5ITpt554,UOip2PE7BLIEov/ZX6VOnsz,Q70h9lpViNCOC8sGVkar9o22LhBTjfP,IVHFuB1JcMlaZHnW,bKE,HZRbwxOiQx+diNopi6Nu,fwyasXgYL3rP314331b,LWxXAiSW4,UlWIjv1HGrC6Ngmt4Nohx,FOa+Lc0bxTDpTwIh2,0+RY,xmRVMqokHHpvsiH0"` + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"p+ExqPV,LwdwKlprzv7cQBQmxN5,vc08P1NwUBnbGsl58LzTW,VVNeXaXmZ8HH1SJEnp6YpVFSFU,pNAOJ,CNChvyDehAmUR1TDodfOusBAx,MS98NnX4Np8nxvEh6Ulv+SMMKMzKvD34C7lGWbb,9MpFF21GnVOYku0NM9Y/hzsK471UCUZ2o+,EY1QfeA06fXlw9wZNoZaXEED5zZPvNWI,,sciE,FIPqgQDUUW1e0GkiBFd5w7mCQ,zW,75XFdEO0Gi"` // 签名方法2 CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` @@ -33,15 +34,15 @@ type ExpertAddition struct { CaptchaToken string `json:"captcha_token"` // 必要且影响登录,由签名决定 - DeviceID string `json:"device_id" required:"true" default:"9aa5c268e7bcfc197a9ad88e2fb330e5"` + DeviceID string `json:"device_id" required:"false" default:""` ClientID string `json:"client_id" required:"true" default:"ZUBzD9J_XPXfn7f7"` ClientSecret string `json:"client_secret" required:"true" default:"yESVmHecEe6F0aou69vl-g"` - ClientVersion string `json:"client_version" required:"true" default:"1.0.7.1938"` + ClientVersion string `json:"client_version" required:"true" default:"1.0.8.2215"` PackageName string `json:"package_name" required:"true" default:"com.xunlei.browser"` // 不影响登录,影响下载速度 - UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.xunlei.browser/1.0.7.1938 netWorkType/5G appid/22062 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/233100 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)"` - DownloadUserAgent string `json:"download_user_agent" required:"true" default:"AndroidDownloadManager/12 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)"` + UserAgent string `json:"user_agent" required:"false" default:""` + DownloadUserAgent string `json:"download_user_agent" required:"false" default:""` // 优先使用视频链接代替下载链接 UseVideoUrl bool `json:"use_video_url"` @@ -76,7 +77,7 @@ type Addition struct { driver.RootID Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` - SafePassword string `json:"safe_password" required:"false"` // 超级保险箱密码 + SafePassword string `json:"safe_password" required:"true"` // 超级保险箱密码 CaptchaToken string `json:"captcha_token"` UseVideoUrl bool `json:"use_video_url" default:"false"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` @@ -90,7 +91,6 @@ func (i *Addition) GetIdentity() string { var config = driver.Config{ Name: "ThunderBrowser", LocalSort: true, - OnlyProxy: true, } var configExpert = driver.Config{ diff --git a/drivers/thunder_browser/util.go b/drivers/thunder_browser/util.go index fd8a4047b1b..a5f6f663e8f 100644 --- a/drivers/thunder_browser/util.go +++ b/drivers/thunder_browser/util.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "regexp" + "strings" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -21,6 +22,32 @@ const ( XLUSER_API_URL = "https://xluser-ssl.xunlei.com/v1" ) +var Algorithms = []string{ + "p+ExqPV", + "LwdwKlprzv7cQBQmxN5", + "vc08P1NwUBnbGsl58LzTW", + "VVNeXaXmZ8HH1SJEnp6YpVFSFU", + "pNAOJ", + "CNChvyDehAmUR1TDodfOusBAx", + "MS98NnX4Np8nxvEh6Ulv+SMMKMzKvD34C7lGWbb", + "9MpFF21GnVOYku0NM9Y/hzsK471UCUZ2o+", + "EY1QfeA06fXlw9wZNoZaXEED5zZPvNWI", + "", + "sciE", + "FIPqgQDUUW1e0GkiBFd5w7mCQ", + "zW", + "75XFdEO0Gi", +} + +const ( + ClientID = "ZUBzD9J_XPXfn7f7" + ClientSecret = "yESVmHecEe6F0aou69vl-g" + ClientVersion = "1.0.8.2215" + PackageName = "com.xunlei.browser" + DownloadUserAgent = "AndroidDownloadManager/13 (Linux; U; Android 13; M2004J7AC Build/SP1A.210812.016)" + SdkVersion = "2.0.3.262" +) + const ( FOLDER = "drive#folder" FILE = "drive#file" @@ -74,6 +101,10 @@ type Common struct { refreshCTokenCk func(token string) } +func (c *Common) SetDeviceID(deviceID string) { + c.DeviceID = deviceID +} + func (c *Common) SetCaptchaToken(captchaToken string) { c.captchaToken = captchaToken } @@ -247,3 +278,41 @@ func EncryptPassword(password string) string { // 将哈希值转换为十六进制字符串 return hex.EncodeToString(hash[:]) } + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "22062", "a5d7416858147a4ab99573872ffccef8") + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} + +func BuildCustomUserAgent(deviceID, appName, sdkVersion, clientVersion, packageName string) string { + //deviceSign := generateDeviceSign(deviceID, packageName) + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) + sb.WriteString("networkType/WIFI ") + sb.WriteString(fmt.Sprintf("appid/%s ", "22062")) + sb.WriteString(fmt.Sprintf("deviceName/Xiaomi_M2004j7ac ")) + sb.WriteString(fmt.Sprintf("deviceModel/M2004J7AC ")) + sb.WriteString(fmt.Sprintf("OSVersion/13 ")) + sb.WriteString(fmt.Sprintf("protocolVersion/301 ")) + sb.WriteString(fmt.Sprintf("platformversion/10 ")) + sb.WriteString(fmt.Sprintf("sdkVersion/%s ", sdkVersion)) + sb.WriteString(fmt.Sprintf("Oauth2Client/0.9 (Linux 4_9_337-perf-sn-uotan-gd9d488809c3d) (JAVA 0) ")) + return sb.String() +} From ca30849e24257572dbee0e2700ac387fc4eb6620 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sun, 7 Jul 2024 13:20:34 +0800 Subject: [PATCH 205/659] feat: add support for halalcloud driver (#6696) --- drivers/all.go | 1 + drivers/halalcloud/driver.go | 406 ++++++++++++++++++++++++++++++++++ drivers/halalcloud/meta.go | 38 ++++ drivers/halalcloud/options.go | 52 +++++ drivers/halalcloud/types.go | 101 +++++++++ drivers/halalcloud/util.go | 385 ++++++++++++++++++++++++++++++++ go.mod | 46 ++-- go.sum | 108 +++++---- pkg/utils/io.go | 2 +- 9 files changed, 1075 insertions(+), 64 deletions(-) create mode 100644 drivers/halalcloud/driver.go create mode 100644 drivers/halalcloud/meta.go create mode 100644 drivers/halalcloud/options.go create mode 100644 drivers/halalcloud/types.go create mode 100644 drivers/halalcloud/util.go diff --git a/drivers/all.go b/drivers/all.go index df8a1ffc72a..785278cf908 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -25,6 +25,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" + _ "github.com/alist-org/alist/v3/drivers/halalcloud" _ "github.com/alist-org/alist/v3/drivers/ilanzou" _ "github.com/alist-org/alist/v3/drivers/ipfs_api" _ "github.com/alist-org/alist/v3/drivers/lanzou" diff --git a/drivers/halalcloud/driver.go b/drivers/halalcloud/driver.go new file mode 100644 index 00000000000..f99b5f6f412 --- /dev/null +++ b/drivers/halalcloud/driver.go @@ -0,0 +1,406 @@ +package halalcloud + +import ( + "context" + "crypto/sha1" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/city404/v6-public-rpc-proto/go/v6/common" + pbPublicUser "github.com/city404/v6-public-rpc-proto/go/v6/user" + pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" + "github.com/rclone/rclone/lib/readers" + "github.com/zzzhr1990/go-common-entity/userfile" + "io" + "net/url" + "path" + "strconv" + "time" +) + +type HalalCloud struct { + *HalalCommon + model.Storage + Addition + + uploadThread int +} + +func (d *HalalCloud) Config() driver.Config { + return config +} + +func (d *HalalCloud) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *HalalCloud) Init(ctx context.Context) error { + d.uploadThread, _ = strconv.Atoi(d.UploadThread) + if d.uploadThread < 1 || d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 3, "3" + } + + if d.HalalCommon == nil { + d.HalalCommon = &HalalCommon{ + Common: &Common{}, + AuthService: &AuthService{ + appID: func() string { + if d.Addition.AppID != "" { + return d.Addition.AppID + } + return AppID + }(), + appVersion: func() string { + if d.Addition.AppVersion != "" { + return d.Addition.AppVersion + } + return AppVersion + }(), + appSecret: func() string { + if d.Addition.AppSecret != "" { + return d.Addition.AppSecret + } + return AppSecret + }(), + tr: &TokenResp{ + RefreshToken: d.Addition.RefreshToken, + }, + }, + UserInfo: &UserInfo{}, + refreshTokenFunc: func(token string) error { + d.Addition.RefreshToken = token + op.MustSaveDriverStorage(d) + return nil + }, + } + } + + // 防止重复登录 + if d.Addition.RefreshToken == "" || !d.IsLogin() { + as, err := d.NewAuthServiceWithOauth() + if err != nil { + d.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + return err + } + d.HalalCommon.AuthService = as + d.SetTokenResp(as.tr) + op.MustSaveDriverStorage(d) + } + var err error + d.HalalCommon.serv, err = d.NewAuthService(d.Addition.RefreshToken) + if err != nil { + return err + } + + return nil +} + +func (d *HalalCloud) Drop(ctx context.Context) error { + return nil +} + +func (d *HalalCloud) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.getFiles(ctx, dir) +} + +func (d *HalalCloud) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return d.getLink(ctx, file, args) +} + +func (d *HalalCloud) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return d.makeDir(ctx, parentDir, dirName) +} + +func (d *HalalCloud) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.move(ctx, srcObj, dstDir) +} + +func (d *HalalCloud) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return d.rename(ctx, srcObj, newName) +} + +func (d *HalalCloud) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.copy(ctx, srcObj, dstDir) +} + +func (d *HalalCloud) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(ctx, obj) +} + +func (d *HalalCloud) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return d.put(ctx, dstDir, stream, up) +} + +func (d *HalalCloud) IsLogin() bool { + if d.AuthService.tr == nil { + return false + } + serv, err := d.NewAuthService(d.Addition.RefreshToken) + if err != nil { + return false + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + result, err := pbPublicUser.NewPubUserClient(serv.GetGrpcConnection()).Get(ctx, &pbPublicUser.User{ + Identity: "", + }) + if result == nil || err != nil { + return false + } + d.UserInfo.Identity = result.Identity + d.UserInfo.CreateTs = result.CreateTs + d.UserInfo.Name = result.Name + d.UserInfo.UpdateTs = result.UpdateTs + return true +} + +type HalalCommon struct { + *Common + *AuthService // 登录信息 + *UserInfo // 用户信息 + refreshTokenFunc func(token string) error + serv *AuthService +} + +func (d *HalalCloud) SetTokenResp(tr *TokenResp) { + d.Addition.RefreshToken = tr.RefreshToken +} + +func (d *HalalCloud) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj, error) { + + files := make([]model.Obj, 0) + limit := int64(100) + token := "" + client := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()) + + opDir := d.GetCurrentDir(dir) + + for { + result, err := client.List(ctx, &pubUserFile.FileListRequest{ + Parent: &pubUserFile.File{Path: opDir}, + ListInfo: &common.ScanListRequest{ + Limit: limit, + Token: token, + }, + }) + if err != nil { + return nil, err + } + + for i := 0; len(result.Files) > i; i++ { + files = append(files, (*Files)(result.Files[i])) + } + + if result.ListInfo == nil || result.ListInfo.Token == "" { + break + } + token = result.ListInfo.Token + + } + return files, nil +} + +func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + + client := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()) + ctx1, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + result, err := client.ParseFileSlice(ctx1, (*pubUserFile.File)(file.(*Files))) + if err != nil { + return nil, err + } + fileAddrs := []*pubUserFile.SliceDownloadInfo{} + var addressDuration int64 + + nodesNumber := len(result.RawNodes) + nodesIndex := nodesNumber - 1 + startIndex, endIndex := 0, nodesIndex + for nodesIndex >= 0 { + if nodesIndex >= 200 { + endIndex = 200 + } else { + endIndex = nodesNumber + } + for ; endIndex <= nodesNumber; endIndex += 200 { + if endIndex == 0 { + endIndex = 1 + } + sliceAddress, err := client.GetSliceDownloadAddress(ctx, &pubUserFile.SliceDownloadAddressRequest{ + Identity: result.RawNodes[startIndex:endIndex], + Version: 1, + }) + if err != nil { + return nil, err + } + addressDuration = sliceAddress.ExpireAt + fileAddrs = append(fileAddrs, sliceAddress.Addresses...) + startIndex = endIndex + nodesIndex -= 200 + } + + } + + size := result.FileSize + chunks := getChunkSizes(result.Sizes) + var finalClosers utils.Closers + resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + length := httpRange.Length + if httpRange.Length >= 0 && httpRange.Start+httpRange.Length >= size { + length = -1 + } + if err != nil { + return nil, fmt.Errorf("open download file failed: %w", err) + } + oo := &openObject{ + ctx: ctx, + d: fileAddrs, + chunk: &[]byte{}, + chunks: &chunks, + skip: httpRange.Start, + sha: result.Sha1, + shaTemp: sha1.New(), + } + finalClosers.Add(oo) + + return readers.NewLimitedReadCloser(oo, length), nil + } + + var duration time.Duration + if addressDuration != 0 { + duration = time.Until(time.UnixMilli(addressDuration)) + } else { + duration = time.Until(time.Now().Add(time.Hour)) + } + + resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: finalClosers} + return &model.Link{ + RangeReadCloser: resultRangeReadCloser, + Expiration: &duration, + }, nil +} + +func (d *HalalCloud) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) { + newDir := userfile.NewFormattedPath(d.GetCurrentOpDir(dir, []string{name}, 0)).GetPath() + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Create(ctx, &pubUserFile.File{ + Path: newDir, + }) + return nil, err +} + +func (d *HalalCloud) move(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { + oldDir := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() + newDir := userfile.NewFormattedPath(d.GetCurrentDir(dir)).GetPath() + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Move(ctx, &pubUserFile.BatchOperationRequest{ + Source: []*pubUserFile.File{ + { + Identity: obj.GetID(), + Path: oldDir, + }, + }, + Dest: &pubUserFile.File{ + Identity: dir.GetID(), + Path: newDir, + }, + }) + return nil, err +} + +func (d *HalalCloud) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) { + id := obj.GetID() + newPath := userfile.NewFormattedPath(d.GetCurrentOpDir(obj, []string{name}, 0)).GetPath() + + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Rename(ctx, &pubUserFile.File{ + Path: newPath, + Identity: id, + Name: name, + }) + return nil, err +} + +func (d *HalalCloud) copy(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { + id := obj.GetID() + sourcePath := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() + if len(id) > 0 { + sourcePath = "" + } + dest := &pubUserFile.File{ + Identity: dir.GetID(), + Path: userfile.NewFormattedPath(d.GetCurrentDir(dir)).GetPath(), + } + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Copy(ctx, &pubUserFile.BatchOperationRequest{ + Source: []*pubUserFile.File{ + { + Path: sourcePath, + Identity: id, + }, + }, + Dest: dest, + }) + return nil, err +} + +func (d *HalalCloud) remove(ctx context.Context, obj model.Obj) error { + id := obj.GetID() + newPath := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() + //if len(id) > 0 { + // newPath = "" + //} + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Delete(ctx, &pubUserFile.BatchOperationRequest{ + Source: []*pubUserFile.File{ + { + Path: newPath, + Identity: id, + }, + }, + }) + return err +} + +func (d *HalalCloud) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + + newDir := path.Join(dstDir.GetPath(), fileStream.GetName()) + + result, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).CreateUploadToken(ctx, &pubUserFile.File{ + Path: newDir, + }) + if err != nil { + return nil, err + } + u, _ := url.Parse(result.Endpoint) + u.Host = "s3." + u.Host + result.Endpoint = u.String() + s, err := session.NewSession(&aws.Config{ + HTTPClient: base.HttpClient, + Credentials: credentials.NewStaticCredentials(result.AccessKey, result.SecretKey, result.Token), + Region: aws.String(result.Region), + Endpoint: aws.String(result.Endpoint), + S3ForcePathStyle: aws.Bool(true), + }) + if err != nil { + return nil, err + } + uploader := s3manager.NewUploader(s, func(u *s3manager.Uploader) { + u.Concurrency = d.uploadThread + }) + if fileStream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = fileStream.GetSize() / (s3manager.MaxUploadParts - 1) + } + _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(result.Bucket), + Key: aws.String(result.Key), + Body: io.TeeReader(fileStream, driver.NewProgress(fileStream.GetSize(), up)), + }) + return nil, err + +} + +var _ driver.Driver = (*HalalCloud)(nil) diff --git a/drivers/halalcloud/meta.go b/drivers/halalcloud/meta.go new file mode 100644 index 00000000000..b60445c061d --- /dev/null +++ b/drivers/halalcloud/meta.go @@ -0,0 +1,38 @@ +package halalcloud + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // define other + RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` + UploadThread string `json:"upload_thread" default:"3" help:"1 <= thread <= 32"` + + AppID string `json:"app_id" required:"true" default:"devDebugger/1.0"` + AppVersion string `json:"app_version" required:"true" default:"1.0.0"` + AppSecret string `json:"app_secret" required:"true" default:"Nkx3Y2xvZ2luLmNu"` +} + +var config = driver.Config{ + Name: "HalalCloud", + LocalSort: false, + OnlyLocal: true, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &HalalCloud{} + }) +} diff --git a/drivers/halalcloud/options.go b/drivers/halalcloud/options.go new file mode 100644 index 00000000000..56e5fdc5c09 --- /dev/null +++ b/drivers/halalcloud/options.go @@ -0,0 +1,52 @@ +package halalcloud + +import "google.golang.org/grpc" + +func defaultOptions() halalOptions { + return halalOptions{ + // onRefreshTokenRefreshed: func(string) {}, + grpcOptions: []grpc.DialOption{ + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(1024 * 1024 * 32)), + // grpc.WithMaxMsgSize(1024 * 1024 * 1024), + }, + } +} + +type HalalOption interface { + apply(*halalOptions) +} + +// halalOptions configure a RPC call. halalOptions are set by the HalalOption +// values passed to Dial. +type halalOptions struct { + onTokenRefreshed func(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64) + grpcOptions []grpc.DialOption +} + +// funcDialOption wraps a function that modifies halalOptions into an +// implementation of the DialOption interface. +type funcDialOption struct { + f func(*halalOptions) +} + +func (fdo *funcDialOption) apply(do *halalOptions) { + fdo.f(do) +} + +func newFuncDialOption(f func(*halalOptions)) *funcDialOption { + return &funcDialOption{ + f: f, + } +} + +func WithRefreshTokenRefreshedCallback(s func(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64)) HalalOption { + return newFuncDialOption(func(o *halalOptions) { + o.onTokenRefreshed = s + }) +} + +func WithGrpcDialOptions(opts ...grpc.DialOption) HalalOption { + return newFuncDialOption(func(o *halalOptions) { + o.grpcOptions = opts + }) +} diff --git a/drivers/halalcloud/types.go b/drivers/halalcloud/types.go new file mode 100644 index 00000000000..9772421264b --- /dev/null +++ b/drivers/halalcloud/types.go @@ -0,0 +1,101 @@ +package halalcloud + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/city404/v6-public-rpc-proto/go/v6/common" + pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" + "google.golang.org/grpc" + "time" +) + +type AuthService struct { + appID string + appVersion string + appSecret string + grpcConnection *grpc.ClientConn + dopts halalOptions + tr *TokenResp +} + +type TokenResp struct { + AccessToken string `json:"accessToken,omitempty"` + AccessTokenExpiredAt int64 `json:"accessTokenExpiredAt,omitempty"` + RefreshToken string `json:"refreshToken,omitempty"` + RefreshTokenExpiredAt int64 `json:"refreshTokenExpiredAt,omitempty"` +} + +type UserInfo struct { + Identity string `json:"identity,omitempty"` + UpdateTs int64 `json:"updateTs,omitempty"` + Name string `json:"name,omitempty"` + CreateTs int64 `json:"createTs,omitempty"` +} + +type OrderByInfo struct { + Field string `json:"field,omitempty"` + Asc bool `json:"asc,omitempty"` +} + +type ListInfo struct { + Token string `json:"token,omitempty"` + Limit int64 `json:"limit,omitempty"` + OrderBy []*OrderByInfo `json:"order_by,omitempty"` + Version int32 `json:"version,omitempty"` +} + +type FilesList struct { + Files []*Files `json:"files,omitempty"` + ListInfo *common.ScanListRequest `json:"list_info,omitempty"` +} + +var _ model.Obj = (*Files)(nil) + +type Files pubUserFile.File + +func (f *Files) GetSize() int64 { + return f.Size +} + +func (f *Files) GetName() string { + return f.Name +} + +func (f *Files) ModTime() time.Time { + return time.UnixMilli(f.UpdateTs) +} + +func (f *Files) CreateTime() time.Time { + return time.UnixMilli(f.UpdateTs) +} + +func (f *Files) IsDir() bool { + return f.Dir +} + +func (f *Files) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f *Files) GetID() string { + if len(f.Identity) == 0 { + f.Identity = "/" + } + return f.Identity +} + +func (f *Files) GetPath() string { + return f.Path +} + +type SteamFile struct { + file model.File +} + +func (s *SteamFile) Read(p []byte) (n int, err error) { + return s.file.Read(p) +} + +func (s *SteamFile) Close() error { + return s.file.Close() +} diff --git a/drivers/halalcloud/util.go b/drivers/halalcloud/util.go new file mode 100644 index 00000000000..33a347e753e --- /dev/null +++ b/drivers/halalcloud/util.go @@ -0,0 +1,385 @@ +package halalcloud + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/tls" + "encoding/hex" + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + pbPublicUser "github.com/city404/v6-public-rpc-proto/go/v6/user" + pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" + "github.com/google/uuid" + "github.com/ipfs/go-cid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "hash" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +const ( + AppID = "devDebugger/1.0" + AppVersion = "1.0.0" + AppSecret = "Nkx3Y2xvZ2luLmNu" +) + +const ( + grpcServer = "grpcuserapi.2dland.cn:443" + grpcServerAuth = "grpcuserapi.2dland.cn" +) + +func (d *HalalCloud) NewAuthServiceWithOauth(options ...HalalOption) (*AuthService, error) { + + aService := &AuthService{} + err2 := errors.New("") + + svc := d.HalalCommon.AuthService + for _, opt := range options { + opt.apply(&svc.dopts) + } + + grpcOptions := svc.dopts.grpcOptions + grpcOptions = append(grpcOptions, grpc.WithAuthority(grpcServerAuth), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})), grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctxx := svc.signContext(method, ctx) + err := invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method + return err + })) + + grpcConnection, err := grpc.NewClient(grpcServer, grpcOptions...) + if err != nil { + return nil, err + } + defer grpcConnection.Close() + userClient := pbPublicUser.NewPubUserClient(grpcConnection) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + stateString := uuid.New().String() + // queryValues.Add("callback", oauthToken.Callback) + oauthToken, err := userClient.CreateAuthToken(ctx, &pbPublicUser.LoginRequest{ + ReturnType: 2, + State: stateString, + ReturnUrl: "", + }) + if err != nil { + return nil, err + } + if len(oauthToken.State) < 1 { + oauthToken.State = stateString + } + + if oauthToken.Url != "" { + + return nil, fmt.Errorf(`need verify: Click Here`, oauthToken.Url) + } + + return aService, err2 + +} + +func (d *HalalCloud) NewAuthService(refreshToken string, options ...HalalOption) (*AuthService, error) { + svc := d.HalalCommon.AuthService + + if len(refreshToken) < 1 { + refreshToken = d.Addition.RefreshToken + } + + if len(d.tr.AccessToken) > 0 { + accessTokenExpiredAt := d.tr.AccessTokenExpiredAt + current := time.Now().UnixMilli() + if accessTokenExpiredAt < current { + // access token expired + d.tr.AccessToken = "" + d.tr.AccessTokenExpiredAt = 0 + } else { + svc.tr.AccessTokenExpiredAt = accessTokenExpiredAt + svc.tr.AccessToken = d.tr.AccessToken + } + } + + for _, opt := range options { + opt.apply(&svc.dopts) + } + + grpcOptions := svc.dopts.grpcOptions + grpcOptions = append(grpcOptions, grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(10*1024*1024), grpc.MaxCallRecvMsgSize(10*1024*1024)), grpc.WithAuthority(grpcServerAuth), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})), grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctxx := svc.signContext(method, ctx) + err := invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method + if err != nil { + grpcStatus, ok := status.FromError(err) + + if ok && grpcStatus.Code() == codes.Unauthenticated && strings.Contains(grpcStatus.Err().Error(), "invalid accesstoken") && len(refreshToken) > 0 { + // refresh token + refreshResponse, err := pbPublicUser.NewPubUserClient(cc).Refresh(ctx, &pbPublicUser.Token{ + RefreshToken: refreshToken, + }) + if err != nil { + return err + } + if len(refreshResponse.AccessToken) > 0 { + svc.tr.AccessToken = refreshResponse.AccessToken + svc.tr.AccessTokenExpiredAt = refreshResponse.AccessTokenExpireTs + svc.OnAccessTokenRefreshed(refreshResponse.AccessToken, refreshResponse.AccessTokenExpireTs, refreshResponse.RefreshToken, refreshResponse.RefreshTokenExpireTs) + } + // retry + ctxx := svc.signContext(method, ctx) + err = invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method + if err != nil { + return err + } else { + return nil + } + } + } + return err + })) + grpcConnection, err := grpc.NewClient(grpcServer, grpcOptions...) + + if err != nil { + return nil, err + } + + svc.grpcConnection = grpcConnection + return svc, err +} + +func (s *AuthService) OnAccessTokenRefreshed(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64) { + s.tr.AccessToken = accessToken + s.tr.AccessTokenExpiredAt = accessTokenExpiredAt + s.tr.RefreshToken = refreshToken + s.tr.RefreshTokenExpiredAt = refreshTokenExpiredAt + + if s.dopts.onTokenRefreshed != nil { + s.dopts.onTokenRefreshed(accessToken, accessTokenExpiredAt, refreshToken, refreshTokenExpiredAt) + } + +} + +func (s *AuthService) GetGrpcConnection() *grpc.ClientConn { + return s.grpcConnection +} + +func (s *AuthService) Close() { + _ = s.grpcConnection.Close() +} + +func (s *AuthService) signContext(method string, ctx context.Context) context.Context { + var kvString []string + currentTimeStamp := strconv.FormatInt(time.Now().UnixMilli(), 10) + bufferedString := bytes.NewBufferString(method) + kvString = append(kvString, "timestamp", currentTimeStamp) + bufferedString.WriteString(currentTimeStamp) + kvString = append(kvString, "appid", AppID) + bufferedString.WriteString(AppID) + kvString = append(kvString, "appversion", AppVersion) + bufferedString.WriteString(AppVersion) + if s.tr != nil && len(s.tr.AccessToken) > 0 { + authorization := "Bearer " + s.tr.AccessToken + kvString = append(kvString, "authorization", authorization) + bufferedString.WriteString(authorization) + } + bufferedString.WriteString(AppSecret) + sign := GetMD5Hash(bufferedString.String()) + kvString = append(kvString, "sign", sign) + return metadata.AppendToOutgoingContext(ctx, kvString...) +} + +func (d *HalalCloud) GetCurrentOpDir(dir model.Obj, args []string, index int) string { + currentDir := dir.GetPath() + if len(currentDir) == 0 { + currentDir = "/" + } + opPath := currentDir + "/" + args[index] + if strings.HasPrefix(args[index], "/") { + opPath = args[index] + } + return opPath +} + +func (d *HalalCloud) GetCurrentDir(dir model.Obj) string { + currentDir := dir.GetPath() + if len(currentDir) == 0 { + currentDir = "/" + } + return currentDir +} + +type Common struct { +} + +func getRawFiles(addr *pubUserFile.SliceDownloadInfo) ([]byte, error) { + + if addr == nil { + return nil, errors.New("addr is nil") + } + + client := http.Client{ + Timeout: time.Duration(60 * time.Second), // Set timeout to 5 seconds + } + resp, err := client.Get(addr.DownloadAddress) + if err != nil { + + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad status: %s, body: %s", resp.Status, body) + } + + if addr.Encrypt > 0 { + cd := uint8(addr.Encrypt) + for idx := 0; idx < len(body); idx++ { + body[idx] = body[idx] ^ cd + } + } + + if addr.StoreType != 10 { + + sourceCid, err := cid.Decode(addr.Identity) + if err != nil { + return nil, err + } + checkCid, err := sourceCid.Prefix().Sum(body) + if err != nil { + return nil, err + } + if !checkCid.Equals(sourceCid) { + return nil, fmt.Errorf("bad cid: %s, body: %s", checkCid.String(), body) + } + } + + return body, nil + +} + +type openObject struct { + ctx context.Context + mu sync.Mutex + d []*pubUserFile.SliceDownloadInfo + id int + skip int64 + chunk *[]byte + chunks *[]chunkSize + closed bool + sha string + shaTemp hash.Hash +} + +// get the next chunk +func (oo *openObject) getChunk(ctx context.Context) (err error) { + if oo.id >= len(*oo.chunks) { + return io.EOF + } + var chunk []byte + err = utils.Retry(3, time.Second, func() (err error) { + chunk, err = getRawFiles(oo.d[oo.id]) + return err + }) + if err != nil { + return err + } + oo.id++ + oo.chunk = &chunk + return nil +} + +// Read reads up to len(p) bytes into p. +func (oo *openObject) Read(p []byte) (n int, err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return 0, fmt.Errorf("read on closed file") + } + // Skip data at the start if requested + for oo.skip > 0 { + //size := 1024 * 1024 + _, size, err := oo.ChunkLocation(oo.id) + if err != nil { + return 0, err + } + if oo.skip < int64(size) { + break + } + oo.id++ + oo.skip -= int64(size) + } + if len(*oo.chunk) == 0 { + err = oo.getChunk(oo.ctx) + if err != nil { + return 0, err + } + if oo.skip > 0 { + *oo.chunk = (*oo.chunk)[oo.skip:] + oo.skip = 0 + } + } + n = copy(p, *oo.chunk) + *oo.chunk = (*oo.chunk)[n:] + + oo.shaTemp.Write(*oo.chunk) + + return n, nil +} + +// Close closed the file - MAC errors are reported here +func (oo *openObject) Close() (err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return nil + } + // 校验Sha1 + if string(oo.shaTemp.Sum(nil)) != oo.sha { + return fmt.Errorf("failed to finish download: %w", err) + } + + oo.closed = true + return nil +} + +func GetMD5Hash(text string) string { + tHash := md5.Sum([]byte(text)) + return hex.EncodeToString(tHash[:]) +} + +// chunkSize describes a size and position of chunk +type chunkSize struct { + position int64 + size int +} + +func getChunkSizes(sliceSize []*pubUserFile.SliceSize) (chunks []chunkSize) { + chunks = make([]chunkSize, 0) + for _, s := range sliceSize { + // 对最后一个做特殊处理 + if s.EndIndex == 0 { + s.EndIndex = s.StartIndex + } + for j := s.StartIndex; j <= s.EndIndex; j++ { + chunks = append(chunks, chunkSize{position: j, size: int(s.Size)}) + } + } + return chunks +} + +func (oo *openObject) ChunkLocation(id int) (position int64, size int, err error) { + if id < 0 || id >= len(*oo.chunks) { + return 0, 0, errors.New("invalid arguments") + } + + return (*oo.chunks)[id].position, (*oo.chunks)[id].size, nil +} diff --git a/go.mod b/go.mod index b18ff9373fb..a28a424fdc4 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/alist-org/alist/v3 -go 1.21 +go 1.22.4 require ( github.com/SheltonZhu/115driver v1.0.22 @@ -11,11 +11,13 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.50.24 + github.com/baidubce/bce-sdk-go v0.9.180 github.com/blevesearch/bleve/v2 v2.3.10 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.17.1 github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/lipgloss v0.9.1 + github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0 github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.6.0 github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 @@ -31,11 +33,12 @@ require ( github.com/go-resty/resty/v2 v2.11.0 github.com/go-webauthn/webauthn v0.10.0 github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/google/uuid v1.5.0 + github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.1 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/boxo v0.12.0 github.com/ipfs/go-ipfs-api v0.7.0 + github.com/jinzhu/copier v0.4.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 github.com/larksuite/oapi-sdk-go/v3 v3.2.5 @@ -57,11 +60,12 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 - golang.org/x/crypto v0.19.0 + github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 + golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/image v0.15.0 - golang.org/x/net v0.21.0 - golang.org/x/oauth2 v0.16.0 + golang.org/x/net v0.26.0 + golang.org/x/oauth2 v0.18.0 golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 @@ -71,9 +75,9 @@ require ( gorm.io/gorm v1.25.10 ) +require github.com/BurntSushi/toml v0.3.1 // indirect + require ( - cloud.google.com/go/compute v1.23.0 // indirect - github.com/BurntSushi/toml v0.3.1 // indirect github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect github.com/RoaringBitmap/roaring v1.2.3 // indirect github.com/abbot/go-http-auth v0.4.0 // indirect @@ -110,7 +114,7 @@ require ( github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect github.com/gabriel-vasile/mimetype v1.4.2 // indirect @@ -126,14 +130,14 @@ require ( github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-tpm v0.9.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-cid v0.4.1 github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgx/v5 v5.3.0 // indirect @@ -154,7 +158,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect @@ -180,7 +184,7 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect github.com/prometheus/client_golang v1.16.0 // indirect @@ -208,15 +212,15 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.5.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.17.0 // indirect - golang.org/x/term v0.17.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.18.0 // indirect - google.golang.org/api v0.134.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/term v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + google.golang.org/api v0.169.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/grpc v1.64.0 + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index de6005d153e..adfba017a68 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ -cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= +cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -40,6 +39,8 @@ github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+I github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/baidubce/bce-sdk-go v0.9.180 h1:7IRGR/YFTdZiMUs0cGowyGcl5N7exzDofR7vdt+syNE= +github.com/baidubce/bce-sdk-go v0.9.180/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -106,6 +107,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0 h1:FJFYghXksa6qVfMR1pcUIShMMhyu0ikoKYVgy70/Ze8= +github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0/go.mod h1:O1dpz9RYVOC21UgZW4vLkBsUzwyM27mEpyEdd83Ia2o= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= @@ -117,8 +120,9 @@ github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuv github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= @@ -135,6 +139,8 @@ github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= @@ -158,6 +164,10 @@ github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SU github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -198,26 +208,27 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4er github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= +github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= @@ -248,6 +259,8 @@ github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHo github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaevor/go-nanoid v1.3.0 h1:nD+iepesZS6pr3uOVf20vR9GdGgJW1HPaR46gtrxzkg= github.com/jaevor/go-nanoid v1.3.0/go.mod h1:SI+jFaPuddYkqkVQoNGHs81navCtH388TcrH0RqFKgY= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -312,8 +325,8 @@ github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxec github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -388,8 +401,9 @@ github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= @@ -497,10 +511,20 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 h1:X+lHsNTlbatQ1cErXIbtyrh+3MTWxqQFS+sBP/wpFXo= +github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22/go.mod h1:1zGRDJd8zlG6P8azG96+uywfh6udYWwhOmUivw+xsuM= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= @@ -519,8 +543,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -544,18 +568,18 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= -golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -583,16 +607,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= +golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -603,8 +627,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -618,25 +642,25 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= -golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= -google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= +google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= +google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/utils/io.go b/pkg/utils/io.go index 7be989c3fd7..e06fb235b8b 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -138,7 +138,7 @@ func (mr *MultiReadable) Close() error { func Retry(attempts int, sleep time.Duration, f func() error) (err error) { for i := 0; i < attempts; i++ { - fmt.Println("This is attempt number", i) + //fmt.Println("This is attempt number", i) if i > 0 { log.Println("retrying after error:", err) time.Sleep(sleep) From 3a3d0adfa099cd3fe8e3a4cd89970e460ef668c3 Mon Sep 17 00:00:00 2001 From: Muione <75424880+Muione@users.noreply.github.com> Date: Sun, 7 Jul 2024 16:50:05 +0800 Subject: [PATCH 206/659] feat: add pikpak offline download function (#6648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add pikpak offline download function * 完善PikPak离线下载功能 * 删除多余的代码 * add task cache to avoid too many requests about API * 优化Status函数 * 完善所有功能,目前测试无BUG * 减少缓存时间，优化添加离线任务的参数 --- drivers/pikpak/driver.go | 90 ++++++++++++++++ drivers/pikpak/types.go | 69 ++++++++++++ internal/offline_download/all.go | 1 + internal/offline_download/pikpak/pikpak.go | 120 +++++++++++++++++++++ internal/offline_download/pikpak/util.go | 43 ++++++++ internal/offline_download/tool/add.go | 11 +- internal/offline_download/tool/download.go | 8 +- 7 files changed, 339 insertions(+), 3 deletions(-) create mode 100644 internal/offline_download/pikpak/pikpak.go create mode 100644 internal/offline_download/pikpak/util.go diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 2dab2a9b066..3ecc31d6bff 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -2,8 +2,10 @@ package pikpak import ( "context" + "encoding/json" "fmt" "net/http" + "strconv" "strings" "github.com/alist-org/alist/v3/drivers/base" @@ -207,4 +209,92 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } +// 离线下载文件 +func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + requestBody := base.Json{ + "kind": "drive#file", + "name": fileName, + "upload_type": "UPLOAD_TYPE_URL", + "url": base.Json{ + "url": fileUrl, + }, + "parent_id": parentDir.GetID(), + "folder_type": "", + } + + var resp OfflineDownloadResp + _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(requestBody) + }, &resp) + + if err != nil { + return nil, err + } + + return &resp.Task, err +} + +/* +获取离线下载任务列表 +phase 可能的取值： +PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING +*/ +func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase []string) ([]OfflineTask, error) { + res := make([]OfflineTask, 0) + url := "https://api-drive.mypikpak.com/drive/v1/tasks" + + if len(phase) == 0 { + phase = []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_COMPLETE", "PHASE_TYPE_PENDING"} + } + params := map[string]string{ + "type": "offline", + "thumbnail_size": "SIZE_SMALL", + "limit": "10000", + "page_token": nextPageToken, + "with": "reference_resource", + } + + // 处理 phase 参数 + if len(phase) > 0 { + filters := base.Json{ + "phase": map[string]string{ + "in": strings.Join(phase, ","), + }, + } + filtersJSON, err := json.Marshal(filters) + if err != nil { + return nil, fmt.Errorf("failed to marshal filters: %w", err) + } + params["filters"] = string(filtersJSON) + } + + var resp OfflineListResp + _, err := d.request(url, http.MethodGet, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(params) + }, &resp) + + if err != nil { + return nil, fmt.Errorf("failed to get offline list: %w", err) + } + res = append(res, resp.Tasks...) + return res, nil +} + +func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + url := "https://api-drive.mypikpak.com/drive/v1/tasks" + params := map[string]string{ + "task_ids": strings.Join(taskIDs, ","), + "delete_files": strconv.FormatBool(deleteFiles), + } + _, err := d.request(url, http.MethodDelete, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(params) + }, nil) + if err != nil { + return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) + } + return nil +} + var _ driver.Driver = (*PikPak)(nil) diff --git a/drivers/pikpak/types.go b/drivers/pikpak/types.go index 489a1efe713..a9928d00ec2 100644 --- a/drivers/pikpak/types.go +++ b/drivers/pikpak/types.go @@ -99,3 +99,72 @@ type UploadTaskData struct { File File `json:"file"` } + +// 添加离线下载响应 +type OfflineDownloadResp struct { + File *string `json:"file"` + Task OfflineTask `json:"task"` + UploadType string `json:"upload_type"` + URL struct { + Kind string `json:"kind"` + } `json:"url"` +} + +// 离线下载列表 +type OfflineListResp struct { + ExpiresIn int64 `json:"expires_in"` + NextPageToken string `json:"next_page_token"` + Tasks []OfflineTask `json:"tasks"` +} + +// offlineTask +type OfflineTask struct { + Callback string `json:"callback"` + CreatedTime string `json:"created_time"` + FileID string `json:"file_id"` + FileName string `json:"file_name"` + FileSize string `json:"file_size"` + IconLink string `json:"icon_link"` + ID string `json:"id"` + Kind string `json:"kind"` + Message string `json:"message"` + Name string `json:"name"` + Params Params `json:"params"` + Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING + Progress int64 `json:"progress"` + ReferenceResource ReferenceResource `json:"reference_resource"` + Space string `json:"space"` + StatusSize int64 `json:"status_size"` + Statuses []string `json:"statuses"` + ThirdTaskID string `json:"third_task_id"` + Type string `json:"type"` + UpdatedTime string `json:"updated_time"` + UserID string `json:"user_id"` +} + +type Params struct { + Age string `json:"age"` + MIMEType *string `json:"mime_type,omitempty"` + PredictType string `json:"predict_type"` + URL string `json:"url"` +} + +type ReferenceResource struct { + Type string `json:"@type"` + Audit interface{} `json:"audit"` + Hash string `json:"hash"` + IconLink string `json:"icon_link"` + ID string `json:"id"` + Kind string `json:"kind"` + Medias []Media `json:"medias"` + MIMEType string `json:"mime_type"` + Name string `json:"name"` + Params map[string]interface{} `json:"params"` + ParentID string `json:"parent_id"` + Phase string `json:"phase"` + Size string `json:"size"` + Space string `json:"space"` + Starred bool `json:"starred"` + Tags []string `json:"tags"` + ThumbnailLink string `json:"thumbnail_link"` +} diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 2229a855468..67869dee8d2 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -3,5 +3,6 @@ package offline_download import ( _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" _ "github.com/alist-org/alist/v3/internal/offline_download/http" + _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" ) diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go new file mode 100644 index 00000000000..618b1442b8a --- /dev/null +++ b/internal/offline_download/pikpak/pikpak.go @@ -0,0 +1,120 @@ +package pikpak + +import ( + "context" + "fmt" + + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" +) + +type PikPak struct { + refreshTaskCache bool +} + +func (p *PikPak) Name() string { + return "pikpak" +} + +func (p *PikPak) Items() []model.SettingItem { + return nil +} + +func (p *PikPak) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (p *PikPak) Init() (string, error) { + p.refreshTaskCache = false + return "ok", nil +} + +func (p *PikPak) IsReady() bool { + return true +} + +func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { + // 添加新任务刷新缓存 + p.refreshTaskCache = true + // args.TempDir 已经被修改为了 DstDirPath + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + pikpakDriver, ok := storage.(*pikpak.PikPak) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") + } + + ctx := context.Background() + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + + t, err := pikpakDriver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + + return t.ID, nil +} + +func (p *PikPak) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + if err != nil { + return err + } + pikpakDriver, ok := storage.(*pikpak.PikPak) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") + } + ctx := context.Background() + err = pikpakDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false) + if err != nil { + return err + } + return nil +} + +func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + if err != nil { + return nil, err + } + pikpakDriver, ok := storage.(*pikpak.PikPak) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") + } + tasks, err := p.GetTasks(pikpakDriver) + if err != nil { + return nil, err + } + s := &tool.Status{ + Progress: 0, + NewGID: "", + Completed: false, + Status: "the task has been deleted", + Err: nil, + } + for _, t := range tasks { + if t.ID == task.GID { + s.Progress = float64(t.Progress) + s.Status = t.Message + s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE") + if t.Phase == "PHASE_TYPE_ERROR" { + s.Err = fmt.Errorf(t.Message) + } + return s, nil + } + } + s.Err = fmt.Errorf("the task has been deleted") + return s, nil +} + +func init() { + tool.Tools.Add(&PikPak{}) +} diff --git a/internal/offline_download/pikpak/util.go b/internal/offline_download/pikpak/util.go new file mode 100644 index 00000000000..f7bf9282621 --- /dev/null +++ b/internal/offline_download/pikpak/util.go @@ -0,0 +1,43 @@ +package pikpak + +import ( + "context" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]pikpak.OfflineTask](16)) +var taskG singleflight.Group[[]pikpak.OfflineTask] + +func (p *PikPak) GetTasks(pikpakDriver *pikpak.PikPak) ([]pikpak.OfflineTask, error) { + key := op.Key(pikpakDriver, "/drive/v1/task") + if !p.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + p.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]pikpak.OfflineTask, error) { + ctx := context.Background() + phase := []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_PENDING", "PHASE_TYPE_COMPLETE"} + tasks, err := pikpakDriver.OfflineList(ctx, "", phase) + if err != nil { + return nil, err + } + // 添加缓存 10s + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]pikpak.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 3da05c8df68..e9bcdc50e52 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,13 +2,14 @@ package tool import ( "context" + "path/filepath" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/op" "github.com/google/uuid" "github.com/pkg/errors" "github.com/xhofe/tache" - "path/filepath" ) type DeletePolicy string @@ -64,11 +65,17 @@ func AddURL(ctx context.Context, args *AddURLArgs) (tache.TaskWithInfo, error) { uid := uuid.NewString() tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) + deletePolicy := args.DeletePolicy + if args.Tool == "pikpak" { + tempDir = args.DstDirPath + // 防止将下载好的文件删除 + deletePolicy = DeleteNever + } t := &DownloadTask{ Url: args.URL, DstDirPath: args.DstDirPath, TempDir: tempDir, - DeletePolicy: args.DeletePolicy, + DeletePolicy: deletePolicy, tool: tool, } DownloadTaskManager.Add(t) diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index f0a5d5d4376..79a29ef0c9a 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -71,6 +71,9 @@ outer: if err != nil { return err } + if t.tool.Name() == "pikpak" { + return nil + } t.Status = "offline download completed, maybe transferring" // hack for qBittorrent if t.tool.Name() == "qBittorrent" { @@ -123,6 +126,9 @@ func (t *DownloadTask) Complete() error { files []File err error ) + if t.tool.Name() == "pikpak" { + return nil + } if getFileser, ok := t.tool.(GetFileser); ok { files = getFileser.GetFiles(t) } else { @@ -132,7 +138,7 @@ func (t *DownloadTask) Complete() error { } } // upload files - for i, _ := range files { + for i := range files { file := files[i] TransferTaskManager.Add(&TransferTask{ file: file, From ca0d66bd01efad671f365b2b160e5b1008a0bfcf Mon Sep 17 00:00:00 2001 From: Hao Jiakang <946392690@qq.com> Date: Sun, 7 Jul 2024 16:50:40 +0800 Subject: [PATCH 207/659] fix: S3 Implementation bug & Support AWS Signature V2 (#6683) * Fix: when S3 PutObject with objectName contains /, aliyundriveopen failed due to KeyNotFound, make dir to fix this. (cherry picked from commit eb24f45771d29a3659e75813734b290d6306cfcf) * Upgrade gofakes3 to v0.0.5, support AWS Signature V2 (cherry picked from commit 3218d7cf2c4e1a8c51fd2414595547fd109a89ac) --------- Co-authored-by: David Hao --- go.mod | 2 +- go.sum | 4 ++-- server/s3/backend.go | 16 ++++++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index a28a424fdc4..ea15c8c021e 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/Xhofe/wopan-sdk-go v0.1.2 - github.com/alist-org/gofakes3 v0.0.4 + github.com/alist-org/gofakes3 v0.0.5 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.50.24 diff --git a/go.sum b/go.sum index adfba017a68..8087ad24dec 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0E github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= -github.com/alist-org/gofakes3 v0.0.4 h1:/ID4+1llsiB8EweLcC65rVmgBZKL95e3P7Wa+aJGUiE= -github.com/alist-org/gofakes3 v0.0.4/go.mod h1:bLPZXt45XYMgaoGGLe5t0d1p13oZTQTptTEDLrku070= +github.com/alist-org/gofakes3 v0.0.5 h1:bLHhLTNg3kIRdx7gsgi9Zg/EW9s3IHwJVRwzUCPR8V0= +github.com/alist-org/gofakes3 v0.0.5/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= diff --git a/server/s3/backend.go b/server/s3/backend.go index c4c6c5a66c7..2987d81e839 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -6,6 +6,7 @@ import ( "context" "encoding/hex" "fmt" + "github.com/pkg/errors" "io" "path" "strings" @@ -21,6 +22,7 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/gofakes3" "github.com/ncw/swift/v2" + log "github.com/sirupsen/logrus" ) var ( @@ -268,9 +270,19 @@ func (b *s3Backend) PutObject( fp := path.Join(bucketPath, objectName) reqPath := path.Dir(fp) fmeta, _ := op.GetNearestMeta(fp) - _, err = fs.Get(context.WithValue(ctx, "meta", fmeta), reqPath, &fs.GetArgs{}) + ctx = context.WithValue(ctx, "meta", fmeta) + + _, err = fs.Get(ctx, reqPath, &fs.GetArgs{}) if err != nil { - return result, gofakes3.KeyNotFound(objectName) + if errs.IsObjectNotFound(err) && strings.Contains(objectName, "/") { + log.Debugf("reqPath: %s not found and objectName contains /, need to makeDir", reqPath) + err = fs.MakeDir(ctx, reqPath, true) + if err != nil { + return result, errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) + } + } else { + return result, gofakes3.KeyNotFound(objectName) + } } var ti time.Time From 33be44adad6a833cdf5df749d21da303e48585a7 Mon Sep 17 00:00:00 2001 From: Mmx Date: Thu, 11 Jul 2024 18:13:22 +0800 Subject: [PATCH 208/659] chore: update polyfill URL due to service unavailability and supply chain attack risk (#6740) --- internal/bootstrap/data/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 93a588e7622..75244d84027 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -131,7 +131,7 @@ func InitialSettings() []model.SettingItem { // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, - {Key: conf.CustomizeHead, PreDefault: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.CustomizeHead, PreDefault: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.CustomizeBody, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.LinkExpiration, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.SignAll, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, From 213fc0232eaac10175b12dbf38fb53e5ac77b055 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:02:31 +0800 Subject: [PATCH 209/659] fix(deps): update module github.com/sheltonzhu/115driver to v1.0.25 (#6447) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ea15c8c021e..bbeee1c94f4 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/alist-org/alist/v3 go 1.22.4 require ( - github.com/SheltonZhu/115driver v1.0.22 + github.com/SheltonZhu/115driver v1.0.25 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/Xhofe/wopan-sdk-go v0.1.2 diff --git a/go.sum b/go.sum index 8087ad24dec..9e37b889726 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVO github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U= github.com/SheltonZhu/115driver v1.0.22/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= +github.com/SheltonZhu/115driver v1.0.25 h1:i101yanLKUwV1Pi7x+vgNOwgz7Hp9JbNmo6BCZ9/4wo= +github.com/SheltonZhu/115driver v1.0.25/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= From 92c65b450e41ed72758bdb060ea952b61712faad Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:02:52 +0800 Subject: [PATCH 210/659] fix(deps): update module golang.org/x/image to v0.18.0 [security] (#6658) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bbeee1c94f4..7311a7466ab 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.24.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 - golang.org/x/image v0.15.0 + golang.org/x/image v0.18.0 golang.org/x/net v0.26.0 golang.org/x/oauth2 v0.18.0 golang.org/x/time v0.5.0 diff --git a/go.sum b/go.sum index 9e37b889726..9fab64a7ba6 100644 --- a/go.sum +++ b/go.sum @@ -552,6 +552,8 @@ golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= +golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= From 80d4fbb870829c886e65a1cdc8a54ea21f654b74 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:03:20 +0800 Subject: [PATCH 211/659] fix(deps): update module github.com/gorilla/websocket to v1.5.3 (#6653) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7311a7466ab..fd9801bf8d4 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/go-webauthn/webauthn v0.10.0 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 - github.com/gorilla/websocket v1.5.1 + github.com/gorilla/websocket v1.5.3 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/boxo v0.12.0 github.com/ipfs/go-ipfs-api v0.7.0 diff --git a/go.sum b/go.sum index 9fab64a7ba6..5419bdfa118 100644 --- a/go.sum +++ b/go.sum @@ -234,6 +234,8 @@ github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= From 37468313844b62f13713344a191f7073ae859a85 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:03:56 +0800 Subject: [PATCH 212/659] chore(deps): update actions-cool/issues-helper action to v3.6.0 (#6513) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/issue_question.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/issue_question.yml b/.github/workflows/issue_question.yml index 1cc9839a68b..3b285ac04ac 100644 --- a/.github/workflows/issue_question.yml +++ b/.github/workflows/issue_question.yml @@ -10,7 +10,7 @@ jobs: if: github.event.label.name == 'question' steps: - name: Create comment - uses: actions-cool/issues-helper@v3.5.2 + uses: actions-cool/issues-helper@v3.6.0 with: actions: 'create-comment' token: ${{ secrets.GITHUB_TOKEN }} From 87192ad07d24e4674820773a781ca5844e46db91 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:04:18 +0800 Subject: [PATCH 213/659] fix(deps): update module github.com/blevesearch/bleve/v2 to v2.4.1 (#6542) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++++------- go.sum | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index fd9801bf8d4..2dced1fe2b8 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.50.24 github.com/baidubce/bce-sdk-go v0.9.180 - github.com/blevesearch/bleve/v2 v2.3.10 + github.com/blevesearch/bleve/v2 v2.4.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.17.1 github.com/charmbracelet/bubbletea v0.25.0 @@ -75,11 +75,15 @@ require ( gorm.io/gorm v1.25.10 ) -require github.com/BurntSushi/toml v0.3.1 // indirect +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + github.com/blevesearch/go-faiss v1.0.19 // indirect + github.com/blevesearch/zapx/v16 v16.1.4 // indirect +) require ( github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect - github.com/RoaringBitmap/roaring v1.2.3 // indirect + github.com/RoaringBitmap/roaring v1.9.3 // indirect github.com/abbot/go-http-auth v0.4.0 // indirect github.com/aead/ecdh v0.2.0 // indirect github.com/andreburgaud/crypt2go v1.2.0 // indirect @@ -88,14 +92,14 @@ require ( github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/blevesearch/bleve_index_api v1.0.6 // indirect - github.com/blevesearch/geo v0.1.18 // indirect + github.com/blevesearch/bleve_index_api v1.1.9 // indirect + github.com/blevesearch/geo v0.1.20 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.2.14 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect diff --git a/go.sum b/go.sum index 5419bdfa118..c7565dabcbf 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= +github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= +github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U= github.com/SheltonZhu/115driver v1.0.22/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/SheltonZhu/115driver v1.0.25 h1:i101yanLKUwV1Pi7x+vgNOwgz7Hp9JbNmo6BCZ9/4wo= @@ -49,14 +51,24 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= +github.com/blevesearch/bleve/v2 v2.4.1 h1:8QWqsifq693mN3h6cSigKqkKUsUfv5hu0FDgz/4bFuA= +github.com/blevesearch/bleve/v2 v2.4.1/go.mod h1:Ezmvsouspi+uVwnDzjIsCeUIT0WuBKlicP5JZnExWzo= github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= +github.com/blevesearch/bleve_index_api v1.1.9 h1:Cpq0Lp3As0Gfk3+PmcoNDRKeI50C5yuFNpj0YlN/bOE= +github.com/blevesearch/bleve_index_api v1.1.9/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= +github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= +github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= +github.com/blevesearch/go-faiss v1.0.19 h1:UKoP8hS7DVsVSRRloNJb4qPfe2UQ99pP4D3oXd23g2A= +github.com/blevesearch/go-faiss v1.0.19/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= @@ -65,6 +77,8 @@ github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCD github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= +github.com/blevesearch/scorch_segment_api/v2 v2.2.14 h1:fgMLMpGWR7u2TdRm7XSZVWhPvMAcdYHh25Lq1fQ6Fjo= +github.com/blevesearch/scorch_segment_api/v2 v2.2.14/go.mod h1:B7+a7vfpY4NsjuTkpv/eY7RZ91Xr90VaJzT2t7upZN8= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= @@ -83,6 +97,8 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7 github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= +github.com/blevesearch/zapx/v16 v16.1.4 h1:TBQfG77g2UUXwfjOVcEtB9pXkg6JBmGXkeZKI67+TiA= +github.com/blevesearch/zapx/v16 v16.1.4/go.mod h1:+Q+Z89Iv7ewhdX2jyE6Qs/RUnN4tZuokaQ0xvTaFmx8= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= From 2fb772c8883dd156c5b4aa2f402b79c59b06125d Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:04:42 +0800 Subject: [PATCH 214/659] fix(deps): update module github.com/meilisearch/meilisearch-go to v0.27.0 (#6436) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2dced1fe2b8..f5ee09870eb 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/larksuite/oapi-sdk-go/v3 v3.2.5 github.com/maruel/natural v1.1.1 - github.com/meilisearch/meilisearch-go v0.26.1 + github.com/meilisearch/meilisearch-go v0.27.0 github.com/minio/sio v0.3.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.2 diff --git a/go.sum b/go.sum index c7565dabcbf..fdff9269f94 100644 --- a/go.sum +++ b/go.sum @@ -358,6 +358,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zk github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/meilisearch/meilisearch-go v0.26.1 h1:3bmo2uLijX7kvBmiZ9LupVfC95TFcRJDgrRTzbOoE4A= github.com/meilisearch/meilisearch-go v0.26.1/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= +github.com/meilisearch/meilisearch-go v0.27.0 h1:lDFq8WzbsZCtt3/byr7GFqfOygWF5iy9TtDgzJo0Ds8= +github.com/meilisearch/meilisearch-go v0.27.0/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= From 99c9632cdc80bd663e37c47631ee26a3f68bc7b4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:05:08 +0800 Subject: [PATCH 215/659] fix(deps): update module github.com/gin-contrib/cors to v1.6.0 [security] (#6708) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 20 ++++++++++---------- go.sum | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index f5ee09870eb..816c4ceeec5 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gaoyb7/115drive-webdav v0.1.8 - github.com/gin-contrib/cors v1.5.0 + github.com/gin-contrib/cors v1.6.0 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.11.0 github.com/go-webauthn/webauthn v0.10.0 @@ -111,24 +111,24 @@ require ( github.com/blevesearch/zapx/v15 v15.3.13 // indirect github.com/bluele/gcache v0.0.2 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect - github.com/bytedance/sonic v1.10.1 // indirect + github.com/bytedance/sonic v1.11.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect - github.com/chenzhuoyu/iasm v0.9.0 // indirect + github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/fxamacker/cbor/v2 v2.5.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-chi/chi/v5 v5.0.10 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.15.5 // indirect + github.com/go-playground/validator/v10 v10.19.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/go-webauthn/x v0.1.6 // indirect github.com/goccy/go-json v0.10.2 // indirect @@ -152,9 +152,9 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect github.com/klauspost/compress v1.17.4 // indirect - github.com/klauspost/cpuid/v2 v2.2.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/kr/fs v0.1.0 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p v0.27.8 // indirect @@ -186,7 +186,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect @@ -208,14 +208,14 @@ require ( github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u2takey/go-utils v0.3.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/arch v0.5.0 // indirect + golang.org/x/arch v0.7.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/term v0.21.0 // indirect diff --git a/go.sum b/go.sum index fdff9269f94..232c87646d1 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc= github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= +github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -125,6 +127,8 @@ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= +github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= +github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0 h1:FJFYghXksa6qVfMR1pcUIShMMhyu0ikoKYVgy70/Ze8= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0/go.mod h1:O1dpz9RYVOC21UgZW4vLkBsUzwyM27mEpyEdd83Ia2o= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= @@ -168,12 +172,16 @@ github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADi github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw= github.com/gaoyb7/115drive-webdav v0.1.8/go.mod h1:BKbeY6j8SKs3+rzBFFALznGxbPmefEm3vA+dGhqgOGU= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/gin-contrib/cors v1.5.0 h1:DgGKV7DDoOn36DFkNtbHrjoRiT5ExCe+PC9/xp7aKvk= github.com/gin-contrib/cors v1.5.0/go.mod h1:TvU7MAZ3EwrPLI2ztzTt3tqgvBCq+wn8WpZmfADjupI= +github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg= +github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= @@ -201,6 +209,8 @@ github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXS github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24= github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= +github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= @@ -309,6 +319,8 @@ github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6K github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -326,6 +338,8 @@ github.com/larksuite/oapi-sdk-go/v3 v3.2.5/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= @@ -413,6 +427,8 @@ github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OI github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= +github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -511,6 +527,8 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -551,6 +569,8 @@ gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= +golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= From 60fc416d8f33df622fb0ca90e1dde6dd85324bf1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:05:55 +0800 Subject: [PATCH 216/659] fix(deps): update module google.golang.org/grpc to v1.64.1 [security] (#6728) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 816c4ceeec5..78c7d881d8f 100644 --- a/go.mod +++ b/go.mod @@ -223,7 +223,7 @@ require ( golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/grpc v1.64.0 + google.golang.org/grpc v1.64.1 google.golang.org/protobuf v1.34.2 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 232c87646d1..26bf657ef03 100644 --- a/go.sum +++ b/go.sum @@ -700,6 +700,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= +google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= +google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= From 9bc2d340a21355d825cc022844b9cdf9e7838bcd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:27:49 +0800 Subject: [PATCH 217/659] fix(deps): update golang.org/x/exp digest to 46b0784 (#6486) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 12 ++++++------ go.sum | 12 ++++++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 78c7d881d8f..0d7a0778d29 100644 --- a/go.mod +++ b/go.mod @@ -61,10 +61,10 @@ require ( github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.24.0 - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/crypto v0.25.0 + golang.org/x/exp v0.0.0-20240707233637-46b078467d37 golang.org/x/image v0.18.0 - golang.org/x/net v0.26.0 + golang.org/x/net v0.27.0 golang.org/x/oauth2 v0.18.0 golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 @@ -217,10 +217,10 @@ require ( go.etcd.io/bbolt v1.3.7 // indirect golang.org/x/arch v0.7.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.23.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/grpc v1.64.1 diff --git a/go.sum b/go.sum index 26bf657ef03..bd7157a8e08 100644 --- a/go.sum +++ b/go.sum @@ -587,8 +587,12 @@ golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58 golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= +golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= @@ -614,6 +618,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -653,6 +659,8 @@ golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -661,6 +669,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -688,6 +698,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= +golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 4e1c67617f7c04f5fb9f53219f188df535938a04 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:28:13 +0800 Subject: [PATCH 218/659] fix(deps): update module github.com/go-webauthn/webauthn to v0.10.2 (#6310) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 0d7a0778d29..3888d328306 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/gin-contrib/cors v1.6.0 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.11.0 - github.com/go-webauthn/webauthn v0.10.0 + github.com/go-webauthn/webauthn v0.10.2 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 @@ -54,7 +54,7 @@ require ( github.com/rclone/rclone v1.63.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 @@ -120,7 +120,7 @@ require ( github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/fxamacker/cbor/v2 v2.5.0 // indirect + github.com/fxamacker/cbor/v2 v2.6.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -130,9 +130,9 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.19.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/go-webauthn/x v0.1.6 // indirect + github.com/go-webauthn/x v0.1.9 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index bd7157a8e08..ebd632afb97 100644 --- a/go.sum +++ b/go.sum @@ -170,6 +170,8 @@ github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rE github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE= github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= +github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= @@ -218,8 +220,12 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.10.0 h1:yuW2e1tXnRAwAvKrR4q4LQmc6XtCMH639/ypZGhZCwk= github.com/go-webauthn/webauthn v0.10.0/go.mod h1:l0NiauXhL6usIKqNLCUM3Qir43GK7ORg8ggold0Uv/Y= +github.com/go-webauthn/webauthn v0.10.2 h1:OG7B+DyuTytrEPFmTX503K77fqs3HDK/0Iv+z8UYbq4= +github.com/go-webauthn/webauthn v0.10.2/go.mod h1:Gd1IDsGAybuvK1NkwUTLbGmeksxuRJjVN2PE/xsPxHs= github.com/go-webauthn/x v0.1.6 h1:QNAX+AWeqRt9loE8mULeWJCqhVG5D/jvdmJ47fIWCkQ= github.com/go-webauthn/x v0.1.6/go.mod h1:W8dFVZ79o4f+nY1eOUICy/uq5dhrRl7mxQkYhXTo0FA= +github.com/go-webauthn/x v0.1.9 h1:v1oeLmoaa+gPOaZqUdDentu6Rl7HkSSsmOT6gxEQHhE= +github.com/go-webauthn/x v0.1.9/go.mod h1:pJNMlIMP1SU7cN8HNlKJpLEnFHCygLCvaLZ8a1xeoQA= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -229,6 +235,8 @@ github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOW github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -510,6 +518,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= From 37d86ff55cc0cd3a7c579932530ff794648f1c42 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:28:25 +0800 Subject: [PATCH 219/659] fix(deps): update module github.com/minio/sio to v0.4.0 (#6446) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3888d328306..ab97e4cb8dc 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/larksuite/oapi-sdk-go/v3 v3.2.5 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.0 - github.com/minio/sio v0.3.0 + github.com/minio/sio v0.4.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.2 github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 diff --git a/go.sum b/go.sum index ebd632afb97..a9e8c1f16a9 100644 --- a/go.sum +++ b/go.sum @@ -386,6 +386,8 @@ github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dz github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= +github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= +github.com/minio/sio v0.4.0/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= From ff20b5a6fbbf7d6354bdae4aca4fe198600c9eca Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:29:05 +0800 Subject: [PATCH 220/659] fix(deps): update module github.com/baidubce/bce-sdk-go to v0.9.184 (#6754) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ab97e4cb8dc..9f14ae8fd8f 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.50.24 - github.com/baidubce/bce-sdk-go v0.9.180 + github.com/baidubce/bce-sdk-go v0.9.184 github.com/blevesearch/bleve/v2 v2.4.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.17.1 diff --git a/go.sum b/go.sum index a9e8c1f16a9..fb8d1c91a68 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,7 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/baidubce/bce-sdk-go v0.9.180 h1:7IRGR/YFTdZiMUs0cGowyGcl5N7exzDofR7vdt+syNE= github.com/baidubce/bce-sdk-go v0.9.180/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= +github.com/baidubce/bce-sdk-go v0.9.184/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= From 65b423c503389caf70cc98f43bfad2d1b5cf84fe Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 17:42:25 +0800 Subject: [PATCH 221/659] fix(deps): update github.com/city404/v6-public-rpc-proto/go digest to 9a9b82a (#6753) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 9f14ae8fd8f..d1c62d3692d 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( github.com/charmbracelet/bubbles v0.17.1 github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/lipgloss v0.9.1 - github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0 + github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.6.0 github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 @@ -65,7 +65,7 @@ require ( golang.org/x/exp v0.0.0-20240707233637-46b078467d37 golang.org/x/image v0.18.0 golang.org/x/net v0.27.0 - golang.org/x/oauth2 v0.18.0 + golang.org/x/oauth2 v0.20.0 golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 @@ -112,7 +112,7 @@ require ( github.com/bluele/gcache v0.0.2 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/bytedance/sonic v1.11.2 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect @@ -222,8 +222,8 @@ require ( golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.23.0 // indirect google.golang.org/api v0.169.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect - google.golang.org/grpc v1.64.1 + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index fb8d1c91a68..5ec2f35161d 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,8 @@ github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.17.1 h1:0SIyjOnkrsfDo88YvPgAWvZMwXe26TP6drRvmkjyUu4= github.com/charmbracelet/bubbles v0.17.1/go.mod h1:9HxZWlkCqz2PRwsCbYl7a3KXvGzFaDHpYbSYMJ+nE3o= github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= @@ -132,6 +134,8 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0 h1:FJFYghXksa6qVfMR1pcUIShMMhyu0ikoKYVgy70/Ze8= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0/go.mod h1:O1dpz9RYVOC21UgZW4vLkBsUzwyM27mEpyEdd83Ia2o= +github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d h1:p5T6ZPvh7nihJfjI9M/W2cbcX7n766u/OGorLmE4xoQ= +github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d/go.mod h1:akxZg8LuwOIeCPRjcDrUS1WWcIwmLNSR2lfe4y85PH4= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= @@ -635,6 +639,8 @@ golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -723,10 +729,14 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= From de8d2d6dc0243cb0f0048b96374427e55f4a1671 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 19:40:57 +0800 Subject: [PATCH 222/659] fix(deps): update module github.com/go-resty/resty/v2 to v2.13.1 (#6759) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d1c62d3692d..9a2de2df2e5 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/gaoyb7/115drive-webdav v0.1.8 github.com/gin-contrib/cors v1.6.0 github.com/gin-gonic/gin v1.9.1 - github.com/go-resty/resty/v2 v2.11.0 + github.com/go-resty/resty/v2 v2.13.1 github.com/go-webauthn/webauthn v0.10.2 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 5ec2f35161d..53a08fd7111 100644 --- a/go.sum +++ b/go.sum @@ -221,6 +221,8 @@ github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaC github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= +github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= +github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.10.0 h1:yuW2e1tXnRAwAvKrR4q4LQmc6XtCMH639/ypZGhZCwk= @@ -602,6 +604,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= @@ -633,6 +637,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= @@ -676,6 +682,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= @@ -686,6 +694,8 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= @@ -700,6 +710,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From ba4df55d6e84c3a47069394480ad8b7cf708d71b Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 13 Jul 2024 19:49:45 +0800 Subject: [PATCH 223/659] fix(deps): upgrade wopan-sdk-go (close #6663) --- drivers/wopan/driver.go | 2 +- drivers/wopan/types.go | 2 +- drivers/wopan/util.go | 2 +- go.mod | 4 +- go.sum | 99 +++-------------------------------------- 5 files changed, 9 insertions(+), 100 deletions(-) diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index e5e26c94a08..bccce4b1c0a 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -5,12 +5,12 @@ import ( "fmt" "strconv" - "github.com/Xhofe/wopan-sdk-go" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + "github.com/xhofe/wopan-sdk-go" ) type Wopan struct { diff --git a/drivers/wopan/types.go b/drivers/wopan/types.go index 8a39c18a807..4025dbab237 100644 --- a/drivers/wopan/types.go +++ b/drivers/wopan/types.go @@ -1,8 +1,8 @@ package template import ( - "github.com/Xhofe/wopan-sdk-go" "github.com/alist-org/alist/v3/internal/model" + "github.com/xhofe/wopan-sdk-go" ) type Object struct { diff --git a/drivers/wopan/util.go b/drivers/wopan/util.go index 2ac4e61baff..b825d6ea9ac 100644 --- a/drivers/wopan/util.go +++ b/drivers/wopan/util.go @@ -3,7 +3,7 @@ package template import ( "time" - "github.com/Xhofe/wopan-sdk-go" + "github.com/xhofe/wopan-sdk-go" ) // do others that not defined in Driver interface diff --git a/go.mod b/go.mod index 9a2de2df2e5..26adf349c0c 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,10 @@ require ( github.com/SheltonZhu/115driver v1.0.25 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 - github.com/Xhofe/wopan-sdk-go v0.1.2 github.com/alist-org/gofakes3 v0.0.5 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.50.24 - github.com/baidubce/bce-sdk-go v0.9.184 github.com/blevesearch/bleve/v2 v2.4.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.17.1 @@ -38,7 +36,6 @@ require ( github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/boxo v0.12.0 github.com/ipfs/go-ipfs-api v0.7.0 - github.com/jinzhu/copier v0.4.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 github.com/larksuite/oapi-sdk-go/v3 v3.2.5 @@ -60,6 +57,7 @@ require ( github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.1 + github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.25.0 golang.org/x/exp v0.0.0-20240707233637-46b078467d37 diff --git a/go.sum b/go.sum index 53a08fd7111..9a8e57a5659 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,12 @@ -cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU= -cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= -cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= +cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= +cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= -github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/SheltonZhu/115driver v1.0.22 h1:Wp8pN7/gK3YwEO5P18ggbIOHM++lo9eP/pBhuvXfI6U= -github.com/SheltonZhu/115driver v1.0.22/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/SheltonZhu/115driver v1.0.25 h1:i101yanLKUwV1Pi7x+vgNOwgz7Hp9JbNmo6BCZ9/4wo= github.com/SheltonZhu/115driver v1.0.25/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= @@ -20,8 +15,6 @@ github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/c github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaVLk4gI/Co0paF92FmxSR6U1zY= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY= -github.com/Xhofe/wopan-sdk-go v0.1.2 h1:6Gh4YTT7b7YHN0OoJ33j7Jm9ru/ckuvcDxPnRmH07jc= -github.com/Xhofe/wopan-sdk-go v0.1.2/go.mod h1:ktLYb4t7rnPFq1AshLaPXq5kZER+DkEagT6/i/in0uo= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= @@ -43,29 +36,18 @@ github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+I github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= -github.com/baidubce/bce-sdk-go v0.9.180 h1:7IRGR/YFTdZiMUs0cGowyGcl5N7exzDofR7vdt+syNE= -github.com/baidubce/bce-sdk-go v0.9.180/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= -github.com/baidubce/bce-sdk-go v0.9.184/go.mod h1:zbYJMQwE4IZuyrJiFO8tO8NbtYiKTFTbwh4eIsqjVdg= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= -github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= github.com/blevesearch/bleve/v2 v2.4.1 h1:8QWqsifq693mN3h6cSigKqkKUsUfv5hu0FDgz/4bFuA= github.com/blevesearch/bleve/v2 v2.4.1/go.mod h1:Ezmvsouspi+uVwnDzjIsCeUIT0WuBKlicP5JZnExWzo= -github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= -github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= github.com/blevesearch/bleve_index_api v1.1.9 h1:Cpq0Lp3As0Gfk3+PmcoNDRKeI50C5yuFNpj0YlN/bOE= github.com/blevesearch/bleve_index_api v1.1.9/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= -github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= -github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= github.com/blevesearch/go-faiss v1.0.19 h1:UKoP8hS7DVsVSRRloNJb4qPfe2UQ99pP4D3oXd23g2A= @@ -76,8 +58,6 @@ github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZG github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= -github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= github.com/blevesearch/scorch_segment_api/v2 v2.2.14 h1:fgMLMpGWR7u2TdRm7XSZVWhPvMAcdYHh25Lq1fQ6Fjo= github.com/blevesearch/scorch_segment_api/v2 v2.2.14/go.mod h1:B7+a7vfpY4NsjuTkpv/eY7RZ91Xr90VaJzT2t7upZN8= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= @@ -106,14 +86,10 @@ github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8 github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= -github.com/bytedance/sonic v1.10.1 h1:7a1wuFXL1cMy7a3f7/VFcEtriuXQnUBhtoVfOZiaysc= -github.com/bytedance/sonic v1.10.1/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.17.1 h1:0SIyjOnkrsfDo88YvPgAWvZMwXe26TP6drRvmkjyUu4= @@ -128,12 +104,9 @@ github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= -github.com/chenzhuoyu/iasm v0.9.0 h1:9fhXjVzq5hUy2gkhhgHl95zG2cEAhw9OSGs8toWWAwo= github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= -github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0 h1:FJFYghXksa6qVfMR1pcUIShMMhyu0ikoKYVgy70/Ze8= -github.com/city404/v6-public-rpc-proto/go v0.0.0-20240621061101-b074815b04a0/go.mod h1:O1dpz9RYVOC21UgZW4vLkBsUzwyM27mEpyEdd83Ia2o= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d h1:p5T6ZPvh7nihJfjI9M/W2cbcX7n766u/OGorLmE4xoQ= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d/go.mod h1:akxZg8LuwOIeCPRjcDrUS1WWcIwmLNSR2lfe4y85PH4= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= @@ -173,20 +146,14 @@ github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5 github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE= -github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw= github.com/gaoyb7/115drive-webdav v0.1.8/go.mod h1:BKbeY6j8SKs3+rzBFFALznGxbPmefEm3vA+dGhqgOGU= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= -github.com/gin-contrib/cors v1.5.0 h1:DgGKV7DDoOn36DFkNtbHrjoRiT5ExCe+PC9/xp7aKvk= -github.com/gin-contrib/cors v1.5.0/go.mod h1:TvU7MAZ3EwrPLI2ztzTt3tqgvBCq+wn8WpZmfADjupI= github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg= github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -214,23 +181,15 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24= -github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= -github.com/go-resty/resty/v2 v2.11.0 h1:i7jMfNOJYMp69lq7qozJP+bjgzfAzeOhuGlyDrqxT/8= -github.com/go-resty/resty/v2 v2.11.0/go.mod h1:iiP/OpA0CkcL3IGt1O0+/SIItFUbkkyw5BGXiVdTu+A= github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-webauthn/webauthn v0.10.0 h1:yuW2e1tXnRAwAvKrR4q4LQmc6XtCMH639/ypZGhZCwk= -github.com/go-webauthn/webauthn v0.10.0/go.mod h1:l0NiauXhL6usIKqNLCUM3Qir43GK7ORg8ggold0Uv/Y= github.com/go-webauthn/webauthn v0.10.2 h1:OG7B+DyuTytrEPFmTX503K77fqs3HDK/0Iv+z8UYbq4= github.com/go-webauthn/webauthn v0.10.2/go.mod h1:Gd1IDsGAybuvK1NkwUTLbGmeksxuRJjVN2PE/xsPxHs= -github.com/go-webauthn/x v0.1.6 h1:QNAX+AWeqRt9loE8mULeWJCqhVG5D/jvdmJ47fIWCkQ= -github.com/go-webauthn/x v0.1.6/go.mod h1:W8dFVZ79o4f+nY1eOUICy/uq5dhrRl7mxQkYhXTo0FA= github.com/go-webauthn/x v0.1.9 h1:v1oeLmoaa+gPOaZqUdDentu6Rl7HkSSsmOT6gxEQHhE= github.com/go-webauthn/x v0.1.9/go.mod h1:pJNMlIMP1SU7cN8HNlKJpLEnFHCygLCvaLZ8a1xeoQA= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -240,8 +199,6 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= -github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= @@ -273,8 +230,6 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksP github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -304,8 +259,6 @@ github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHo github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaevor/go-nanoid v1.3.0 h1:nD+iepesZS6pr3uOVf20vR9GdGgJW1HPaR46gtrxzkg= github.com/jaevor/go-nanoid v1.3.0/go.mod h1:SI+jFaPuddYkqkVQoNGHs81navCtH388TcrH0RqFKgY= -github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= -github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -332,8 +285,6 @@ github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHU github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= -github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= @@ -351,8 +302,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.2.5 h1:MkmkfCHzvmi35EId9SeFPJMZ8bUsijnxwneAWHnnk0k= github.com/larksuite/oapi-sdk-go/v3 v3.2.5/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= @@ -385,14 +334,10 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/meilisearch/meilisearch-go v0.26.1 h1:3bmo2uLijX7kvBmiZ9LupVfC95TFcRJDgrRTzbOoE4A= -github.com/meilisearch/meilisearch-go v0.26.1/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/meilisearch/meilisearch-go v0.27.0 h1:lDFq8WzbsZCtt3/byr7GFqfOygWF5iy9TtDgzJo0Ds8= github.com/meilisearch/meilisearch-go v0.27.0/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= -github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus= -github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw= github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= github.com/minio/sio v0.4.0/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -442,8 +387,6 @@ github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= -github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= @@ -525,7 +468,6 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= @@ -544,8 +486,6 @@ github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4y github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= @@ -565,6 +505,8 @@ github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3K github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.1 h1:O5QY4cVjIGELx3UGh6LbVAc18MWGXgRNQjMt72x6w/8= github.com/xhofe/tache v0.1.1/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= +github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -586,13 +528,10 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= -golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -603,20 +542,13 @@ golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= -golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= -golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -636,15 +568,10 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= -golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -681,11 +608,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -693,11 +617,8 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -709,14 +630,12 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -727,8 +646,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -739,14 +656,8 @@ google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= -google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= -google.golang.org/grpc v1.64.1 h1:LKtvyfbX3UGVPFcGqJ9ItpVWW6oN/2XqTxfAnwRRXiA= -google.golang.org/grpc v1.64.1/go.mod h1:hiQF4LFZelK2WKaP6W0L92zGHtiQdZxk8CrSdvyjeP0= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= From 9de40f89766bd69085e8fe846d343901174d745b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 19:50:31 +0800 Subject: [PATCH 224/659] fix(deps): update module github.com/spf13/cobra to v1.8.1 (#6757) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 26adf349c0c..a1e1a332b84 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/pquerna/otp v1.4.0 github.com/rclone/rclone v1.63.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.8.0 + github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 diff --git a/go.sum b/go.sum index 9a8e57a5659..9b478501104 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,7 @@ github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHo github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -452,6 +453,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= From 9e0482afbb0d3490cf3d98d79fbd0f75b9ac6349 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 19:51:32 +0800 Subject: [PATCH 225/659] fix(deps): update module github.com/larksuite/oapi-sdk-go/v3 to v3.2.8 (#6756) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a1e1a332b84..92c55ed1011 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 - github.com/larksuite/oapi-sdk-go/v3 v3.2.5 + github.com/larksuite/oapi-sdk-go/v3 v3.2.8 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.0 github.com/minio/sio v0.4.0 diff --git a/go.sum b/go.sum index 9b478501104..139728bab53 100644 --- a/go.sum +++ b/go.sum @@ -302,6 +302,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.2.5 h1:MkmkfCHzvmi35EId9SeFPJMZ8bUsijnxwneAWHnnk0k= github.com/larksuite/oapi-sdk-go/v3 v3.2.5/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/larksuite/oapi-sdk-go/v3 v3.2.8 h1:elbufnS+gQVOkzX9JLkS/N9u3ay/IAIE17nE4kNoYZ4= +github.com/larksuite/oapi-sdk-go/v3 v3.2.8/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= From a04da3ec500be37cefa723cb959664e03aa217d9 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 19:06:43 +0800 Subject: [PATCH 226/659] fix(deps): update module github.com/gin-contrib/cors to v1.7.2 (#6770) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 11 +++++++---- go.sum | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 92c55ed1011..bc8d252db4d 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gaoyb7/115drive-webdav v0.1.8 - github.com/gin-contrib/cors v1.6.0 + github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.13.1 github.com/go-webauthn/webauthn v0.10.2 @@ -77,6 +77,9 @@ require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/blevesearch/go-faiss v1.0.19 // indirect github.com/blevesearch/zapx/v16 v16.1.4 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect ) require ( @@ -109,7 +112,7 @@ require ( github.com/blevesearch/zapx/v15 v15.3.13 // indirect github.com/bluele/gcache v0.0.2 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect - github.com/bytedance/sonic v1.11.2 // indirect + github.com/bytedance/sonic v1.11.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect @@ -126,7 +129,7 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.19.0 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/go-webauthn/x v0.1.9 // indirect github.com/goccy/go-json v0.10.2 // indirect @@ -184,7 +187,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/pelletier/go-toml/v2 v2.1.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.1 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect diff --git a/go.sum b/go.sum index 139728bab53..5a34a88ad7f 100644 --- a/go.sum +++ b/go.sum @@ -88,6 +88,10 @@ github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1 github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -109,6 +113,10 @@ github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0 github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d h1:p5T6ZPvh7nihJfjI9M/W2cbcX7n766u/OGorLmE4xoQ= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d/go.mod h1:akxZg8LuwOIeCPRjcDrUS1WWcIwmLNSR2lfe4y85PH4= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= @@ -157,6 +165,8 @@ github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9 github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg= github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= +github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw= +github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= @@ -184,6 +194,8 @@ github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXS github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= @@ -392,6 +404,8 @@ github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OI github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= +github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -462,6 +476,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= From 81b0afc3491a8c9dcc34b57eac949b90eb80ff24 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 19:07:09 +0800 Subject: [PATCH 227/659] fix(deps): update module github.com/dlclark/regexp2 to v1.11.2 (#6769) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bc8d252db4d..f0c010ff732 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 github.com/disintegration/imaging v1.6.2 github.com/djherbis/times v1.6.0 - github.com/dlclark/regexp2 v1.10.0 + github.com/dlclark/regexp2 v1.11.2 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 diff --git a/go.sum b/go.sum index 5a34a88ad7f..c3bac365f6b 100644 --- a/go.sum +++ b/go.sum @@ -146,6 +146,8 @@ github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.2 h1:/u628IuisSTwri5/UKloiIsH8+qF2Pu7xEQX+yIKg68= +github.com/dlclark/regexp2 v1.11.2/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= From 3e949fcf335fea3528a9cff74b277ed3e4c40682 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 19:08:03 +0800 Subject: [PATCH 228/659] fix(deps): update module github.com/charmbracelet/bubbles to v0.18.0 (#6765) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f0c010ff732..0f35da60c99 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/aws/aws-sdk-go v1.50.24 github.com/blevesearch/bleve/v2 v2.4.1 github.com/caarlos0/env/v9 v9.0.0 - github.com/charmbracelet/bubbles v0.17.1 + github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/lipgloss v0.9.1 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d @@ -197,7 +197,7 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rfjakob/eme v1.1.2 // indirect - github.com/rivo/uniseg v0.4.4 // indirect + github.com/rivo/uniseg v0.4.6 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect diff --git a/go.sum b/go.sum index c3bac365f6b..6736e8f08c9 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.17.1 h1:0SIyjOnkrsfDo88YvPgAWvZMwXe26TP6drRvmkjyUu4= github.com/charmbracelet/bubbles v0.17.1/go.mod h1:9HxZWlkCqz2PRwsCbYl7a3KXvGzFaDHpYbSYMJ+nE3o= +github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= +github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= @@ -444,6 +446,8 @@ github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg= +github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= From 3c483ace4f0e9b7ae1c5a3e9381245dc61594107 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 19:09:38 +0800 Subject: [PATCH 229/659] fix(deps): update module gorm.io/driver/sqlite to v1.5.6 (#6763) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0f35da60c99..d0e3994ba2e 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 - gorm.io/driver/sqlite v1.5.5 + gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.10 ) diff --git a/go.sum b/go.sum index 6736e8f08c9..d13795c121a 100644 --- a/go.sum +++ b/go.sum @@ -719,6 +719,8 @@ gorm.io/driver/postgres v1.4.8 h1:NDWizaclb7Q2aupT0jkwK8jx1HVCNzt+PQ8v/VnxviA= gorm.io/driver/postgres v1.4.8/go.mod h1:O9MruWGNLUBUWVYfWuBClpf3HeGjOoybY0SNmCs3wsw= gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= +gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= +gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= From cd663f78af61ed6c59fe2698d13220513ceb0666 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 19:36:10 +0800 Subject: [PATCH 230/659] fix(deps): update module github.com/charmbracelet/bubbletea to v0.26.6 (#6766) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 12 +++++++++--- go.sum | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d0e3994ba2e..3ef80de217c 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/blevesearch/bleve/v2 v2.4.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.18.0 - github.com/charmbracelet/bubbletea v0.25.0 + github.com/charmbracelet/bubbletea v0.26.6 github.com/charmbracelet/lipgloss v0.9.1 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d github.com/coreos/go-oidc v2.2.1+incompatible @@ -78,8 +78,14 @@ require ( github.com/blevesearch/go-faiss v1.0.19 // indirect github.com/blevesearch/zapx/v16 v16.1.4 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/charmbracelet/x/ansi v0.1.2 // indirect + github.com/charmbracelet/x/input v0.1.0 // indirect + github.com/charmbracelet/x/term v0.1.1 // indirect + github.com/charmbracelet/x/windows v0.1.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect ) require ( @@ -175,7 +181,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.2 // indirect @@ -197,7 +203,7 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/rfjakob/eme v1.1.2 // indirect - github.com/rivo/uniseg v0.4.6 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect github.com/shirou/gopsutil/v3 v3.23.7 // indirect diff --git a/go.sum b/go.sum index d13795c121a..7bf7368844d 100644 --- a/go.sum +++ b/go.sum @@ -102,8 +102,18 @@ github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/ github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= +github.com/charmbracelet/bubbletea v0.26.6 h1:zTCWSuST+3yZYZnVSvbXwKOPRSNZceVeqpzOLN2zq1s= +github.com/charmbracelet/bubbletea v0.26.6/go.mod h1:dz8CWPlfCCGLFbBlTY4N7bjLiyOGDJEnd2Muu7pOWhk= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= +github.com/charmbracelet/x/ansi v0.1.2 h1:6+LR39uG8DE6zAmbu023YlqjJHkYXDF1z36ZwzO4xZY= +github.com/charmbracelet/x/ansi v0.1.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/input v0.1.0 h1:TEsGSfZYQyOtp+STIjyBq6tpRaorH0qpwZUj8DavAhQ= +github.com/charmbracelet/x/input v0.1.0/go.mod h1:ZZwaBxPF7IG8gWWzPUVqHEtWhc1+HXJPNuerJGRGZ28= +github.com/charmbracelet/x/term v0.1.1 h1:3cosVAiPOig+EV4X9U+3LDgtwwAoEzJjNdwbXDjF6yI= +github.com/charmbracelet/x/term v0.1.1/go.mod h1:wB1fHt5ECsu3mXYusyzcngVWWlu1KKUmmLhfgr/Flxw= +github.com/charmbracelet/x/windows v0.1.0 h1:gTaxdvzDM5oMa/I2ZNF7wN78X/atWemG9Wph7Ika2k4= +github.com/charmbracelet/x/windows v0.1.0/go.mod h1:GLEO/l+lizvFDBPLIOk+49gdX49L9YWMB5t+DZd0jkQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -152,6 +162,8 @@ github.com/dlclark/regexp2 v1.11.2 h1:/u628IuisSTwri5/UKloiIsH8+qF2Pu7xEQX+yIKg6 github.com/dlclark/regexp2 v1.11.2/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= @@ -376,6 +388,8 @@ github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= @@ -448,6 +462,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg= github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -533,6 +549,8 @@ github.com/xhofe/tache v0.1.1 h1:O5QY4cVjIGELx3UGh6LbVAc18MWGXgRNQjMt72x6w/8= github.com/xhofe/tache v0.1.1/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -620,6 +638,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 96297051007f572e88bf5fa5413d7fe490129d75 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 19:40:46 +0800 Subject: [PATCH 231/659] fix(deps): update module gorm.io/gorm to v1.25.11 (#6764) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3ef80de217c..86da9fef6aa 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( gorm.io/driver/mysql v1.4.7 gorm.io/driver/postgres v1.4.8 gorm.io/driver/sqlite v1.5.6 - gorm.io/gorm v1.25.10 + gorm.io/gorm v1.25.11 ) require ( diff --git a/go.sum b/go.sum index 7bf7368844d..2d67c96ae13 100644 --- a/go.sum +++ b/go.sum @@ -744,6 +744,8 @@ gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= From 9128647970ca484cf24f66901e8742db4167fcfa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 19:41:23 +0800 Subject: [PATCH 232/659] fix(deps): update module github.com/rclone/rclone to v1.67.0 (#6780) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 32 ++++++++++++++++---------------- go.sum | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 86da9fef6aa..2aea974db45 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/alist-org/gofakes3 v0.0.5 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.50.24 + github.com/aws/aws-sdk-go v1.53.7 github.com/blevesearch/bleve/v2 v2.4.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.18.0 @@ -48,7 +48,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 github.com/pquerna/otp v1.4.0 - github.com/rclone/rclone v1.63.1 + github.com/rclone/rclone v1.67.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 @@ -131,8 +131,8 @@ require ( github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-chi/chi/v5 v5.0.10 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-chi/chi/v5 v5.0.12 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect @@ -158,7 +158,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -166,7 +166,7 @@ require ( github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p v0.27.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect - github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -198,21 +198,21 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/rfjakob/eme v1.1.2 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect - github.com/shirou/gopsutil/v3 v3.23.7 // indirect + github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect + github.com/shirou/gopsutil/v3 v3.24.4 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tklauser/go-sysconf v0.3.11 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.13 // indirect + github.com/tklauser/numcpus v0.7.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u2takey/go-utils v0.3.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect @@ -220,8 +220,8 @@ require ( github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.etcd.io/bbolt v1.3.7 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.7.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect diff --git a/go.sum b/go.sum index 2d67c96ae13..2a22c5adb4a 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.50.24 h1:3o2Pg7mOoVL0jv54vWtuafoZqAeEXLhm1tltWA2GcEw= github.com/aws/aws-sdk-go v1.50.24/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.53.7 h1:ZSsRYHLRxsbO2rJR2oPMz0SUkJLnBkN+1meT95B6Ixs= +github.com/aws/aws-sdk-go v1.53.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= @@ -190,6 +192,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= +github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -197,6 +201,8 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -313,6 +319,8 @@ github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -346,6 +354,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= +github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= +github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= @@ -446,14 +456,24 @@ github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rclone/rclone v1.63.1 h1:iITCUNBfAXnguHjRPFq+w/gGIW0L0las78h4H5CH2Ms= github.com/rclone/rclone v1.63.1/go.mod h1:eUQaKsf1wJfHKB0RDoM8RaPAeRB2eI/Qw+Vc9Ho5FGM= +github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= +github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -473,8 +493,12 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= +github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= +github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= +github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= +github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -517,9 +541,14 @@ github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= +github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= +github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= @@ -556,10 +585,14 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 h1:X+lHsNTlbatQ1cErXIbtyrh+3MTWxqQFS+sBP/wpFXo= github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22/go.mod h1:1zGRDJd8zlG6P8azG96+uywfh6udYWwhOmUivw+xsuM= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= @@ -653,7 +686,9 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= From 83048e6c7cd19fecac17d3de7d14567247cc8c55 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 20:22:50 +0800 Subject: [PATCH 233/659] fix(deps): update module github.com/charmbracelet/lipgloss to v0.12.1 (#6768) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 2aea974db45..3be02b60b54 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.26.6 - github.com/charmbracelet/lipgloss v0.9.1 + github.com/charmbracelet/lipgloss v0.12.1 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.6.0 @@ -78,7 +78,7 @@ require ( github.com/blevesearch/go-faiss v1.0.19 // indirect github.com/blevesearch/zapx/v16 v16.1.4 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect - github.com/charmbracelet/x/ansi v0.1.2 // indirect + github.com/charmbracelet/x/ansi v0.1.4 // indirect github.com/charmbracelet/x/input v0.1.0 // indirect github.com/charmbracelet/x/term v0.1.1 // indirect github.com/charmbracelet/x/windows v0.1.0 // indirect diff --git a/go.sum b/go.sum index 2a22c5adb4a..8ee52ae6499 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,12 @@ github.com/charmbracelet/bubbletea v0.26.6 h1:zTCWSuST+3yZYZnVSvbXwKOPRSNZceVeqp github.com/charmbracelet/bubbletea v0.26.6/go.mod h1:dz8CWPlfCCGLFbBlTY4N7bjLiyOGDJEnd2Muu7pOWhk= github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= +github.com/charmbracelet/lipgloss v0.12.1 h1:/gmzszl+pedQpjCOH+wFkZr/N90Snz40J/NR7A0zQcs= +github.com/charmbracelet/lipgloss v0.12.1/go.mod h1:V2CiwIuhx9S1S1ZlADfOj9HmxeMAORuz5izHb0zGbB8= github.com/charmbracelet/x/ansi v0.1.2 h1:6+LR39uG8DE6zAmbu023YlqjJHkYXDF1z36ZwzO4xZY= github.com/charmbracelet/x/ansi v0.1.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= +github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/input v0.1.0 h1:TEsGSfZYQyOtp+STIjyBq6tpRaorH0qpwZUj8DavAhQ= github.com/charmbracelet/x/input v0.1.0/go.mod h1:ZZwaBxPF7IG8gWWzPUVqHEtWhc1+HXJPNuerJGRGZ28= github.com/charmbracelet/x/term v0.1.1 h1:3cosVAiPOig+EV4X9U+3LDgtwwAoEzJjNdwbXDjF6yI= From fe1040a36731e68090ab3feb6c2930cc860aa198 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 14 Jul 2024 20:29:23 +0800 Subject: [PATCH 234/659] chore(lark): don't use `github.com/ipfs/boxo/path` --- drivers/lark/driver.go | 19 ++++----- go.mod | 7 +--- go.sum | 89 +----------------------------------------- 3 files changed, 13 insertions(+), 102 deletions(-) diff --git a/drivers/lark/driver.go b/drivers/lark/driver.go index 6783aa5241d..d2672300444 100644 --- a/drivers/lark/driver.go +++ b/drivers/lark/driver.go @@ -4,18 +4,19 @@ import ( "context" "errors" "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/ipfs/boxo/path" lark "github.com/larksuite/oapi-sdk-go/v3" larkcore "github.com/larksuite/oapi-sdk-go/v3/core" larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" "golang.org/x/time/rate" - "io" - "net/http" - "strconv" - "time" ) type Lark struct { @@ -37,7 +38,7 @@ func (c *Lark) GetAddition() driver.Additional { func (c *Lark) Init(ctx context.Context) error { c.client = lark.NewClient(c.AppId, c.AppSecret, lark.WithTokenCache(newTokenCache())) - paths := path.SplitList(c.RootFolderPath) + paths := strings.Split(c.RootFolderPath, "/") token := "" var ok bool @@ -113,7 +114,7 @@ func (c *Lark) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] f := model.Object{ ID: *file.Token, - Path: path.Join([]string{c.RootFolderPath, dir.GetPath(), *file.Name}), + Path: strings.Join([]string{c.RootFolderPath, dir.GetPath(), *file.Name}, "/"), Name: *file.Name, Size: 0, Modified: time.Unix(modifiedUnix, 0), @@ -170,7 +171,7 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* }, }, nil } else { - url := path.Join([]string{c.TenantUrlPrefix, "file", token}) + url := strings.Join([]string{c.TenantUrlPrefix, "file", token}, "/") return &model.Link{ URL: url, @@ -201,7 +202,7 @@ func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) return &model.Object{ ID: *resp.Data.Token, - Path: path.Join([]string{c.RootFolderPath, parentDir.GetPath(), dirName}), + Path: strings.Join([]string{c.RootFolderPath, parentDir.GetPath(), dirName}, "/"), Name: dirName, Size: 0, IsFolder: true, diff --git a/go.mod b/go.mod index 3be02b60b54..8e3e5522544 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,6 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hirochachacha/go-smb2 v1.1.0 - github.com/ipfs/boxo v0.12.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 @@ -85,6 +84,7 @@ require ( github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/ipfs/boxo v0.12.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect ) @@ -120,9 +120,6 @@ require ( github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect - github.com/chenzhuoyu/iasm v0.9.1 // indirect - github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect @@ -173,7 +170,6 @@ require ( github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -183,7 +179,6 @@ require ( github.com/mschoch/smat v0.2.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect - github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.15.2 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect diff --git a/go.sum b/go.sum index 8ee52ae6499..9fb869f5d35 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,6 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.50.24 h1:3o2Pg7mOoVL0jv54vWtuafoZqAeEXLhm1tltWA2GcEw= -github.com/aws/aws-sdk-go v1.50.24/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.53.7 h1:ZSsRYHLRxsbO2rJR2oPMz0SUkJLnBkN+1meT95B6Ixs= github.com/aws/aws-sdk-go v1.53.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= @@ -86,10 +84,6 @@ github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= -github.com/bytedance/sonic v1.11.2 h1:ywfwo0a/3j9HR8wsYGWsIWl2mvRsI950HyoxiBERw5A= -github.com/bytedance/sonic v1.11.2/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf50Hi0u/g4= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= @@ -98,20 +92,12 @@ github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/bubbles v0.17.1 h1:0SIyjOnkrsfDo88YvPgAWvZMwXe26TP6drRvmkjyUu4= -github.com/charmbracelet/bubbles v0.17.1/go.mod h1:9HxZWlkCqz2PRwsCbYl7a3KXvGzFaDHpYbSYMJ+nE3o= github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= -github.com/charmbracelet/bubbletea v0.25.0 h1:bAfwk7jRz7FKFl9RzlIULPkStffg5k6pNt5dywy4TcM= -github.com/charmbracelet/bubbletea v0.25.0/go.mod h1:EN3QDR1T5ZdWmdfDzYcqOCAps45+QIJbLOBxmVNWNNg= github.com/charmbracelet/bubbletea v0.26.6 h1:zTCWSuST+3yZYZnVSvbXwKOPRSNZceVeqpzOLN2zq1s= github.com/charmbracelet/bubbletea v0.26.6/go.mod h1:dz8CWPlfCCGLFbBlTY4N7bjLiyOGDJEnd2Muu7pOWhk= -github.com/charmbracelet/lipgloss v0.9.1 h1:PNyd3jvaJbg4jRHKWXnCj1akQm4rh8dbEzN1p/u1KWg= -github.com/charmbracelet/lipgloss v0.9.1/go.mod h1:1mPmG4cxScwUQALAAnacHaigiiHB9Pmr+v1VEawJl6I= github.com/charmbracelet/lipgloss v0.12.1 h1:/gmzszl+pedQpjCOH+wFkZr/N90Snz40J/NR7A0zQcs= github.com/charmbracelet/lipgloss v0.12.1/go.mod h1:V2CiwIuhx9S1S1ZlADfOj9HmxeMAORuz5izHb0zGbB8= -github.com/charmbracelet/x/ansi v0.1.2 h1:6+LR39uG8DE6zAmbu023YlqjJHkYXDF1z36ZwzO4xZY= -github.com/charmbracelet/x/ansi v0.1.2/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/input v0.1.0 h1:TEsGSfZYQyOtp+STIjyBq6tpRaorH0qpwZUj8DavAhQ= @@ -122,26 +108,16 @@ github.com/charmbracelet/x/windows v0.1.0 h1:gTaxdvzDM5oMa/I2ZNF7wN78X/atWemG9Wp github.com/charmbracelet/x/windows v0.1.0/go.mod h1:GLEO/l+lizvFDBPLIOk+49gdX49L9YWMB5t+DZd0jkQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= -github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d h1:77cEq6EriyTZ0g/qfRdp61a3Uu/AWrgIq2s0ClJV1g0= -github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d/go.mod h1:8EPpVsBuRksnlj1mLy4AWzRNQYxauNi62uWcE3to6eA= -github.com/chenzhuoyu/iasm v0.9.0/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= -github.com/chenzhuoyu/iasm v0.9.1 h1:tUHQJXo3NhBqw6s33wkGn9SP3bvrWLdlVIJ3hQBL7P0= -github.com/chenzhuoyu/iasm v0.9.1/go.mod h1:Xjy2NpN3h7aUqeqM+woSuuvxmIe6+DDsiNLIrkAmYog= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d h1:p5T6ZPvh7nihJfjI9M/W2cbcX7n766u/OGorLmE4xoQ= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d/go.mod h1:akxZg8LuwOIeCPRjcDrUS1WWcIwmLNSR2lfe4y85PH4= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= -github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= @@ -162,8 +138,6 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= -github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0= -github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.2 h1:/u628IuisSTwri5/UKloiIsH8+qF2Pu7xEQX+yIKg68= github.com/dlclark/regexp2 v1.11.2/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= @@ -185,8 +159,6 @@ github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKe github.com/gaoyb7/115drive-webdav v0.1.8/go.mod h1:BKbeY6j8SKs3+rzBFFALznGxbPmefEm3vA+dGhqgOGU= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= -github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg= -github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw= github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -194,8 +166,6 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= -github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -203,7 +173,6 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -218,8 +187,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn0+wvQ3bZ8b/AU4= -github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= @@ -244,7 +211,6 @@ github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFig github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -321,8 +287,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -340,8 +304,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/larksuite/oapi-sdk-go/v3 v3.2.5 h1:MkmkfCHzvmi35EId9SeFPJMZ8bUsijnxwneAWHnnk0k= -github.com/larksuite/oapi-sdk-go/v3 v3.2.5/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/larksuite/oapi-sdk-go/v3 v3.2.8 h1:elbufnS+gQVOkzX9JLkS/N9u3ay/IAIE17nE4kNoYZ4= github.com/larksuite/oapi-sdk-go/v3 v3.2.8/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -356,8 +318,6 @@ github.com/libp2p/go-libp2p v0.27.8/go.mod h1:eCFFtd0s5i/EVKR7+5Ki8bM7qwkNW3TPTT github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= -github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -372,13 +332,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= -github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/meilisearch/meilisearch-go v0.27.0 h1:lDFq8WzbsZCtt3/byr7GFqfOygWF5iy9TtDgzJo0Ds8= github.com/meilisearch/meilisearch-go v0.27.0/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= @@ -400,14 +357,10 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= -github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= -github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= -github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= @@ -434,8 +387,6 @@ github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= -github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= @@ -458,49 +409,30 @@ github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8 github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/rclone/rclone v1.63.1 h1:iITCUNBfAXnguHjRPFq+w/gGIW0L0las78h4H5CH2Ms= -github.com/rclone/rclone v1.63.1/go.mod h1:eUQaKsf1wJfHKB0RDoM8RaPAeRB2eI/Qw+Vc9Ho5FGM= github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rivo/uniseg v0.4.6 h1:Sovz9sDSwbOz9tgUy8JpT+KgCkPYJEN/oYzlJiYTNLg= -github.com/rivo/uniseg v0.4.6/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 h1:WnNuhiq+FOY3jNj6JXFT+eLN3CQ/oPIsDPRanvwsmbI= -github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= -github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= -github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= @@ -517,8 +449,6 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:s github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -543,13 +473,9 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= @@ -587,14 +513,10 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJu github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 h1:X+lHsNTlbatQ1cErXIbtyrh+3MTWxqQFS+sBP/wpFXo= github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22/go.mod h1:1zGRDJd8zlG6P8azG96+uywfh6udYWwhOmUivw+xsuM= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= @@ -655,7 +577,6 @@ golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -685,11 +606,9 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -775,14 +694,10 @@ gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= gorm.io/driver/postgres v1.4.8 h1:NDWizaclb7Q2aupT0jkwK8jx1HVCNzt+PQ8v/VnxviA= gorm.io/driver/postgres v1.4.8/go.mod h1:O9MruWGNLUBUWVYfWuBClpf3HeGjOoybY0SNmCs3wsw= -gorm.io/driver/sqlite v1.5.5 h1:7MDMtUZhV065SilG62E0MquljeArQZNfJnjd9i9gx3E= -gorm.io/driver/sqlite v1.5.5/go.mod h1:6NgQ7sQWAIFsPrJJl1lSNSu2TABh0ZZ/zm5fosATavE= gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= -gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= -gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= From 17f78b948aef0135f62634b1ef75991570b7e01e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 20:58:20 +0800 Subject: [PATCH 235/659] fix(deps): update module gorm.io/driver/mysql to v1.5.7 (#6782) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8e3e5522544..3265f00fc82 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 - gorm.io/driver/mysql v1.4.7 + gorm.io/driver/mysql v1.5.7 gorm.io/driver/postgres v1.4.8 gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.11 diff --git a/go.sum b/go.sum index 9fb869f5d35..99ecf9ebf66 100644 --- a/go.sum +++ b/go.sum @@ -692,12 +692,15 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= +gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= +gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.4.8 h1:NDWizaclb7Q2aupT0jkwK8jx1HVCNzt+PQ8v/VnxviA= gorm.io/driver/postgres v1.4.8/go.mod h1:O9MruWGNLUBUWVYfWuBClpf3HeGjOoybY0SNmCs3wsw= gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= From a6ff6a94df55293aba376d2a54ba3dd5570f9e34 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 20:58:41 +0800 Subject: [PATCH 236/659] fix(deps): update module golang.org/x/oauth2 to v0.21.0 (#6781) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 3265f00fc82..8f7e469076f 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( golang.org/x/exp v0.0.0-20240707233637-46b078467d37 golang.org/x/image v0.18.0 golang.org/x/net v0.27.0 - golang.org/x/oauth2 v0.20.0 + golang.org/x/oauth2 v0.21.0 golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 diff --git a/go.sum b/go.sum index 99ecf9ebf66..011b9379cc7 100644 --- a/go.sum +++ b/go.sum @@ -577,6 +577,8 @@ golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= +golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 736ba4403133d1ea90e52922c0653392906d9b96 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 20:58:55 +0800 Subject: [PATCH 237/659] fix(deps): update module github.com/gin-gonic/gin to v1.10.0 (#6771) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8f7e469076f..de051105170 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gaoyb7/115drive-webdav v0.1.8 github.com/gin-contrib/cors v1.7.2 - github.com/gin-gonic/gin v1.9.1 + github.com/gin-gonic/gin v1.10.0 github.com/go-resty/resty/v2 v2.13.1 github.com/go-webauthn/webauthn v0.10.2 github.com/golang-jwt/jwt/v4 v4.5.0 @@ -188,7 +188,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/pelletier/go-toml/v2 v2.2.1 // indirect + github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect @@ -217,7 +217,7 @@ require ( github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect - golang.org/x/arch v0.7.0 // indirect + golang.org/x/arch v0.8.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.22.0 // indirect golang.org/x/term v0.22.0 // indirect diff --git a/go.sum b/go.sum index 011b9379cc7..f69b7af0942 100644 --- a/go.sum +++ b/go.sum @@ -166,6 +166,8 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= @@ -389,6 +391,8 @@ github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OI github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -533,6 +537,8 @@ gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= From 8278d3875ba01360d8f1cce1f1c921c9ea4b234a Mon Sep 17 00:00:00 2001 From: seiuneko <25706824+seiuneko@users.noreply.github.com> Date: Sun, 14 Jul 2024 20:59:24 +0800 Subject: [PATCH 238/659] fix: ignore os.ErrClosed error on repeated FileStream close operations (#6762) Also resolves the issue where S3 PutObject returns a 500 status code. --- internal/stream/stream.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 40482f45a36..2c9543c1d94 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -51,7 +51,11 @@ func (f *FileStream) IsForceStreamUpload() bool { func (f *FileStream) Close() error { var err1, err2 error + err1 = f.Closers.Close() + if errors.Is(err1, os.ErrClosed) { + err1 = nil + } if f.tmpFile != nil { err2 = os.RemoveAll(f.tmpFile.Name()) if err2 != nil { From 488ebaa1aff0797816dfaa789745d998889ff755 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 14 Jul 2024 21:04:27 +0800 Subject: [PATCH 239/659] fix(deps): update module github.com/aws/aws-sdk-go to v1.54.19 (#6170) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index de051105170..c852b13715f 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/alist-org/gofakes3 v0.0.5 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.53.7 + github.com/aws/aws-sdk-go v1.54.19 github.com/blevesearch/bleve/v2 v2.4.1 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.18.0 diff --git a/go.sum b/go.sum index f69b7af0942..3900a728bda 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.53.7 h1:ZSsRYHLRxsbO2rJR2oPMz0SUkJLnBkN+1meT95B6Ixs= github.com/aws/aws-sdk-go v1.53.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI= +github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= From a93937f80da8011c2dbd6464967d190b20ed8301 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sun, 14 Jul 2024 21:07:00 +0800 Subject: [PATCH 240/659] fix(pikpak): add captcha_token generation function (#6775) closes #6752 closes #6760 --- drivers/pikpak/driver.go | 42 ++++++- drivers/pikpak/types.go | 65 +++++++++++ drivers/pikpak/util.go | 237 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 342 insertions(+), 2 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 3ecc31d6bff..23198455a74 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/alist-org/alist/v3/internal/op" "net/http" "strconv" "strings" @@ -25,7 +26,7 @@ import ( type PikPak struct { model.Storage Addition - + *Common oauth2Token oauth2.TokenSource } @@ -43,6 +44,20 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" } + if d.Common == nil { + d.Common = &Common{ + client: base.NewRestyClient(), + CaptchaToken: "", + UserID: "", + DeviceID: utils.GetMD5EncodeStr(d.Username + d.Password), + UserAgent: BuildCustomUserAgent(utils.GetMD5EncodeStr(d.Username+d.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, ""), + RefreshCTokenCk: func(token string) { + d.Common.CaptchaToken = token + op.MustSaveDriverStorage(d) + }, + } + } + oauth2Config := &oauth2.Config{ ClientID: d.ClientID, ClientSecret: d.ClientSecret, @@ -60,6 +75,14 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.Password, ) })) + + // 获取CaptchaToken + _ = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Username) + + // 获取用户ID + _ = d.GetUserID(ctx) + // 更新UserAgent + d.Common.UserAgent = BuildCustomUserAgent(d.Common.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, d.Common.UserID) return nil } @@ -79,7 +102,7 @@ func (d *PikPak) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp File - _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.com/drive/v1/files/%s?_magic=2021&thumbnail_size=SIZE_LARGE", file.GetID()), + _, err := d.requestWithCaptchaToken(fmt.Sprintf("https://api-drive.mypikpak.com/drive/v1/files/%s?_magic=2021&thumbnail_size=SIZE_LARGE", file.GetID()), http.MethodGet, nil, &resp) if err != nil { return nil, err @@ -297,4 +320,19 @@ func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, delet return nil } +func (d *PikPak) GetUserID(ctx context.Context) error { + url := "https://api-drive.mypikpak.com/vip/v1/vip/info" + var resp VipInfo + _, err := d.requestWithCaptchaToken(url, http.MethodGet, func(req *resty.Request) { + req.SetContext(ctx) + }, &resp) + if err != nil { + return fmt.Errorf("failed to get user id : %w", err) + } + if resp.Data.UserID != "" { + d.Common.SetUserID(resp.Data.UserID) + } + return nil +} + var _ driver.Driver = (*PikPak)(nil) diff --git a/drivers/pikpak/types.go b/drivers/pikpak/types.go index a9928d00ec2..a831642e27f 100644 --- a/drivers/pikpak/types.go +++ b/drivers/pikpak/types.go @@ -1,6 +1,7 @@ package pikpak import ( + "fmt" "strconv" "time" @@ -168,3 +169,67 @@ type ReferenceResource struct { Tags []string `json:"tags"` ThumbnailLink string `json:"thumbnail_link"` } + +type ErrResp struct { + ErrorCode int64 `json:"error_code"` + ErrorMsg string `json:"error"` + ErrorDescription string `json:"error_description"` + // ErrorDetails interface{} `json:"error_details"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) +} + +type CaptchaTokenRequest struct { + Action string `json:"action"` + CaptchaToken string `json:"captcha_token"` + ClientID string `json:"client_id"` + DeviceID string `json:"device_id"` + Meta map[string]string `json:"meta"` + RedirectUri string `json:"redirect_uri"` +} + +type CaptchaTokenResponse struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Url string `json:"url"` +} + +type VipInfo struct { + Data struct { + Expire time.Time `json:"expire"` + ExtUserInfo struct { + UserRegion string `json:"userRegion"` + } `json:"extUserInfo"` + ExtType string `json:"ext_type"` + FeeRecord string `json:"fee_record"` + Restricted struct { + Result bool `json:"result"` + Content struct { + Text string `json:"text"` + Color string `json:"color"` + DeepLink string `json:"deepLink"` + } `json:"content"` + LearnMore struct { + Text string `json:"text"` + Color string `json:"color"` + DeepLink string `json:"deepLink"` + } `json:"learnMore"` + } `json:"restricted"` + Status string `json:"status"` + Type string `json:"type"` + UserID string `json:"user_id"` + VipItem []struct { + Type string `json:"type"` + Description string `json:"description"` + Status string `json:"status"` + Expire time.Time `json:"expire"` + SurplusDay int `json:"surplus_day"` + } `json:"vipItem"` + } `json:"data"` +} diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 0edfc384eba..a794001a3b3 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -1,8 +1,16 @@ package pikpak import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" "errors" + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" "net/http" + "regexp" + "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/go-resty/resty/v2" @@ -10,6 +18,26 @@ import ( // do others that not defined in Driver interface +var Algorithms = []string{ + "PAe56I7WZ6FCSkFy77A96jHWcQA27ui80Qy4", + "SUbmk67TfdToBAEe2cZyP8vYVeN", + "1y3yFSZVWiGN95fw/2FQlRuH/Oy6WnO", + "8amLtHJpGzHPz4m9hGz7r+i+8dqQiAk", + "tmIEq5yl2g/XWwM3sKZkY4SbL8YUezrvxPksNabUJ", + "4QvudeJwgJuSf/qb9/wjC21L5aib", + "D1RJd+FZ+LBbt+dAmaIyYrT9gxJm0BB", + "1If", + "iGZr/SJPUFRkwvC174eelKy", +} + +const ( + ClientID = "YNxT9w7GMdWvEOKa" + ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + ClientVersion = "1.46.2" + PackageName = "com.pikcloud.pikpak" + SdkVersion = "2.0.4.204000 " +) + func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() @@ -38,6 +66,44 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r return res.Body(), nil } +func (d *PikPak) requestWithCaptchaToken(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + + data, err := d.request(url, method, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "User-Agent": d.GetUserAgent(), + "X-Device-ID": d.GetDeviceID(), + "X-Captcha-Token": d.GetCaptchaToken(), + }) + if callback != nil { + callback(req) + } + }, resp) + + errResp, ok := err.(*ErrResp) + if !ok { + return nil, err + } + + switch errResp.ErrorCode { + case 0: + return data, nil + //case 4122, 4121, 10, 16: + // if d.refreshTokenFunc != nil { + // if err = xc.refreshTokenFunc(); err == nil { + // break + // } + // } + // return nil, err + case 9: // 验证码token过期 + if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.Common.UserID); err != nil { + return nil, err + } + default: + return nil, err + } + return d.request(url, method, callback, resp) +} + func (d *PikPak) getFiles(id string) ([]File, error) { res := make([]File, 0) pageToken := "first" @@ -65,3 +131,174 @@ func (d *PikPak) getFiles(id string) ([]File, error) { } return res, nil } + +func GetAction(method string, url string) string { + urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] + return method + ":" + urlpath +} + +type Common struct { + client *resty.Client + CaptchaToken string + UserID string + // 必要值,签名相关 + DeviceID string + UserAgent string + // 验证码token刷新成功回调 + RefreshCTokenCk func(token string) +} + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} + +func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { + deviceSign := generateDeviceSign(deviceID, packageName) + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) + sb.WriteString("protocolVersion/200 ") + sb.WriteString("accesstype/ ") + sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) + sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) + sb.WriteString("action_type/ ") + sb.WriteString("networktype/WIFI ") + sb.WriteString("sessionid/ ") + sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) + sb.WriteString("providername/NONE ") + sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) + sb.WriteString("refresh_token/ ") + sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) + sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) + sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) + sb.WriteString(fmt.Sprintf("appname/android-%s ", appName)) + sb.WriteString(fmt.Sprintf("session_origin/ ")) + sb.WriteString(fmt.Sprintf("grant_type/ ")) + sb.WriteString(fmt.Sprintf("appid/ ")) + sb.WriteString(fmt.Sprintf("clientip/ ")) + sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) + sb.WriteString(fmt.Sprintf("osversion/13 ")) + sb.WriteString(fmt.Sprintf("platformversion/10 ")) + sb.WriteString(fmt.Sprintf("accessmode/ ")) + sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) + + return sb.String() +} + +func (c *Common) SetDeviceID(deviceID string) { + c.DeviceID = deviceID +} + +func (c *Common) SetUserID(userID string) { + c.UserID = userID +} + +func (c *Common) SetUserAgent(userAgent string) { + c.UserAgent = userAgent +} + +func (c *Common) SetCaptchaToken(captchaToken string) { + c.CaptchaToken = captchaToken +} +func (c *Common) GetCaptchaToken() string { + return c.CaptchaToken +} + +func (c *Common) GetUserAgent() string { + return c.UserAgent +} + +func (c *Common) GetDeviceID() string { + return c.DeviceID +} + +// RefreshCaptchaTokenAtLogin 刷新验证码token(登录后) +func (d *PikPak) RefreshCaptchaTokenAtLogin(action, userID string) error { + metas := map[string]string{ + "client_version": ClientVersion, + "package_name": PackageName, + "user_id": userID, + } + metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() + return d.refreshCaptchaToken(action, metas) +} + +// RefreshCaptchaTokenInLogin 刷新验证码token(登录时) +func (d *PikPak) RefreshCaptchaTokenInLogin(action, username string) error { + metas := make(map[string]string) + if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { + metas["email"] = username + } else if len(username) >= 11 && len(username) <= 18 { + metas["phone_number"] = username + } else { + metas["username"] = username + } + return d.refreshCaptchaToken(action, metas) +} + +// GetCaptchaSign 获取验证码签名 +func (c *Common) GetCaptchaSign() (timestamp, sign string) { + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(ClientID, ClientVersion, PackageName, c.DeviceID, timestamp) + for _, algorithm := range Algorithms { + str = utils.GetMD5EncodeStr(str + algorithm) + } + sign = "1." + str + return +} + +// 刷新验证码token +func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) error { + param := CaptchaTokenRequest{ + Action: action, + CaptchaToken: d.Common.CaptchaToken, + ClientID: ClientID, + DeviceID: d.Common.DeviceID, + Meta: metas, + RedirectUri: "xlaccsdk01://xbase.cloud/callback?state=harbor", + } + var e ErrResp + var resp CaptchaTokenResponse + _, err := d.request("https://user.mypikpak.com/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + req.SetError(&e).SetBody(param) + }, &resp) + + if err != nil { + return err + } + + if e.IsError() { + return &e + } + + if resp.Url != "" { + return fmt.Errorf(`need verify: Click Here`, resp.Url) + } + + if resp.CaptchaToken == "" { + return fmt.Errorf("empty captchaToken") + } + + if d.Common.RefreshCTokenCk != nil { + d.Common.RefreshCTokenCk(resp.CaptchaToken) + } + d.Common.SetCaptchaToken(resp.CaptchaToken) + return nil +} From 049575b5a524d96dcfaad8df29fd15812a165b9d Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:00:05 +0800 Subject: [PATCH 241/659] fix(pikpak): captcha_token not refreshing correctly (#6788) --- drivers/pikpak/driver.go | 26 +++++++++++++------------- drivers/pikpak/types.go | 40 ---------------------------------------- drivers/pikpak/util.go | 11 +++++------ 3 files changed, 18 insertions(+), 59 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 23198455a74..1d086e0a5cf 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -76,11 +76,11 @@ func (d *PikPak) Init(ctx context.Context) (err error) { ) })) - // 获取CaptchaToken - _ = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Username) - // 获取用户ID - _ = d.GetUserID(ctx) + _ = d.GetUserID() + + // 获取CaptchaToken + _ = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Common.UserID) // 更新UserAgent d.Common.UserAgent = BuildCustomUserAgent(d.Common.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, d.Common.UserID) return nil @@ -320,17 +320,17 @@ func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, delet return nil } -func (d *PikPak) GetUserID(ctx context.Context) error { - url := "https://api-drive.mypikpak.com/vip/v1/vip/info" - var resp VipInfo - _, err := d.requestWithCaptchaToken(url, http.MethodGet, func(req *resty.Request) { - req.SetContext(ctx) - }, &resp) +func (d *PikPak) GetUserID() error { + + token, err := d.oauth2Token.Token() if err != nil { - return fmt.Errorf("failed to get user id : %w", err) + return err } - if resp.Data.UserID != "" { - d.Common.SetUserID(resp.Data.UserID) + + userID := token.Extra("sub").(string) + + if userID != "" { + d.Common.SetUserID(userID) } return nil } diff --git a/drivers/pikpak/types.go b/drivers/pikpak/types.go index a831642e27f..b27b905568a 100644 --- a/drivers/pikpak/types.go +++ b/drivers/pikpak/types.go @@ -10,11 +10,6 @@ import ( hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" ) -type RespErr struct { - ErrorCode int `json:"error_code"` - Error string `json:"error"` -} - type Files struct { Files []File `json:"files"` NextPageToken string `json:"next_page_token"` @@ -174,7 +169,6 @@ type ErrResp struct { ErrorCode int64 `json:"error_code"` ErrorMsg string `json:"error"` ErrorDescription string `json:"error_description"` - // ErrorDetails interface{} `json:"error_details"` } func (e *ErrResp) IsError() bool { @@ -199,37 +193,3 @@ type CaptchaTokenResponse struct { ExpiresIn int64 `json:"expires_in"` Url string `json:"url"` } - -type VipInfo struct { - Data struct { - Expire time.Time `json:"expire"` - ExtUserInfo struct { - UserRegion string `json:"userRegion"` - } `json:"extUserInfo"` - ExtType string `json:"ext_type"` - FeeRecord string `json:"fee_record"` - Restricted struct { - Result bool `json:"result"` - Content struct { - Text string `json:"text"` - Color string `json:"color"` - DeepLink string `json:"deepLink"` - } `json:"content"` - LearnMore struct { - Text string `json:"text"` - Color string `json:"color"` - DeepLink string `json:"deepLink"` - } `json:"learnMore"` - } `json:"restricted"` - Status string `json:"status"` - Type string `json:"type"` - UserID string `json:"user_id"` - VipItem []struct { - Type string `json:"type"` - Description string `json:"description"` - Status string `json:"status"` - Expire time.Time `json:"expire"` - SurplusDay int `json:"surplus_day"` - } `json:"vipItem"` - } `json:"data"` -} diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index a794001a3b3..d4cf64f05a1 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -4,7 +4,6 @@ import ( "crypto/md5" "crypto/sha1" "encoding/hex" - "errors" "fmt" "github.com/alist-org/alist/v3/pkg/utils" "net/http" @@ -53,15 +52,15 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r if resp != nil { req.SetResult(resp) } - var e RespErr + var e ErrResp req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } - if e.ErrorCode != 0 { - return nil, errors.New(e.Error) + if e.IsError() { + return nil, &e } return res.Body(), nil } @@ -101,7 +100,7 @@ func (d *PikPak) requestWithCaptchaToken(url string, method string, callback bas default: return nil, err } - return d.request(url, method, callback, resp) + return d.requestWithCaptchaToken(url, method, callback, resp) } func (d *PikPak) getFiles(id string) ([]File, error) { @@ -264,7 +263,7 @@ func (c *Common) GetCaptchaSign() (timestamp, sign string) { return } -// 刷新验证码token +// refreshCaptchaToken 刷新CaptchaToken func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) error { param := CaptchaTokenRequest{ Action: action, From cee00005ab20d10e2dbce6e9c3eca271acb55647 Mon Sep 17 00:00:00 2001 From: Sakura-Byte <42319937+Sakura-Byte@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:21:06 +0800 Subject: [PATCH 242/659] feat: add support for Onedrive Sharelink driver (#6793) * feat: add support for Onedrive Sharelink driver * fix(Onedrive Sharelink): use internal UA --- drivers/all.go | 1 + drivers/onedrive_sharelink/driver.go | 131 ++++++++++ drivers/onedrive_sharelink/meta.go | 32 +++ drivers/onedrive_sharelink/types.go | 77 ++++++ drivers/onedrive_sharelink/util.go | 363 +++++++++++++++++++++++++++ 5 files changed, 604 insertions(+) create mode 100644 drivers/onedrive_sharelink/driver.go create mode 100644 drivers/onedrive_sharelink/meta.go create mode 100644 drivers/onedrive_sharelink/types.go create mode 100644 drivers/onedrive_sharelink/util.go diff --git a/drivers/all.go b/drivers/all.go index 785278cf908..2cb01d748b0 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -36,6 +36,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/netease_music" _ "github.com/alist-org/alist/v3/drivers/onedrive" _ "github.com/alist-org/alist/v3/drivers/onedrive_app" + _ "github.com/alist-org/alist/v3/drivers/onedrive_sharelink" _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" _ "github.com/alist-org/alist/v3/drivers/quark_uc" diff --git a/drivers/onedrive_sharelink/driver.go b/drivers/onedrive_sharelink/driver.go new file mode 100644 index 00000000000..1282409e4b7 --- /dev/null +++ b/drivers/onedrive_sharelink/driver.go @@ -0,0 +1,131 @@ +package onedrive_sharelink + +import ( + "context" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" + log "github.com/sirupsen/logrus" +) + +type OnedriveSharelink struct { + model.Storage + cron *cron.Cron + Addition +} + +func (d *OnedriveSharelink) Config() driver.Config { + return config +} + +func (d *OnedriveSharelink) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *OnedriveSharelink) Init(ctx context.Context) error { + // Initialize error variable + var err error + + // If there is "-my" in the URL, it is NOT a SharePoint link + d.IsSharepoint = !strings.Contains(d.ShareLinkURL, "-my") + + // Initialize cron job to run every hour + d.cron = cron.NewCron(time.Hour * 1) + d.cron.Do(func() { + var err error + d.Headers, err = d.getHeaders() + if err != nil { + log.Errorf("%+v", err) + } + }) + + // Get initial headers + d.Headers, err = d.getHeaders() + if err != nil { + return err + } + + return nil +} + +func (d *OnedriveSharelink) Drop(ctx context.Context) error { + return nil +} + +func (d *OnedriveSharelink) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + path := dir.GetPath() + files, err := d.getFiles(path) + if err != nil { + return nil, err + } + + // Convert the slice of files to the required model.Obj format + return utils.SliceConvert(files, func(src Item) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *OnedriveSharelink) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + // Get the unique ID of the file + uniqueId := file.GetID() + // Cut the first char and the last char + uniqueId = uniqueId[1 : len(uniqueId)-1] + url := d.downloadLinkPrefix + uniqueId + header := d.Headers + + // If the headers are older than 30 minutes, get new headers + if d.HeaderTime < time.Now().Unix()-1800 { + var err error + log.Debug("headers are older than 30 minutes, get new headers") + header, err = d.getHeaders() + if err != nil { + return nil, err + } + } + + return &model.Link{ + URL: url, + Header: header, + }, nil +} + +func (d *OnedriveSharelink) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + // TODO create folder, optional + return errs.NotImplement +} + +func (d *OnedriveSharelink) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + // TODO move obj, optional + return errs.NotImplement +} + +func (d *OnedriveSharelink) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + // TODO rename obj, optional + return errs.NotImplement +} + +func (d *OnedriveSharelink) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + // TODO copy obj, optional + return errs.NotImplement +} + +func (d *OnedriveSharelink) Remove(ctx context.Context, obj model.Obj) error { + // TODO remove obj, optional + return errs.NotImplement +} + +func (d *OnedriveSharelink) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + // TODO upload file, optional + return errs.NotImplement +} + +//func (d *OnedriveSharelink) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*OnedriveSharelink)(nil) diff --git a/drivers/onedrive_sharelink/meta.go b/drivers/onedrive_sharelink/meta.go new file mode 100644 index 00000000000..6f1ccfc4591 --- /dev/null +++ b/drivers/onedrive_sharelink/meta.go @@ -0,0 +1,32 @@ +package onedrive_sharelink + +import ( + "net/http" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + ShareLinkURL string `json:"url" required:"true"` + ShareLinkPassword string `json:"password"` + IsSharepoint bool + downloadLinkPrefix string + Headers http.Header + HeaderTime int64 +} + +var config = driver.Config{ + Name: "Onedrive Sharelink", + OnlyProxy: true, + NoUpload: true, + DefaultRoot: "/", + CheckStatus: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &OnedriveSharelink{} + }) +} diff --git a/drivers/onedrive_sharelink/types.go b/drivers/onedrive_sharelink/types.go new file mode 100644 index 00000000000..24334250263 --- /dev/null +++ b/drivers/onedrive_sharelink/types.go @@ -0,0 +1,77 @@ +package onedrive_sharelink + +import ( + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +// FolderResp represents the structure of the folder response from the OneDrive API. +type FolderResp struct { + // Data holds the nested structure of the response. + Data struct { + Legacy struct { + RenderListData struct { + ListData struct { + Items []Item `json:"Row"` // Items contains the list of items in the folder. + } `json:"ListData"` + } `json:"renderListDataAsStream"` + } `json:"legacy"` + } `json:"data"` +} + +// Item represents an individual item in the folder. +type Item struct { + ObjType string `json:"FSObjType"` // ObjType indicates if the item is a file or folder. + Name string `json:"FileLeafRef"` // Name is the name of the item. + ModifiedTime time.Time `json:"Modified."` // ModifiedTime is the last modified time of the item. + Size string `json:"File_x0020_Size"` // Size is the size of the item in string format. + Id string `json:"UniqueId"` // Id is the unique identifier of the item. +} + +// fileToObj converts an Item to an ObjThumb. +func fileToObj(f Item) *model.ObjThumb { + // Convert Size from string to int64. + size, _ := strconv.ParseInt(f.Size, 10, 64) + // Convert ObjType from string to int. + objtype, _ := strconv.Atoi(f.ObjType) + + // Create a new ObjThumb with the converted values. + file := &model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.ModifiedTime, + Size: size, + IsFolder: objtype == 1, // Check if the item is a folder. + ID: f.Id, + }, + Thumbnail: model.Thumbnail{}, + } + return file +} + +// GraphQLNEWRequest represents the structure of a new GraphQL request. +type GraphQLNEWRequest struct { + ListData struct { + NextHref string `json:"NextHref"` // NextHref is the link to the next set of data. + Row []Item `json:"Row"` // Row contains the list of items. + } `json:"ListData"` +} + +// GraphQLRequest represents the structure of a GraphQL request. +type GraphQLRequest struct { + Data struct { + Legacy struct { + RenderListDataAsStream struct { + ListData struct { + NextHref string `json:"NextHref"` // NextHref is the link to the next set of data. + Row []Item `json:"Row"` // Row contains the list of items. + } `json:"ListData"` + ViewMetadata struct { + ListViewXml string `json:"ListViewXml"` // ListViewXml contains the XML of the list view. + } `json:"ViewMetadata"` + } `json:"renderListDataAsStream"` + } `json:"legacy"` + } `json:"data"` +} diff --git a/drivers/onedrive_sharelink/util.go b/drivers/onedrive_sharelink/util.go new file mode 100644 index 00000000000..4a1c92b5af8 --- /dev/null +++ b/drivers/onedrive_sharelink/util.go @@ -0,0 +1,363 @@ +package onedrive_sharelink + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + log "github.com/sirupsen/logrus" + "golang.org/x/net/html" +) + +// NewNoRedirectClient creates an HTTP client that doesn't follow redirects +func NewNoRedirectCLient() *http.Client { + return &http.Client{ + Timeout: time.Hour * 48, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + }, + // Prevent following redirects + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } +} + +// getCookiesWithPassword fetches cookies required for authenticated access using the provided password +func getCookiesWithPassword(link, password string) (string, error) { + // Send GET request + resp, err := http.Get(link) + if err != nil { + return "", err + } + defer resp.Body.Close() + + // Parse the HTML response + doc, err := html.Parse(resp.Body) + if err != nil { + return "", err + } + + // Initialize variables to store form data + var viewstate, eventvalidation, postAction string + + // Recursive function to find input fields by their IDs + var findInputFields func(*html.Node) + findInputFields = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "input" { + for _, attr := range n.Attr { + if attr.Key == "id" { + switch attr.Val { + case "__VIEWSTATE": + viewstate = getAttrValue(n, "value") + case "__EVENTVALIDATION": + eventvalidation = getAttrValue(n, "value") + } + } + } + } + if n.Type == html.ElementNode && n.Data == "form" { + for _, attr := range n.Attr { + if attr.Key == "id" && attr.Val == "inputForm" { + postAction = getAttrValue(n, "action") + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + findInputFields(c) + } + } + findInputFields(doc) + + // Prepare the new URL for the POST request + linkParts, err := url.Parse(link) + if err != nil { + return "", err + } + + newURL := fmt.Sprintf("%s://%s%s", linkParts.Scheme, linkParts.Host, postAction) + + // Prepare the request body + data := url.Values{ + "txtPassword": []string{password}, + "__EVENTVALIDATION": []string{eventvalidation}, + "__VIEWSTATE": []string{viewstate}, + "__VIEWSTATEENCRYPTED": []string{""}, + } + + client := &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + // Send the POST request, preventing redirects + resp, err = client.PostForm(newURL, data) + if err != nil { + return "", err + } + + // Extract the desired cookie value + cookie := resp.Cookies() + var fedAuthCookie string + for _, c := range cookie { + if c.Name == "FedAuth" { + fedAuthCookie = c.Value + break + } + } + if fedAuthCookie == "" { + return "", fmt.Errorf("wrong password") + } + return fmt.Sprintf("FedAuth=%s;", fedAuthCookie), nil +} + +// getAttrValue retrieves the value of the specified attribute from an HTML node +func getAttrValue(n *html.Node, key string) string { + for _, attr := range n.Attr { + if attr.Key == key { + return attr.Val + } + } + return "" +} + +// getHeaders constructs and returns the necessary HTTP headers for accessing the OneDrive share link +func (d *OnedriveSharelink) getHeaders() (http.Header, error) { + header := http.Header{} + header.Set("User-Agent", base.UserAgent) + header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") + + // Save current timestamp to d.HeaderTime + d.HeaderTime = time.Now().Unix() + + if d.ShareLinkPassword == "" { + // Create a no-redirect client + clientNoDirect := NewNoRedirectCLient() + req, err := http.NewRequest("GET", d.ShareLinkURL, nil) + if err != nil { + return nil, err + } + // Set headers for the request + req.Header = header + answerNoRedirect, err := clientNoDirect.Do(req) + if err != nil { + return nil, err + } + redirectUrl := answerNoRedirect.Header.Get("Location") + log.Debugln("redirectUrl:", redirectUrl) + if redirectUrl == "" { + return nil, fmt.Errorf("password protected link. Please provide password") + } + header.Set("Cookie", answerNoRedirect.Header.Get("Set-Cookie")) + header.Set("Referer", redirectUrl) + + // Extract the host part of the redirect URL and set it as the authority + u, err := url.Parse(redirectUrl) + if err != nil { + return nil, err + } + header.Set("authority", u.Host) + return header, nil + } else { + cookie, err := getCookiesWithPassword(d.ShareLinkURL, d.ShareLinkPassword) + if err != nil { + return nil, err + } + header.Set("Cookie", cookie) + header.Set("Referer", d.ShareLinkURL) + header.Set("authority", strings.Split(strings.Split(d.ShareLinkURL, "//")[1], "/")[0]) + return header, nil + } +} + +// getFiles retrieves the files from the OneDrive share link at the specified path +func (d *OnedriveSharelink) getFiles(path string) ([]Item, error) { + clientNoDirect := NewNoRedirectCLient() + req, err := http.NewRequest("GET", d.ShareLinkURL, nil) + if err != nil { + return nil, err + } + header := req.Header + redirectUrl := "" + if d.ShareLinkPassword == "" { + header.Set("User-Agent", base.UserAgent) + header.Set("accept-language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6") + req.Header = header + answerNoRedirect, err := clientNoDirect.Do(req) + if err != nil { + return nil, err + } + redirectUrl = answerNoRedirect.Header.Get("Location") + } else { + header = d.Headers + req.Header = header + answerNoRedirect, err := clientNoDirect.Do(req) + if err != nil { + return nil, err + } + redirectUrl = answerNoRedirect.Header.Get("Location") + } + redirectSplitURL := strings.Split(redirectUrl, "/") + req.Header = d.Headers + downloadLinkPrefix := "" + rootFolderPre := "" + + // Determine the appropriate URL and root folder based on whether the link is SharePoint + if d.IsSharepoint { + // update req url + req.URL, err = url.Parse(redirectUrl) + if err != nil { + return nil, err + } + // Get redirectUrl + answer, err := clientNoDirect.Do(req) + if err != nil { + d.Headers, err = d.getHeaders() + if err != nil { + return nil, err + } + return d.getFiles(path) + } + defer answer.Body.Close() + re := regexp.MustCompile(`templateUrl":"(.*?)"`) + body, err := io.ReadAll(answer.Body) + if err != nil { + return nil, err + } + template := re.FindString(string(body)) + template = template[strings.Index(template, "templateUrl\":\"")+len("templateUrl\":\""):] + template = template[:strings.Index(template, "?id=")] + template = template[:strings.LastIndex(template, "/")] + downloadLinkPrefix = template + "/download.aspx?UniqueId=" + params, err := url.ParseQuery(redirectUrl[strings.Index(redirectUrl, "?")+1:]) + if err != nil { + return nil, err + } + rootFolderPre = params.Get("id") + } else { + redirectUrlCut := redirectUrl[:strings.LastIndex(redirectUrl, "/")] + downloadLinkPrefix = redirectUrlCut + "/download.aspx?UniqueId=" + params, err := url.ParseQuery(redirectUrl[strings.Index(redirectUrl, "?")+1:]) + if err != nil { + return nil, err + } + rootFolderPre = params.Get("id") + } + d.downloadLinkPrefix = downloadLinkPrefix + rootFolder, err := url.QueryUnescape(rootFolderPre) + if err != nil { + return nil, err + } + log.Debugln("rootFolder:", rootFolder) + // Extract the relative path up to and including "Documents" + relativePath := strings.Split(rootFolder, "Documents")[0] + "Documents" + + // URL encode the relative path + relativeUrl := url.QueryEscape(relativePath) + // Replace underscores and hyphens in the encoded relative path + relativeUrl = strings.Replace(relativeUrl, "_", "%5F", -1) + relativeUrl = strings.Replace(relativeUrl, "-", "%2D", -1) + + // If the path is not the root, append the path to the root folder + if path != "/" { + rootFolder = rootFolder + path + } + + // URL encode the full root folder path + rootFolderUrl := url.QueryEscape(rootFolder) + // Replace underscores and hyphens in the encoded root folder URL + rootFolderUrl = strings.Replace(rootFolderUrl, "_", "%5F", -1) + rootFolderUrl = strings.Replace(rootFolderUrl, "-", "%2D", -1) + + log.Debugln("relativePath:", relativePath, "relativeUrl:", relativeUrl, "rootFolder:", rootFolder, "rootFolderUrl:", rootFolderUrl) + + // Construct the GraphQL query with the encoded paths + graphqlVar := fmt.Sprintf(`{"query":"query (\n $listServerRelativeUrl: String!,$renderListDataAsStreamParameters: RenderListDataAsStreamParameters!,$renderListDataAsStreamQueryString: String!\n )\n {\n \n legacy {\n \n renderListDataAsStream(\n listServerRelativeUrl: $listServerRelativeUrl,\n parameters: $renderListDataAsStreamParameters,\n queryString: $renderListDataAsStreamQueryString\n )\n }\n \n \n perf {\n executionTime\n overheadTime\n parsingTime\n queryCount\n validationTime\n resolvers {\n name\n queryCount\n resolveTime\n waitTime\n }\n }\n }","variables":{"listServerRelativeUrl":"%s","renderListDataAsStreamParameters":{"renderOptions":5707527,"allowMultipleValueFilterForTaxonomyFields":true,"addRequiredFields":true,"folderServerRelativeUrl":"%s"},"renderListDataAsStreamQueryString":"@a1=\'%s\'&RootFolder=%s&TryNewExperienceSingle=TRUE"}}`, relativePath, rootFolder, relativeUrl, rootFolderUrl) + tempHeader := make(http.Header) + for k, v := range d.Headers { + tempHeader[k] = v + } + tempHeader["Content-Type"] = []string{"application/json;odata=verbose"} + + client := &http.Client{} + postUrl := strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/v2.1/graphql" + req, err = http.NewRequest("POST", postUrl, strings.NewReader(graphqlVar)) + if err != nil { + return nil, err + } + req.Header = tempHeader + + resp, err := client.Do(req) + if err != nil { + d.Headers, err = d.getHeaders() + if err != nil { + return nil, err + } + return d.getFiles(path) + } + defer resp.Body.Close() + var graphqlReq GraphQLRequest + json.NewDecoder(resp.Body).Decode(&graphqlReq) + log.Debugln("graphqlReq:", graphqlReq) + filesData := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row + if graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref != "" { + nextHref := graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" + nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) + log.Debugln("nextHref:", nextHref) + filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) + + listViewXml := graphqlReq.Data.Legacy.RenderListDataAsStream.ViewMetadata.ListViewXml + log.Debugln("listViewXml:", listViewXml) + renderListDataAsStreamVar := `{"parameters":{"__metadata":{"type":"SP.RenderListDataParameters"},"RenderOptions":1216519,"ViewXml":"REPLACEME","AllowMultipleValueFilterForTaxonomyFields":true,"AddRequiredFields":true}}` + listViewXml = strings.Replace(listViewXml, `"`, `\"`, -1) + renderListDataAsStreamVar = strings.Replace(renderListDataAsStreamVar, "REPLACEME", listViewXml, -1) + + graphqlReqNEW := GraphQLNEWRequest{} + postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref + req, _ = http.NewRequest("POST", postUrl, strings.NewReader(renderListDataAsStreamVar)) + req.Header = tempHeader + + resp, err := client.Do(req) + if err != nil { + d.Headers, err = d.getHeaders() + if err != nil { + return nil, err + } + return d.getFiles(path) + } + defer resp.Body.Close() + json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) + for graphqlReqNEW.ListData.NextHref != "" { + graphqlReqNEW = GraphQLNEWRequest{} + postUrl = strings.Join(redirectSplitURL[:len(redirectSplitURL)-3], "/") + "/_api/web/GetListUsingPath(DecodedUrl=@a1)/RenderListDataAsStream" + nextHref + req, _ = http.NewRequest("POST", postUrl, strings.NewReader(renderListDataAsStreamVar)) + req.Header = tempHeader + resp, err := client.Do(req) + if err != nil { + d.Headers, err = d.getHeaders() + if err != nil { + return nil, err + } + return d.getFiles(path) + } + defer resp.Body.Close() + json.NewDecoder(resp.Body).Decode(&graphqlReqNEW) + nextHref = graphqlReqNEW.ListData.NextHref + "&@a1=REPLACEME&TryNewExperienceSingle=TRUE" + nextHref = strings.Replace(nextHref, "REPLACEME", "%27"+relativeUrl+"%27", -1) + filesData = append(filesData, graphqlReqNEW.ListData.Row...) + } + filesData = append(filesData, graphqlReqNEW.ListData.Row...) + } else { + filesData = append(filesData, graphqlReq.Data.Legacy.RenderListDataAsStream.ListData.Row...) + } + return filesData, nil +} From f2a24881d020e8be77ce7699316298f28da4ec83 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:21:54 +0800 Subject: [PATCH 243/659] fix(deps): update module gorm.io/driver/postgres to v1.5.9 (#6783) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 5 +++-- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index c852b13715f..153fb133025 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 - gorm.io/driver/postgres v1.4.8 + gorm.io/driver/postgres v1.5.9 gorm.io/driver/sqlite v1.5.6 gorm.io/gorm v1.25.11 ) @@ -85,6 +85,7 @@ require ( github.com/cloudwego/iasm v0.2.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/ipfs/boxo v0.12.0 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect ) @@ -148,7 +149,7 @@ require ( github.com/ipfs/go-cid v0.4.1 github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.3.0 // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect github.com/jaevor/go-nanoid v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect diff --git a/go.sum b/go.sum index 3900a728bda..8bf554e8c93 100644 --- a/go.sum +++ b/go.sum @@ -265,7 +265,11 @@ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaevor/go-nanoid v1.3.0 h1:nD+iepesZS6pr3uOVf20vR9GdGgJW1HPaR46gtrxzkg= github.com/jaevor/go-nanoid v1.3.0/go.mod h1:SI+jFaPuddYkqkVQoNGHs81navCtH388TcrH0RqFKgY= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= @@ -706,6 +710,8 @@ gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= gorm.io/driver/postgres v1.4.8 h1:NDWizaclb7Q2aupT0jkwK8jx1HVCNzt+PQ8v/VnxviA= gorm.io/driver/postgres v1.4.8/go.mod h1:O9MruWGNLUBUWVYfWuBClpf3HeGjOoybY0SNmCs3wsw= +gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= +gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= From c9a18f4de6603a9023d6fa8791c6d1582f027a2e Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:22:17 +0800 Subject: [PATCH 244/659] chore(deps): update benjlevesque/short-sha action to v3 (#6784) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b75010ef8e7..b059a20b0d9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - uses: benjlevesque/short-sha@v2.2 + - uses: benjlevesque/short-sha@v3.0 id: short-sha - name: Install dependencies From 5ef7a27be3fa997910483169ebabc6e08db267ca Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:22:35 +0800 Subject: [PATCH 245/659] chore(deps): update docker/build-push-action action to v6 (#6785) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/build_docker.yml | 4 ++-- .github/workflows/release_docker.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 731b0159011..c6c301f3ba8 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -65,7 +65,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci @@ -80,7 +80,7 @@ jobs: - name: Build and push with ffmpeg id: docker_build_ffmpeg - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ffmpeg diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 8bff6a3d9b3..3078d0c3094 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -51,7 +51,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci @@ -71,7 +71,7 @@ jobs: - name: Build and push with ffmpeg id: docker_build_ffmpeg - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ffmpeg From fe081d0ebc5a0881f3e00f9843b0323bbd606582 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:22:54 +0800 Subject: [PATCH 246/659] chore(deps): update softprops/action-gh-release action to v2 (#6786) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- .github/workflows/release_android.yml | 2 +- .github/workflows/release_linux_musl.yml | 2 +- .github/workflows/release_linux_musl_arm.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d43d39d0c3b..6ef38566143 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -42,7 +42,7 @@ jobs: bash build.sh release - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: build/compress/* prerelease: false diff --git a/.github/workflows/release_android.yml b/.github/workflows/release_android.yml index c696ddb743a..7e071cbe2ff 100644 --- a/.github/workflows/release_android.yml +++ b/.github/workflows/release_android.yml @@ -29,6 +29,6 @@ jobs: bash build.sh release android - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: build/compress/* diff --git a/.github/workflows/release_linux_musl.yml b/.github/workflows/release_linux_musl.yml index 9ec79af6fdf..bb5291a9ac2 100644 --- a/.github/workflows/release_linux_musl.yml +++ b/.github/workflows/release_linux_musl.yml @@ -29,6 +29,6 @@ jobs: bash build.sh release linux_musl - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: build/compress/* diff --git a/.github/workflows/release_linux_musl_arm.yml b/.github/workflows/release_linux_musl_arm.yml index 8ddbc4f42cc..0e8a9618a76 100644 --- a/.github/workflows/release_linux_musl_arm.yml +++ b/.github/workflows/release_linux_musl_arm.yml @@ -29,6 +29,6 @@ jobs: bash build.sh release linux_musl_arm - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: build/compress/* From 2b74999703a7143346fc9f853c2c33e591be0a61 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:31:09 +0800 Subject: [PATCH 247/659] fix(deps): update module github.com/alist-org/gofakes3 to v0.0.6 (#6802) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: David Hao Co-authored-by: Ke Wang --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 153fb133025..ec8d94276d0 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/SheltonZhu/115driver v1.0.25 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 - github.com/alist-org/gofakes3 v0.0.5 + github.com/alist-org/gofakes3 v0.0.6 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.54.19 diff --git a/go.sum b/go.sum index 8bf554e8c93..57382406917 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= github.com/alist-org/gofakes3 v0.0.5 h1:bLHhLTNg3kIRdx7gsgi9Zg/EW9s3IHwJVRwzUCPR8V0= github.com/alist-org/gofakes3 v0.0.5/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= +github.com/alist-org/gofakes3 v0.0.6 h1:kenkDSqOIJt5ZDJ9KW91YkwplFXpfToPDjP3Bd6GZRg= +github.com/alist-org/gofakes3 v0.0.6/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= From 2d57529e7728aa2512c8663c226217e7b97cea83 Mon Sep 17 00:00:00 2001 From: seiuneko <25706824+seiuneko@users.noreply.github.com> Date: Sat, 20 Jul 2024 12:27:18 +0800 Subject: [PATCH 248/659] fix(123pan): add warning for mismatched file count when listing files (#6814) Fixes an issue where using `file_name` order could result in incorrect file counts compared to response fields. --- drivers/123/driver.go | 2 +- drivers/123/types.go | 3 ++- drivers/123/util.go | 8 +++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 240027405d5..bd7089e85d2 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -53,7 +53,7 @@ func (d *Pan123) Drop(ctx context.Context) error { } func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files, err := d.getFiles(dir.GetID()) + files, err := d.getFiles(dir.GetID(), dir.GetName()) if err != nil { return nil, err } diff --git a/drivers/123/types.go b/drivers/123/types.go index b79be12e201..a8682c52fc9 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -87,8 +87,9 @@ var _ model.Thumb = (*File)(nil) type Files struct { //BaseResp Data struct { - InfoList []File `json:"InfoList"` Next string `json:"Next"` + Total int `json:"Total"` + InfoList []File `json:"InfoList"` } `json:"data"` } diff --git a/drivers/123/util.go b/drivers/123/util.go index 9d5d6780167..86816df21d7 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -3,6 +3,7 @@ package _123 import ( "errors" "fmt" + log "github.com/sirupsen/logrus" "hash/crc32" "math" "math/rand" @@ -232,8 +233,9 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r return body, nil } -func (d *Pan123) getFiles(parentId string) ([]File, error) { +func (d *Pan123) getFiles(parentId string, name string) ([]File, error) { page := 1 + total := 0 res := make([]File, 0) // 2024-02-06 fix concurrency by 123pan for { @@ -265,9 +267,13 @@ func (d *Pan123) getFiles(parentId string) ([]File, error) { } page++ res = append(res, resp.Data.InfoList...) + total = resp.Data.Total if len(resp.Data.InfoList) == 0 || resp.Data.Next == "-1" { break } } + if len(res) != total { + log.Warnf("incorrect file count from remote at %s: expected %d, got %d", name, total, len(res)) + } return res, nil } From cbd4bef81440e5124243734fd86a88713a190936 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 21 Jul 2024 20:29:32 +0800 Subject: [PATCH 249/659] fix(123pan): use local sort (close #6820) --- drivers/123/meta.go | 7 ++++--- drivers/123/util.go | 8 +++++--- drivers/123_share/meta.go | 6 +++--- drivers/123_share/util.go | 4 ++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/drivers/123/meta.go b/drivers/123/meta.go index 0c3c6a2dffb..cb2cbc15ba0 100644 --- a/drivers/123/meta.go +++ b/drivers/123/meta.go @@ -9,14 +9,15 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - AccessToken string + //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` + //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + AccessToken string } var config = driver.Config{ Name: "123Pan", DefaultRoot: "0", + LocalSort: true, } func init() { diff --git a/drivers/123/util.go b/drivers/123/util.go index 86816df21d7..2b736a50b4b 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -17,6 +17,7 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" resty "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" + log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface @@ -248,8 +249,8 @@ func (d *Pan123) getFiles(parentId string, name string) ([]File, error) { "driveId": "0", "limit": "100", "next": "0", - "orderBy": d.OrderBy, - "orderDirection": d.OrderDirection, + "orderBy": "file_id", + "orderDirection": "desc", "parentFileId": parentId, "trashed": "false", "SearchData": "", @@ -259,12 +260,13 @@ func (d *Pan123) getFiles(parentId string, name string) ([]File, error) { "operateType": "4", "inDirectSpace": "false", } - _, err := d.request(FileList, http.MethodGet, func(req *resty.Request) { + _res, err := d.request(FileList, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } + log.Debug(string(_res)) page++ res = append(res, resp.Data.InfoList...) total = resp.Data.Total diff --git a/drivers/123_share/meta.go b/drivers/123_share/meta.go index ce39b7eee07..7cbcba27724 100644 --- a/drivers/123_share/meta.go +++ b/drivers/123_share/meta.go @@ -9,9 +9,9 @@ type Addition struct { ShareKey string `json:"sharekey" required:"true"` SharePwd string `json:"sharepassword"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - AccessToken string `json:"accesstoken" type:"text"` + //OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` + //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + AccessToken string `json:"accesstoken" type:"text"` } var config = driver.Config{ diff --git a/drivers/123_share/util.go b/drivers/123_share/util.go index b22b7cc4547..a192993bf19 100644 --- a/drivers/123_share/util.go +++ b/drivers/123_share/util.go @@ -92,8 +92,8 @@ func (d *Pan123Share) getFiles(parentId string) ([]File, error) { query := map[string]string{ "limit": "100", "next": "0", - "orderBy": d.OrderBy, - "orderDirection": d.OrderDirection, + "orderBy": "file_id", + "orderDirection": "desc", "parentFileId": parentId, "Page": strconv.Itoa(page), "shareKey": d.ShareKey, From e5f53d6deea8910d5d8e6a61008982c1c203ed89 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 21 Jul 2024 20:31:52 +0800 Subject: [PATCH 250/659] chore: go mod tidy --- go.sum | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/go.sum b/go.sum index 57382406917..c2c25ed1bad 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,6 @@ github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0E github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= -github.com/alist-org/gofakes3 v0.0.5 h1:bLHhLTNg3kIRdx7gsgi9Zg/EW9s3IHwJVRwzUCPR8V0= -github.com/alist-org/gofakes3 v0.0.5/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= github.com/alist-org/gofakes3 v0.0.6 h1:kenkDSqOIJt5ZDJ9KW91YkwplFXpfToPDjP3Bd6GZRg= github.com/alist-org/gofakes3 v0.0.6/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= @@ -32,8 +30,6 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.53.7 h1:ZSsRYHLRxsbO2rJR2oPMz0SUkJLnBkN+1meT95B6Ixs= -github.com/aws/aws-sdk-go v1.53.7/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI= github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= @@ -168,8 +164,6 @@ github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQk github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= @@ -265,18 +259,14 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= -github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaevor/go-nanoid v1.3.0 h1:nD+iepesZS6pr3uOVf20vR9GdGgJW1HPaR46gtrxzkg= github.com/jaevor/go-nanoid v1.3.0/go.mod h1:SI+jFaPuddYkqkVQoNGHs81navCtH388TcrH0RqFKgY= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= @@ -397,8 +387,6 @@ github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/pelletier/go-toml/v2 v2.2.1 h1:9TA9+T8+8CUCO2+WYnDLCgrYi9+omqKXyjDtosvtEhg= -github.com/pelletier/go-toml/v2 v2.2.1/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= @@ -543,8 +531,6 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.7.0 h1:pskyeJh/3AmoQ8CPE95vxHLqp1G1GfGNXTmcl9NEKTc= -golang.org/x/arch v0.7.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -558,7 +544,6 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= @@ -589,8 +574,6 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= -golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -706,18 +689,12 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= -gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= -gorm.io/driver/postgres v1.4.8 h1:NDWizaclb7Q2aupT0jkwK8jx1HVCNzt+PQ8v/VnxviA= -gorm.io/driver/postgres v1.4.8/go.mod h1:O9MruWGNLUBUWVYfWuBClpf3HeGjOoybY0SNmCs3wsw= gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= -gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= From 94f257e55761921532719aebc23a412175e06081 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 21 Jul 2024 20:48:48 +0800 Subject: [PATCH 251/659] fix(local): crush on android closes #5874 closes #6567 --- drivers/local/driver.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 4efee6d6c6c..abe5c6b5744 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -21,7 +21,7 @@ import ( "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" - "github.com/djherbis/times" + "github.com/alist-org/times" log "github.com/sirupsen/logrus" _ "golang.org/x/image/webp" ) diff --git a/go.mod b/go.mod index ec8d94276d0..8d79af157ab 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.6 + github.com/alist-org/times v0.0.0-20240721124318-c2e3da27cc69 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.54.19 @@ -20,7 +21,6 @@ require ( github.com/deckarep/golang-set/v2 v2.6.0 github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 github.com/disintegration/imaging v1.6.2 - github.com/djherbis/times v1.6.0 github.com/dlclark/regexp2 v1.11.2 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.6 diff --git a/go.sum b/go.sum index c2c25ed1bad..a62a94907e9 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= github.com/alist-org/gofakes3 v0.0.6 h1:kenkDSqOIJt5ZDJ9KW91YkwplFXpfToPDjP3Bd6GZRg= github.com/alist-org/gofakes3 v0.0.6/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= +github.com/alist-org/times v0.0.0-20240721124318-c2e3da27cc69 h1:E9QJ4vVTu1KYRhelnCsQImCsbl7NlkH3Yxs3/L2ldDk= +github.com/alist-org/times v0.0.0-20240721124318-c2e3da27cc69/go.mod h1:oPJwGY3sLmGgcJamGumz//0A35f4BwQRacyqLNcJTOU= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= @@ -136,8 +138,6 @@ github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 h1:OtSeLS5y0Uy01jaKK4m github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8/go.mod h1:apkPC/CR3s48O2D7Y++n1XWEpgPNNCjXYga3PPbJe2E= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= -github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= -github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/dlclark/regexp2 v1.11.2 h1:/u628IuisSTwri5/UKloiIsH8+qF2Pu7xEQX+yIKg68= github.com/dlclark/regexp2 v1.11.2/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= From d4e3355f5692a690a8aa02298b0052e4340a3e55 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 21 Jul 2024 20:50:07 +0800 Subject: [PATCH 252/659] chore: duplicate import typo --- drivers/123/util.go | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/123/util.go b/drivers/123/util.go index 2b736a50b4b..540201f7035 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -3,7 +3,6 @@ package _123 import ( "errors" "fmt" - log "github.com/sirupsen/logrus" "hash/crc32" "math" "math/rand" From 5fa70e40107180816b0c9b1c1a2325e0b5318bdb Mon Sep 17 00:00:00 2001 From: seiuneko <25706824+seiuneko@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:08:59 +0800 Subject: [PATCH 253/659] perf(123pan): optimize rate limiting (#6859) - eliminating fixed 200 ms delay in getFiles to prevent thread starvation - allowing cancellation via context to mitigate potential DoS attacks by immediately cancelling excessive requests --- drivers/123/driver.go | 16 +++++++--------- drivers/123/util.go | 10 +++++----- drivers/123_share/driver.go | 13 +++++++------ drivers/123_share/util.go | 8 ++++---- 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index bd7089e85d2..aeda7fcf742 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -53,7 +53,7 @@ func (d *Pan123) Drop(ctx context.Context) error { } func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files, err := d.getFiles(dir.GetID(), dir.GetName()) + files, err := d.getFiles(ctx, dir.GetID(), dir.GetName()) if err != nil { return nil, err } @@ -247,9 +247,6 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } _, err = uploader.UploadWithContext(ctx, input) } - if err != nil { - return err - } _, err = d.request(UploadComplete, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileId": resp.Data.FileId, @@ -258,11 +255,12 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } -func (d *Pan123) APIRateLimit(api string) bool { - limiter, _ := d.apiRateLimit.LoadOrStore(api, - rate.NewLimiter(rate.Every(time.Millisecond*700), 1)) - ins := limiter.(*rate.Limiter) - return ins.Allow() +func (d *Pan123) APIRateLimit(ctx context.Context, api string) error { + value, _ := d.apiRateLimit.LoadOrStore(api, + rate.NewLimiter(rate.Every(700*time.Millisecond), 1)) + limiter := value.(*rate.Limiter) + + return limiter.Wait(ctx) } var _ driver.Driver = (*Pan123)(nil) diff --git a/drivers/123/util.go b/drivers/123/util.go index 540201f7035..73c73b3b3b3 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -1,6 +1,7 @@ package _123 import ( + "context" "errors" "fmt" "hash/crc32" @@ -14,7 +15,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" - resty "github.com/go-resty/resty/v2" + "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" ) @@ -233,15 +234,14 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r return body, nil } -func (d *Pan123) getFiles(parentId string, name string) ([]File, error) { +func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { page := 1 total := 0 res := make([]File, 0) // 2024-02-06 fix concurrency by 123pan for { - if !d.APIRateLimit(FileList) { - time.Sleep(time.Millisecond * 200) - continue + if err := d.APIRateLimit(ctx, FileList); err != nil { + return nil, err } var resp Files query := map[string]string{ diff --git a/drivers/123_share/driver.go b/drivers/123_share/driver.go index 7fca7cc145e..9c1f3803710 100644 --- a/drivers/123_share/driver.go +++ b/drivers/123_share/driver.go @@ -45,7 +45,7 @@ func (d *Pan123Share) Drop(ctx context.Context) error { func (d *Pan123Share) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { // TODO return the files list, required - files, err := d.getFiles(dir.GetID()) + files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err } @@ -150,11 +150,12 @@ func (d *Pan123Share) Put(ctx context.Context, dstDir model.Obj, stream model.Fi // return nil, errs.NotSupport //} -func (d *Pan123Share) APIRateLimit(api string) bool { - limiter, _ := d.apiRateLimit.LoadOrStore(api, - rate.NewLimiter(rate.Every(time.Millisecond*700), 1)) - ins := limiter.(*rate.Limiter) - return ins.Allow() +func (d *Pan123Share) APIRateLimit(ctx context.Context, api string) error { + value, _ := d.apiRateLimit.LoadOrStore(api, + rate.NewLimiter(rate.Every(700*time.Millisecond), 1)) + limiter := value.(*rate.Limiter) + + return limiter.Wait(ctx) } var _ driver.Driver = (*Pan123Share)(nil) diff --git a/drivers/123_share/util.go b/drivers/123_share/util.go index a192993bf19..80ea8f0ca46 100644 --- a/drivers/123_share/util.go +++ b/drivers/123_share/util.go @@ -1,6 +1,7 @@ package _123Share import ( + "context" "errors" "fmt" "hash/crc32" @@ -80,13 +81,12 @@ func (d *Pan123Share) request(url string, method string, callback base.ReqCallba return body, nil } -func (d *Pan123Share) getFiles(parentId string) ([]File, error) { +func (d *Pan123Share) getFiles(ctx context.Context, parentId string) ([]File, error) { page := 1 res := make([]File, 0) for { - if !d.APIRateLimit(FileList) { - time.Sleep(time.Millisecond * 200) - continue + if err := d.APIRateLimit(ctx, FileList); err != nil { + return nil, err } var resp Files query := map[string]string{ From 4a42bc5083a171cba24a0fdcc2781a2e8dfbd9cc Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:09:48 +0800 Subject: [PATCH 254/659] fix(lanzou): not find file page param (#6862 close #6857) * fix(lanzou):not find file page param * fix(labzou): change lanzouo.com to lanzoui.com --- drivers/lanzou/driver.go | 3 +++ drivers/lanzou/meta.go | 3 ++- drivers/lanzou/util.go | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index cdb56f79658..9e73f0525c2 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -30,6 +30,9 @@ func (d *LanZou) GetAddition() driver.Additional { } func (d *LanZou) Init(ctx context.Context) (err error) { + if d.UserAgent == "" { + d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/89.0.4389.111 Safari/537.39" + } switch d.Type { case "account": _, err := d.Login() diff --git a/drivers/lanzou/meta.go b/drivers/lanzou/meta.go index c8db6448476..1e8826cadeb 100644 --- a/drivers/lanzou/meta.go +++ b/drivers/lanzou/meta.go @@ -16,7 +16,8 @@ type Addition struct { driver.RootID SharePassword string `json:"share_password"` BaseUrl string `json:"baseUrl" required:"true" default:"https://pc.woozooo.com" help:"basic URL for file operation"` - ShareUrl string `json:"shareUrl" required:"true" default:"https://pan.lanzouo.com" help:"used to get the sharing page"` + ShareUrl string `json:"shareUrl" required:"true" default:"https://pan.lanzoui.com" help:"used to get the sharing page"` + UserAgent string `json:"user_agent" required:"true" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/89.0.4389.111 Safari/537.39"` RepairFileInfo bool `json:"repair_file_info" help:"To use webdav, you need to enable it"` } diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index 8aeba8113e1..abc2c400119 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -106,7 +106,8 @@ func (d *LanZou) request(url string, method string, callback base.ReqCallback, u } req.SetHeaders(map[string]string{ - "Referer": "https://pc.woozooo.com", + "Referer": "https://pc.woozooo.com", + "User-Agent": d.UserAgent, }) if d.Cookie != "" { From 1aff75868880f2049cf42c67bdaf8f9a6580b62a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:11:12 +0800 Subject: [PATCH 255/659] fix(deps): update github.com/alist-org/times digest to efa0c7d (#6840) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 8d79af157ab..04526a34649 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.6 - github.com/alist-org/times v0.0.0-20240721124318-c2e3da27cc69 + github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.54.19 diff --git a/go.sum b/go.sum index a62a94907e9..d2163aab527 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,8 @@ github.com/alist-org/gofakes3 v0.0.6 h1:kenkDSqOIJt5ZDJ9KW91YkwplFXpfToPDjP3Bd6G github.com/alist-org/gofakes3 v0.0.6/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= github.com/alist-org/times v0.0.0-20240721124318-c2e3da27cc69 h1:E9QJ4vVTu1KYRhelnCsQImCsbl7NlkH3Yxs3/L2ldDk= github.com/alist-org/times v0.0.0-20240721124318-c2e3da27cc69/go.mod h1:oPJwGY3sLmGgcJamGumz//0A35f4BwQRacyqLNcJTOU= +github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 h1:pIEI87zhv8ZzQcu65rTL7kqirrs8dR6HDiXrqWat2Fk= +github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92/go.mod h1:oPJwGY3sLmGgcJamGumz//0A35f4BwQRacyqLNcJTOU= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= From aeae47c9bf96d18df80fc29da8927eb6b0808942 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 25 Jul 2024 20:13:01 +0800 Subject: [PATCH 256/659] fix(deps): update module github.com/larksuite/oapi-sdk-go/v3 to v3.3.0 (#6812) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 04526a34649..50b756ec88b 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 - github.com/larksuite/oapi-sdk-go/v3 v3.2.8 + github.com/larksuite/oapi-sdk-go/v3 v3.3.0 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.0 github.com/minio/sio v0.4.0 diff --git a/go.sum b/go.sum index d2163aab527..f683774a0f1 100644 --- a/go.sum +++ b/go.sum @@ -308,6 +308,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.2.8 h1:elbufnS+gQVOkzX9JLkS/N9u3ay/IAIE17nE4kNoYZ4= github.com/larksuite/oapi-sdk-go/v3 v3.2.8/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/larksuite/oapi-sdk-go/v3 v3.3.0 h1:aCtFUiYgoRUW+aaWzVYw8jSzMe4A71rPEIn1DyHcNrY= +github.com/larksuite/oapi-sdk-go/v3 v3.3.0/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= From 8b5727a0aa6a8277cc8776042f6cb0f527ea4c6f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:27:56 +0800 Subject: [PATCH 257/659] fix(deps): update golang.org/x/exp digest to 8a7402a (#6801) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 50b756ec88b..4d9809fb006 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.25.0 - golang.org/x/exp v0.0.0-20240707233637-46b078467d37 + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/image v0.18.0 golang.org/x/net v0.27.0 golang.org/x/oauth2 v0.21.0 diff --git a/go.sum b/go.sum index f683774a0f1..072e5591a56 100644 --- a/go.sum +++ b/go.sum @@ -554,6 +554,8 @@ golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= From af9c6afd255b9a1763f26946687995ec808505a0 Mon Sep 17 00:00:00 2001 From: Hao Jiakang Date: Sat, 27 Jul 2024 20:06:05 +0800 Subject: [PATCH 258/659] feat: update alist-org/gofakes3 to v0.0.7 to support create folder in PutObject (#6880) --- go.mod | 2 +- go.sum | 8 ++------ server/s3/backend.go | 17 ++++++++++++++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 4d9809fb006..8bed00741b5 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/SheltonZhu/115driver v1.0.25 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 - github.com/alist-org/gofakes3 v0.0.6 + github.com/alist-org/gofakes3 v0.0.7 github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible diff --git a/go.sum b/go.sum index 072e5591a56..d82af70ecf2 100644 --- a/go.sum +++ b/go.sum @@ -19,10 +19,8 @@ github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0E github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= -github.com/alist-org/gofakes3 v0.0.6 h1:kenkDSqOIJt5ZDJ9KW91YkwplFXpfToPDjP3Bd6GZRg= -github.com/alist-org/gofakes3 v0.0.6/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= -github.com/alist-org/times v0.0.0-20240721124318-c2e3da27cc69 h1:E9QJ4vVTu1KYRhelnCsQImCsbl7NlkH3Yxs3/L2ldDk= -github.com/alist-org/times v0.0.0-20240721124318-c2e3da27cc69/go.mod h1:oPJwGY3sLmGgcJamGumz//0A35f4BwQRacyqLNcJTOU= +github.com/alist-org/gofakes3 v0.0.7 h1:0cDGI7fLBrqumhCBto9T3ZYCL71AyGZ1l+xxJgjqe8s= +github.com/alist-org/gofakes3 v0.0.7/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 h1:pIEI87zhv8ZzQcu65rTL7kqirrs8dR6HDiXrqWat2Fk= github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92/go.mod h1:oPJwGY3sLmGgcJamGumz//0A35f4BwQRacyqLNcJTOU= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= @@ -306,8 +304,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/larksuite/oapi-sdk-go/v3 v3.2.8 h1:elbufnS+gQVOkzX9JLkS/N9u3ay/IAIE17nE4kNoYZ4= -github.com/larksuite/oapi-sdk-go/v3 v3.2.8/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/larksuite/oapi-sdk-go/v3 v3.3.0 h1:aCtFUiYgoRUW+aaWzVYw8jSzMe4A71rPEIn1DyHcNrY= github.com/larksuite/oapi-sdk-go/v3 v3.3.0/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= diff --git a/server/s3/backend.go b/server/s3/backend.go index 2987d81e839..e0cfd9676b0 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -267,8 +267,19 @@ func (b *s3Backend) PutObject( } bucketPath := bucket.Path + isDir := strings.HasSuffix(objectName, "/") + log.Debugf("isDir: %v", isDir) + fp := path.Join(bucketPath, objectName) - reqPath := path.Dir(fp) + log.Debugf("fp: %s, bucketPath: %s, objectName: %s", fp, bucketPath, objectName) + + var reqPath string + if isDir { + reqPath = fp + "/" + } else { + reqPath = path.Dir(fp) + } + log.Debugf("reqPath: %s", reqPath) fmeta, _ := op.GetNearestMeta(fp) ctx = context.WithValue(ctx, "meta", fmeta) @@ -285,6 +296,10 @@ func (b *s3Backend) PutObject( } } + if isDir { + return result, nil + } + var ti time.Time if val, ok := meta["X-Amz-Meta-Mtime"]; ok { From 87caaf2459548d0daab0087a6c6d22817f17a85d Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sat, 3 Aug 2024 13:11:09 +0800 Subject: [PATCH 259/659] fix: out of order when database is not sqlite3 (#6560) --- internal/bootstrap/data/setting.go | 1 + internal/db/meta.go | 2 +- internal/db/settingitem.go | 3 ++- internal/db/storage.go | 9 +++------ internal/db/user.go | 2 +- internal/db/util.go | 5 +++++ internal/model/setting.go | 1 + 7 files changed, 14 insertions(+), 9 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 75244d84027..920a7a2d118 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -34,6 +34,7 @@ func initSettings() { // create or save setting for i := range initialSettingItems { item := &initialSettingItems[i] + item.Index = uint(i) if item.PreDefault == "" { item.PreDefault = item.Value } diff --git a/internal/db/meta.go b/internal/db/meta.go index 8b6a605e809..14532637a7b 100644 --- a/internal/db/meta.go +++ b/internal/db/meta.go @@ -34,7 +34,7 @@ func GetMetas(pageIndex, pageSize int) (metas []model.Meta, count int64, err err if err = metaDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get metas count") } - if err = metaDB.Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&metas).Error; err != nil { + if err = metaDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&metas).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find metas") } return metas, count, nil diff --git a/internal/db/settingitem.go b/internal/db/settingitem.go index 2ba0c665acd..9e61bfca00b 100644 --- a/internal/db/settingitem.go +++ b/internal/db/settingitem.go @@ -49,7 +49,8 @@ func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) { func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) { var settingItems []model.SettingItem - if err := db.Where(fmt.Sprintf("%s in ?", columnName("group")), groups).Find(&settingItems).Error; err != nil { + err := db.Order(columnName("index")).Where(fmt.Sprintf("%s in ?", columnName("group")), groups).Find(&settingItems).Error + if err != nil { return nil, errors.WithStack(err) } return settingItems, nil diff --git a/internal/db/storage.go b/internal/db/storage.go index d4e0730f064..376d42d7bf9 100644 --- a/internal/db/storage.go +++ b/internal/db/storage.go @@ -2,7 +2,6 @@ package db import ( "fmt" - "sort" "github.com/alist-org/alist/v3/internal/model" "github.com/pkg/errors" @@ -36,7 +35,7 @@ func GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) { return nil, 0, errors.Wrapf(err, "failed get storages count") } var storages []model.Storage - if err := storageDB.Order(columnName("order")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { + if err := addStorageOrder(storageDB).Order(columnName("order")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { return nil, 0, errors.WithStack(err) } return storages, count, nil @@ -63,11 +62,9 @@ func GetStorageByMountPath(mountPath string) (*model.Storage, error) { func GetEnabledStorages() ([]model.Storage, error) { var storages []model.Storage - if err := db.Where(fmt.Sprintf("%s = ?", columnName("disabled")), false).Find(&storages).Error; err != nil { + err := addStorageOrder(db).Where(fmt.Sprintf("%s = ?", columnName("disabled")), false).Find(&storages).Error + if err != nil { return nil, errors.WithStack(err) } - sort.Slice(storages, func(i, j int) bool { - return storages[i].Order < storages[j].Order - }) return storages, nil } diff --git a/internal/db/user.go b/internal/db/user.go index 822926664c9..8c9641b2c55 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -54,7 +54,7 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err if err := userDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get users count") } - if err := userDB.Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&users).Error; err != nil { + if err := userDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&users).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find users") } return users, count, nil diff --git a/internal/db/util.go b/internal/db/util.go index 38a06bcdacc..57f615bdeb1 100644 --- a/internal/db/util.go +++ b/internal/db/util.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/alist-org/alist/v3/internal/conf" + "gorm.io/gorm" ) func columnName(name string) string { @@ -12,3 +13,7 @@ func columnName(name string) string { } return fmt.Sprintf("`%s`", name) } + +func addStorageOrder(db *gorm.DB) *gorm.DB { + return db.Order(fmt.Sprintf("%s, %s", columnName("order"), columnName("id"))) +} diff --git a/internal/model/setting.go b/internal/model/setting.go index 9893124890e..c474935ed49 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -29,6 +29,7 @@ type SettingItem struct { Options string `json:"options"` // values for select Group int `json:"group"` // use to group setting in frontend Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. + Index uint `json:"index"` } func (s SettingItem) IsDeprecated() bool { From a6bead90d7543e3cf1e45e6517ff9bf53a4d35bc Mon Sep 17 00:00:00 2001 From: Sakana <1850575996@qq.com> Date: Sun, 4 Aug 2024 12:28:19 +0800 Subject: [PATCH 260/659] feat: add support for lenovonas_share driver (#6921) --- drivers/all.go | 1 + drivers/lenovonas_share/driver.go | 121 ++++++++++++++++++++++++++++++ drivers/lenovonas_share/meta.go | 33 ++++++++ drivers/lenovonas_share/types.go | 82 ++++++++++++++++++++ drivers/lenovonas_share/util.go | 36 +++++++++ 5 files changed, 273 insertions(+) create mode 100644 drivers/lenovonas_share/driver.go create mode 100644 drivers/lenovonas_share/meta.go create mode 100644 drivers/lenovonas_share/types.go create mode 100644 drivers/lenovonas_share/util.go diff --git a/drivers/all.go b/drivers/all.go index 2cb01d748b0..b976f92f2a6 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -29,6 +29,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/ilanzou" _ "github.com/alist-org/alist/v3/drivers/ipfs_api" _ "github.com/alist-org/alist/v3/drivers/lanzou" + _ "github.com/alist-org/alist/v3/drivers/lenovonas_share" _ "github.com/alist-org/alist/v3/drivers/local" _ "github.com/alist-org/alist/v3/drivers/mediatrack" _ "github.com/alist-org/alist/v3/drivers/mega" diff --git a/drivers/lenovonas_share/driver.go b/drivers/lenovonas_share/driver.go new file mode 100644 index 00000000000..12e8514325f --- /dev/null +++ b/drivers/lenovonas_share/driver.go @@ -0,0 +1,121 @@ +package LenovoNasShare + +import ( + "context" + "net/http" + + "github.com/go-resty/resty/v2" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type LenovoNasShare struct { + model.Storage + Addition + stoken string +} + +func (d *LenovoNasShare) Config() driver.Config { + return config +} + +func (d *LenovoNasShare) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *LenovoNasShare) Init(ctx context.Context) error { + if d.Host == "" { + d.Host = "https://siot-share.lenovo.com.cn" + } + query := map[string]string{ + "code": d.ShareId, + "password": d.SharePwd, + } + resp, err := d.request(d.Host+"/oneproxy/api/share/v1/access", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, nil) + if err != nil { + return err + } + d.stoken = utils.Json.Get(resp, "data", "stoken").ToString() + return nil +} + +func (d *LenovoNasShare) Drop(ctx context.Context) error { + return nil +} + +func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]File, 0) + + var resp Files + query := map[string]string{ + "code": d.ShareId, + "num": "5000", + "stoken": d.stoken, + "path": dir.GetPath(), + } + _, err := d.request(d.Host+"/oneproxy/api/share/v1/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return nil, err + } + files = append(files, resp.Data.List...) + + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return src, nil + }) +} + +func (d *LenovoNasShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + query := map[string]string{ + "code": d.ShareId, + "stoken": d.stoken, + "path": file.GetPath(), + } + resp, err := d.request(d.Host+"/oneproxy/api/share/v1/file/link", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, nil) + if err != nil { + return nil, err + } + downloadUrl := d.Host + "/oneproxy/api/share/v1/file/download?code=" + d.ShareId + "&dtoken=" + utils.Json.Get(resp, "data", "param", "dtoken").ToString() + + link := model.Link{ + URL: downloadUrl, + Header: http.Header{ + "Referer": []string{"https://siot-share.lenovo.com.cn"}, + }, + } + return &link, nil +} + +func (d *LenovoNasShare) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *LenovoNasShare) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *LenovoNasShare) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *LenovoNasShare) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *LenovoNasShare) Remove(ctx context.Context, obj model.Obj) error { + return errs.NotImplement +} + +func (d *LenovoNasShare) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*LenovoNasShare)(nil) diff --git a/drivers/lenovonas_share/meta.go b/drivers/lenovonas_share/meta.go new file mode 100644 index 00000000000..0bf80555739 --- /dev/null +++ b/drivers/lenovonas_share/meta.go @@ -0,0 +1,33 @@ +package LenovoNasShare + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + ShareId string `json:"share_id" required:"true" help:"The part after the last / in the shared link"` + SharePwd string `json:"share_pwd" required:"true" help:"The password of the shared link"` + Host string `json:"host" required:"true" default:"https://siot-share.lenovo.com.cn" help:"You can change it to your local area network"` +} + +var config = driver.Config{ + Name: "LenovoNasShare", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: true, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &LenovoNasShare{} + }) +} diff --git a/drivers/lenovonas_share/types.go b/drivers/lenovonas_share/types.go new file mode 100644 index 00000000000..77b966d3bee --- /dev/null +++ b/drivers/lenovonas_share/types.go @@ -0,0 +1,82 @@ +package LenovoNasShare + +import ( + "encoding/json" + "time" + + "github.com/alist-org/alist/v3/pkg/utils" + + _ "github.com/alist-org/alist/v3/internal/model" +) + +func (f *File) UnmarshalJSON(data []byte) error { + type Alias File + aux := &struct { + CreateAt int64 `json:"time"` + UpdateAt int64 `json:"chtime"` + *Alias + }{ + Alias: (*Alias)(f), + } + + if err := json.Unmarshal(data, aux); err != nil { + return err + } + + f.CreateAt = time.Unix(aux.CreateAt, 0) + f.UpdateAt = time.Unix(aux.UpdateAt, 0) + + return nil +} + +type File struct { + FileName string `json:"name"` + Size int64 `json:"size"` + CreateAt time.Time `json:"time"` + UpdateAt time.Time `json:"chtime"` + Path string `json:"path"` + Type string `json:"type"` +} + +func (f File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f File) GetPath() string { + return f.Path +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) ModTime() time.Time { + return f.UpdateAt +} + +func (f File) CreateTime() time.Time { + return f.CreateAt +} + +func (f File) IsDir() bool { + return f.Type == "dir" +} + +func (f File) GetID() string { + return f.GetPath() +} + +func (f File) Thumb() string { + return "" +} + +type Files struct { + Data struct { + List []File `json:"list"` + HasMore bool `json:"has_more"` + } `json:"data"` +} diff --git a/drivers/lenovonas_share/util.go b/drivers/lenovonas_share/util.go new file mode 100644 index 00000000000..ccf3af042a4 --- /dev/null +++ b/drivers/lenovonas_share/util.go @@ -0,0 +1,36 @@ +package LenovoNasShare + +import ( + "errors" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + jsoniter "github.com/json-iterator/go" +) + +func (d *LenovoNasShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "origin": "https://siot-share.lenovo.com.cn", + "referer": "https://siot-share.lenovo.com.cn/", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) alist-client", + "platform": "web", + "app-version": "3", + }) + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + body := res.Body() + result := utils.Json.Get(body, "result").ToBool() + if !result { + return nil, errors.New(jsoniter.Get(body, "error", "msg").ToString()) + } + return body, nil +} From 81258d3e8abdf4fc9960fdce6aa4bd822dcf5504 Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Sun, 4 Aug 2024 12:32:39 +0800 Subject: [PATCH 261/659] feat: invalidate token on logout (#6923 close #6792) --- server/common/auth.go | 23 +++++++++++++++++++++++ server/handles/auth.go | 9 +++++++++ server/router.go | 1 + 3 files changed, 33 insertions(+) diff --git a/server/common/auth.go b/server/common/auth.go index b6a79b752aa..0de718cf9e8 100644 --- a/server/common/auth.go +++ b/server/common/auth.go @@ -3,6 +3,7 @@ package common import ( "time" + "github.com/Xhofe/go-cache" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/golang-jwt/jwt/v4" @@ -17,6 +18,8 @@ type UserClaims struct { jwt.RegisteredClaims } +var validTokenCache = cache.NewMemCache[bool]() + func GenerateToken(user *model.User) (tokenString string, err error) { claim := UserClaims{ Username: user.Username, @@ -28,6 +31,10 @@ func GenerateToken(user *model.User) (tokenString string, err error) { }} token := jwt.NewWithClaims(jwt.SigningMethodHS256, claim) tokenString, err = token.SignedString(SecretKey) + if err != nil { + return "", err + } + validTokenCache.Set(tokenString, true) return tokenString, err } @@ -35,6 +42,9 @@ func ParseToken(tokenString string) (*UserClaims, error) { token, err := jwt.ParseWithClaims(tokenString, &UserClaims{}, func(token *jwt.Token) (interface{}, error) { return SecretKey, nil }) + if IsTokenInvalidated(tokenString) { + return nil, errors.New("token is invalidated") + } if err != nil { if ve, ok := err.(*jwt.ValidationError); ok { if ve.Errors&jwt.ValidationErrorMalformed != 0 { @@ -53,3 +63,16 @@ func ParseToken(tokenString string) (*UserClaims, error) { } return nil, errors.New("couldn't handle this token") } + +func InvalidateToken(tokenString string) error { + if tokenString == "" { + return nil // don't invalidate empty guest token + } + validTokenCache.Del(tokenString) + return nil +} + +func IsTokenInvalidated(tokenString string) bool { + _, ok := validTokenCache.Get(tokenString) + return !ok +} diff --git a/server/handles/auth.go b/server/handles/auth.go index 209bdd3a2b8..e1f512c4dc1 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -181,3 +181,12 @@ func Verify2FA(c *gin.Context) { common.SuccessResp(c) } } + +func LogOut(c *gin.Context) { + err := common.InvalidateToken(c.GetHeader("Authorization")) + if err != nil { + common.ErrorResp(c, err, 500) + } else { + common.SuccessResp(c) + } +} diff --git a/server/router.go b/server/router.go index 5f784aa4b7d..5be593f7497 100644 --- a/server/router.go +++ b/server/router.go @@ -54,6 +54,7 @@ func Init(e *gin.Engine) { auth.POST("/me/update", handles.UpdateCurrent) auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) + auth.GET("/auth/logout", handles.LogOut) // auth api.GET("/auth/sso", handles.SSOLoginRedirect) From 2e4265a778778f6236d4a42112064632d8c48a2c Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 4 Aug 2024 18:28:35 +0800 Subject: [PATCH 262/659] feat: deleting folders is not allowed (close #6933) --- internal/op/fs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/op/fs.go b/internal/op/fs.go index 5c9c9f3f138..ed28039529f 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -466,6 +466,9 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } + if utils.PathEqual(path, "/") { + return errors.New("delete root folder is not allowed, please goto the manage page to delete the storage instead") + } path = utils.FixAndCleanPath(path) rawObj, err := Get(ctx, storage, path) if err != nil { From d4285b7c6c28304d57c5833ccdf7adcdace2126e Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sun, 4 Aug 2024 19:03:24 +0800 Subject: [PATCH 263/659] fix(halalcloud): fix some custom fields not taking effect & update appID and appSecret (#6938) --- drivers/halalcloud/meta.go | 4 ++-- drivers/halalcloud/util.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/drivers/halalcloud/meta.go b/drivers/halalcloud/meta.go index b60445c061d..d4040323eb0 100644 --- a/drivers/halalcloud/meta.go +++ b/drivers/halalcloud/meta.go @@ -12,9 +12,9 @@ type Addition struct { RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` UploadThread string `json:"upload_thread" default:"3" help:"1 <= thread <= 32"` - AppID string `json:"app_id" required:"true" default:"devDebugger/1.0"` + AppID string `json:"app_id" required:"true" default:"alist/10001"` AppVersion string `json:"app_version" required:"true" default:"1.0.0"` - AppSecret string `json:"app_secret" required:"true" default:"Nkx3Y2xvZ2luLmNu"` + AppSecret string `json:"app_secret" required:"true" default:"bR4SJwOkvnG5WvVJ"` } var config = driver.Config{ diff --git a/drivers/halalcloud/util.go b/drivers/halalcloud/util.go index 33a347e753e..be1b9b36551 100644 --- a/drivers/halalcloud/util.go +++ b/drivers/halalcloud/util.go @@ -29,9 +29,9 @@ import ( ) const ( - AppID = "devDebugger/1.0" + AppID = "alist/10001" AppVersion = "1.0.0" - AppSecret = "Nkx3Y2xvZ2luLmNu" + AppSecret = "bR4SJwOkvnG5WvVJ" ) const ( @@ -179,16 +179,16 @@ func (s *AuthService) signContext(method string, ctx context.Context) context.Co bufferedString := bytes.NewBufferString(method) kvString = append(kvString, "timestamp", currentTimeStamp) bufferedString.WriteString(currentTimeStamp) - kvString = append(kvString, "appid", AppID) - bufferedString.WriteString(AppID) - kvString = append(kvString, "appversion", AppVersion) - bufferedString.WriteString(AppVersion) + kvString = append(kvString, "appid", s.appID) + bufferedString.WriteString(s.appID) + kvString = append(kvString, "appversion", s.appVersion) + bufferedString.WriteString(s.appVersion) if s.tr != nil && len(s.tr.AccessToken) > 0 { authorization := "Bearer " + s.tr.AccessToken kvString = append(kvString, "authorization", authorization) bufferedString.WriteString(authorization) } - bufferedString.WriteString(AppSecret) + bufferedString.WriteString(s.appSecret) sign := GetMD5Hash(bufferedString.String()) kvString = append(kvString, "sign", sign) return metadata.AppendToOutgoingContext(ctx, kvString...) From f2727095d96e3d33c457f9ef7c56ca4047047a3b Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Tue, 6 Aug 2024 22:13:32 +0800 Subject: [PATCH 264/659] fix(thunder_browser): fix space parameter not handled correctly in some cases & update some parameters (#6952) --- drivers/thunder_browser/driver.go | 198 +++++------------------------- drivers/thunder_browser/meta.go | 4 +- drivers/thunder_browser/types.go | 33 +++-- drivers/thunder_browser/util.go | 37 +++--- 4 files changed, 71 insertions(+), 201 deletions(-) diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index a389f6102fc..96dd7e8ecce 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -15,8 +15,8 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" + "io" "net/http" - "regexp" "strings" ) @@ -309,7 +309,7 @@ type XunLeiBrowserCommon struct { } func (xc *XunLeiBrowserCommon) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - return xc.getFiles(ctx, dir.GetID(), args.ReqPath) + return xc.getFiles(ctx, dir, args.ReqPath) } func (xc *XunLeiBrowserCommon) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -317,17 +317,10 @@ func (xc *XunLeiBrowserCommon) Link(ctx context.Context, file model.Obj, args mo params := map[string]string{ "_magic": "2021", - "space": "SPACE_BROWSER", + "space": file.(*Files).GetSpace(), "thumbnail_size": "SIZE_LARGE", "with": "url", } - // 对 "迅雷云盘" 内的文件 特殊处理 - if file.GetPath() == ThunderDriveFileID { - params = map[string]string{} - } else if file.GetPath() == ThunderBrowserDriveSafeFileID { - // 对 "超级保险箱" 内的文件 特殊处理 - params["space"] = "SPACE_BROWSER_SAFE" - } _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) @@ -361,22 +354,9 @@ func (xc *XunLeiBrowserCommon) MakeDir(ctx context.Context, parentDir model.Obj, "kind": FOLDER, "name": dirName, "parent_id": parentDir.GetID(), - "space": "SPACE_BROWSER", - } - if parentDir.GetPath() == ThunderDriveFileID { - js = base.Json{ - "kind": FOLDER, - "name": dirName, - "parent_id": parentDir.GetID(), - } - } else if parentDir.GetPath() == ThunderBrowserDriveSafeFileID { - js = base.Json{ - "kind": FOLDER, - "name": dirName, - "parent_id": parentDir.GetID(), - "space": "SPACE_BROWSER_SAFE", - } + "space": parentDir.(*Files).GetSpace(), } + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) @@ -386,33 +366,14 @@ func (xc *XunLeiBrowserCommon) MakeDir(ctx context.Context, parentDir model.Obj, func (xc *XunLeiBrowserCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - srcSpace := "SPACE_BROWSER" - dstSpace := "SPACE_BROWSER" - - // 对 "超级保险箱" 内的文件 特殊处理 - if srcObj.GetPath() == ThunderBrowserDriveSafeFileID { - srcSpace = "SPACE_BROWSER_SAFE" - } - if dstDir.GetPath() == ThunderBrowserDriveSafeFileID { - dstSpace = "SPACE_BROWSER_SAFE" - } - params := map[string]string{ - "_from": dstSpace, + "_from": srcObj.(*Files).GetSpace(), } js := base.Json{ - "to": base.Json{"parent_id": dstDir.GetID(), "space": dstSpace}, - "space": srcSpace, + "to": base.Json{"parent_id": dstDir.GetID(), "space": dstDir.(*Files).GetSpace()}, + "space": srcObj.(*Files).GetSpace(), "ids": []string{srcObj.GetID()}, } - // 对 "迅雷云盘" 内的文件 特殊处理 - if srcObj.GetPath() == ThunderDriveFileID { - params = map[string]string{} - js = base.Json{ - "to": base.Json{"parent_id": dstDir.GetID()}, - "ids": []string{srcObj.GetID()}, - } - } _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) @@ -425,16 +386,7 @@ func (xc *XunLeiBrowserCommon) Move(ctx context.Context, srcObj, dstDir model.Ob func (xc *XunLeiBrowserCommon) Rename(ctx context.Context, srcObj model.Obj, newName string) error { params := map[string]string{ - "space": "SPACE_BROWSER", - } - // 对 "迅雷云盘" 内的文件 特殊处理 - if srcObj.GetPath() == ThunderDriveFileID { - params = map[string]string{} - } else if srcObj.GetPath() == ThunderBrowserDriveSafeFileID { - // 对 "超级保险箱" 内的文件 特殊处理 - params = map[string]string{ - "space": "SPACE_BROWSER_SAFE", - } + "space": srcObj.(*Files).GetSpace(), } _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { @@ -448,33 +400,14 @@ func (xc *XunLeiBrowserCommon) Rename(ctx context.Context, srcObj model.Obj, new func (xc *XunLeiBrowserCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - srcSpace := "SPACE_BROWSER" - dstSpace := "SPACE_BROWSER" - - // 对 "超级保险箱" 内的文件 特殊处理 - if srcObj.GetPath() == ThunderBrowserDriveSafeFileID { - srcSpace = "SPACE_BROWSER_SAFE" - } - if dstDir.GetPath() == ThunderBrowserDriveSafeFileID { - dstSpace = "SPACE_BROWSER_SAFE" - } - params := map[string]string{ - "_from": dstSpace, + "_from": srcObj.(*Files).GetSpace(), } js := base.Json{ - "to": base.Json{"parent_id": dstDir.GetID(), "space": dstSpace}, - "space": srcSpace, + "to": base.Json{"parent_id": dstDir.GetID(), "space": dstDir.(*Files).GetSpace()}, + "space": srcObj.(*Files).GetSpace(), "ids": []string{srcObj.GetID()}, } - // 对 "迅雷云盘" 内的文件 特殊处理 - if srcObj.GetPath() == ThunderDriveFileID { - params = map[string]string{} - js = base.Json{ - "to": base.Json{"parent_id": dstDir.GetID()}, - "ids": []string{srcObj.GetID()}, - } - } _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) @@ -488,30 +421,17 @@ func (xc *XunLeiBrowserCommon) Remove(ctx context.Context, obj model.Obj) error js := base.Json{ "ids": []string{obj.GetID()}, - "space": "SPACE_BROWSER", + "space": obj.(*Files).GetSpace(), } - // 对 "迅雷云盘" 内的文件 特殊处理 - if obj.GetPath() == ThunderDriveFileID { - js = base.Json{ - "ids": []string{obj.GetID()}, - } - } else if obj.GetPath() == ThunderBrowserDriveSafeFileID { - // 对 "超级保险箱" 内的文件 特殊处理 - js = base.Json{ - "ids": []string{obj.GetID()}, - "space": "SPACE_BROWSER_SAFE", - } - } - // 先判断是否是特殊情况 - if obj.GetPath() == ThunderDriveFileID { + if obj.(*Files).GetSpace() == ThunderDriveSpace { _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { r.SetContext(ctx) r.SetPathParam("fileID", obj.GetID()) r.SetBody("{}") }, nil) return err - } else if obj.GetPath() == ThunderBrowserDriveSafeFileID { + } else if obj.(*Files).GetSpace() == ThunderBrowserDriveSafeSpace || obj.(*Files).GetSpace() == ThunderDriveSafeSpace { _, err := xc.Request(FILE_API_URL+":batchDelete", http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) @@ -557,29 +477,7 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream "size": stream.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, - "space": "SPACE_BROWSER", - } - // 对 "迅雷云盘" 内的文件 特殊处理 - if dstDir.GetPath() == ThunderDriveFileID { - js = base.Json{ - "kind": FILE, - "parent_id": dstDir.GetID(), - "name": stream.GetName(), - "size": stream.GetSize(), - "hash": gcid, - "upload_type": UPLOAD_TYPE_RESUMABLE, - } - } else if dstDir.GetPath() == ThunderBrowserDriveSafeFileID { - // 对 "超级保险箱" 内的文件 特殊处理 - js = base.Json{ - "kind": FILE, - "parent_id": dstDir.GetID(), - "name": stream.GetName(), - "size": stream.GetSize(), - "hash": gcid, - "upload_type": UPLOAD_TYPE_RESUMABLE, - "space": "SPACE_BROWSER_SAFE", - } + "space": dstDir.(*Files).GetSpace(), } var resp UploadTaskResponse @@ -610,58 +508,35 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), - Body: stream, + Body: io.TeeReader(stream, driver.NewProgress(stream.GetSize(), up)), }) return err } return nil } -func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, folderId string, path string) ([]model.Obj, error) { +func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, dir model.Obj, path string) ([]model.Obj, error) { files := make([]model.Obj, 0) var pageToken string for { var fileList FileList - folderSpace := "SPACE_BROWSER" + folderSpace := "" + switch dirF := dir.(type) { + case *Files: + folderSpace = dirF.GetSpace() + default: + // 处理 根目录的情况 + folderSpace = ThunderBrowserDriveSpace + } params := map[string]string{ - "parent_id": folderId, + "parent_id": dir.GetID(), "page_token": pageToken, "space": folderSpace, "filters": `{"trashed":{"eq":false}}`, + "with": "url", "with_audit": "true", "thumbnail_size": "SIZE_LARGE", } - var fileType int8 - // 处理特殊目录 “迅雷云盘” 设置特殊的 params 以便正常访问 - pattern1 := fmt.Sprintf(`^/.*/%s(/.*)?$`, ThunderDriveFolderName) - thunderDriveMatch, _ := regexp.MatchString(pattern1, path) - // 处理特殊目录 “超级保险箱” 设置特殊的 params 以便正常访问 - pattern2 := fmt.Sprintf(`^/.*/%s(/.*)?$`, ThunderBrowserDriveSafeFolderName) - thunderBrowserDriveSafeMatch, _ := regexp.MatchString(pattern2, path) - - // 如果是 "迅雷云盘" 内的 - if folderId == ThunderDriveFileID || thunderDriveMatch { - params = map[string]string{ - "space": "", - "__type": "drive", - "refresh": "true", - "__sync": "true", - "parent_id": folderId, - "page_token": pageToken, - "with_audit": "true", - "limit": "100", - "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, - } - // 如果不是 "迅雷云盘"的"根目录" - if folderId == ThunderDriveFileID { - params["parent_id"] = "" - } - fileType = ThunderDriveType - } else if thunderBrowserDriveSafeMatch { - // 如果是 "超级保险箱" 内的 - fileType = ThunderBrowserDriveSafeType - params["space"] = "SPACE_BROWSER_SAFE" - } _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { r.SetContext(ctx) @@ -670,24 +545,13 @@ func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, folderId string, pa if err != nil { return nil, err } - // 对文件夹也进行处理 - fileList.FolderType = fileType - for i := 0; i < len(fileList.Files); i++ { - file := &fileList.Files[i] - // 标记 文件夹内的文件 - file.FileType = fileList.FolderType + for i := range fileList.Files { // 解决 "迅雷云盘" 重复出现问题————迅雷后端发送错误 - if file.Name == ThunderDriveFolderName && file.ID == "" && file.FolderType == ThunderDriveFolderType && folderId != "" { + if fileList.Files[i].FolderType == ThunderDriveFolderType && fileList.Files[i].ID == "" && fileList.Files[i].Space == "" && dir.GetID() != "" { continue } - // 处理特殊目录 “迅雷云盘” 设置特殊的文件夹ID - if file.Name == ThunderDriveFolderName && file.ID == "" && file.FolderType == ThunderDriveFolderType { - file.ID = ThunderDriveFileID - } else if file.Name == ThunderBrowserDriveSafeFolderName && file.FolderType == ThunderBrowserDriveSafeFolderType { - file.FileType = ThunderBrowserDriveSafeType - } - files = append(files, file) + files = append(files, &fileList.Files[i]) } if fileList.NextPageToken == "" { diff --git a/drivers/thunder_browser/meta.go b/drivers/thunder_browser/meta.go index 247353b7b0f..f535ea6cd47 100644 --- a/drivers/thunder_browser/meta.go +++ b/drivers/thunder_browser/meta.go @@ -25,7 +25,7 @@ type ExpertAddition struct { SafePassword string `json:"safe_password" required:"true" help:"super safe password"` // 超级保险箱密码 // 签名方法1 - Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"p+ExqPV,LwdwKlprzv7cQBQmxN5,vc08P1NwUBnbGsl58LzTW,VVNeXaXmZ8HH1SJEnp6YpVFSFU,pNAOJ,CNChvyDehAmUR1TDodfOusBAx,MS98NnX4Np8nxvEh6Ulv+SMMKMzKvD34C7lGWbb,9MpFF21GnVOYku0NM9Y/hzsK471UCUZ2o+,EY1QfeA06fXlw9wZNoZaXEED5zZPvNWI,,sciE,FIPqgQDUUW1e0GkiBFd5w7mCQ,zW,75XFdEO0Gi"` + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"uWRwO7gPfdPB/0NfPtfQO+71,F93x+qPluYy6jdgNpq+lwdH1ap6WOM+nfz8/V,0HbpxvpXFsBK5CoTKam,dQhzbhzFRcawnsZqRETT9AuPAJ+wTQso82mRv,SAH98AmLZLRa6DB2u68sGhyiDh15guJpXhBzI,unqfo7Z64Rie9RNHMOB,7yxUdFADp3DOBvXdz0DPuKNVT35wqa5z0DEyEvf,RBG,ThTWPG5eC0UBqlbQ+04nZAptqGCdpv9o55A"` // 签名方法2 CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` @@ -37,7 +37,7 @@ type ExpertAddition struct { DeviceID string `json:"device_id" required:"false" default:""` ClientID string `json:"client_id" required:"true" default:"ZUBzD9J_XPXfn7f7"` ClientSecret string `json:"client_secret" required:"true" default:"yESVmHecEe6F0aou69vl-g"` - ClientVersion string `json:"client_version" required:"true" default:"1.0.8.2215"` + ClientVersion string `json:"client_version" required:"true" default:"1.10.0.2633"` PackageName string `json:"package_name" required:"true" default:"com.xunlei.browser"` // 不影响登录,影响下载速度 diff --git a/drivers/thunder_browser/types.go b/drivers/thunder_browser/types.go index 774b34bb287..b3e21d2bc08 100644 --- a/drivers/thunder_browser/types.go +++ b/drivers/thunder_browser/types.go @@ -114,8 +114,8 @@ type Files struct { ModifiedTime CustomTime `json:"modified_time"` IconLink string `json:"icon_link"` ThumbnailLink string `json:"thumbnail_link"` - // Md5Checksum string `json:"md5_checksum"` - Hash string `json:"hash"` + Md5Checksum string `json:"md5_checksum"` + Hash string `json:"hash"` // Links map[string]Link `json:"links"` // Phase string `json:"phase"` // Audit struct { @@ -153,12 +153,22 @@ type Files struct { OriginalURL string `json:"original_url"` //Params struct{} `json:"params"` //OriginalFileIndex int `json:"original_file_index"` - //Space string `json:"space"` + Space string `json:"space"` //Apps []interface{} `json:"apps"` //Writable bool `json:"writable"` FolderType string `json:"folder_type"` //Collection interface{} `json:"collection"` - FileType int8 + SortName string `json:"sort_name"` + UserModifiedTime CustomTime `json:"user_modified_time"` + //SpellName []interface{} `json:"spell_name"` + //FileCategory string `json:"file_category"` + //Tags []interface{} `json:"tags"` + //ReferenceEvents []interface{} `json:"reference_events"` + //ReferenceResource interface{} `json:"reference_resource"` + //Params0 struct { + // PlatformIcon string `json:"platform_icon"` + // SmallThumbnail string `json:"small_thumbnail"` + //} `json:"params,omitempty"` } func (c *Files) GetHash() utils.HashInfo { @@ -172,16 +182,19 @@ func (c *Files) ModTime() time.Time { return c.ModifiedTime.Time } func (c *Files) IsDir() bool { return c.Kind == FOLDER } func (c *Files) GetID() string { return c.ID } func (c *Files) GetPath() string { - // 对特殊文件进行特殊处理 - if c.FileType == ThunderDriveType { - return ThunderDriveFileID - } else if c.FileType == ThunderBrowserDriveSafeType { - return ThunderBrowserDriveSafeFileID - } return "" } func (c *Files) Thumb() string { return c.ThumbnailLink } +func (c *Files) GetSpace() string { + if c.Space != "" { + return c.Space + } else { + // "迅雷云盘" 文件夹内 Space 为空 + return "" + } +} + /* * 上传 **/ diff --git a/drivers/thunder_browser/util.go b/drivers/thunder_browser/util.go index a5f6f663e8f..befd1a904c8 100644 --- a/drivers/thunder_browser/util.go +++ b/drivers/thunder_browser/util.go @@ -23,29 +23,24 @@ const ( ) var Algorithms = []string{ - "p+ExqPV", - "LwdwKlprzv7cQBQmxN5", - "vc08P1NwUBnbGsl58LzTW", - "VVNeXaXmZ8HH1SJEnp6YpVFSFU", - "pNAOJ", - "CNChvyDehAmUR1TDodfOusBAx", - "MS98NnX4Np8nxvEh6Ulv+SMMKMzKvD34C7lGWbb", - "9MpFF21GnVOYku0NM9Y/hzsK471UCUZ2o+", - "EY1QfeA06fXlw9wZNoZaXEED5zZPvNWI", - "", - "sciE", - "FIPqgQDUUW1e0GkiBFd5w7mCQ", - "zW", - "75XFdEO0Gi", + "uWRwO7gPfdPB/0NfPtfQO+71", + "F93x+qPluYy6jdgNpq+lwdH1ap6WOM+nfz8/V", + "0HbpxvpXFsBK5CoTKam", + "dQhzbhzFRcawnsZqRETT9AuPAJ+wTQso82mRv", + "SAH98AmLZLRa6DB2u68sGhyiDh15guJpXhBzI", + "unqfo7Z64Rie9RNHMOB", + "7yxUdFADp3DOBvXdz0DPuKNVT35wqa5z0DEyEvf", + "RBG", + "ThTWPG5eC0UBqlbQ+04nZAptqGCdpv9o55A", } const ( ClientID = "ZUBzD9J_XPXfn7f7" ClientSecret = "yESVmHecEe6F0aou69vl-g" - ClientVersion = "1.0.8.2215" + ClientVersion = "1.10.0.2633" PackageName = "com.xunlei.browser" DownloadUserAgent = "AndroidDownloadManager/13 (Linux; U; Android 13; M2004J7AC Build/SP1A.210812.016)" - SdkVersion = "2.0.3.262" + SdkVersion = "233100" ) const ( @@ -62,12 +57,10 @@ const ( ) const ( - ThunderDriveFileID = "XXXXXXXXXXXXXXXXXXXXXXXXXX" - ThunderBrowserDriveSafeFileID = "YYYYYYYYYYYYYYYYYYYYYYYYYY" - ThunderDriveFolderName = "迅雷云盘" - ThunderBrowserDriveSafeFolderName = "超级保险箱" - ThunderDriveType = 1 - ThunderBrowserDriveSafeType = 2 + ThunderDriveSpace = "" + ThunderDriveSafeSpace = "SPACE_SAFE" + ThunderBrowserDriveSpace = "SPACE_BROWSER" + ThunderBrowserDriveSafeSpace = "SPACE_BROWSER_SAFE" ThunderDriveFolderType = "DEFAULT_ROOT" ThunderBrowserDriveSafeFolderType = "BROWSER_SAFE" ) From 74f8295960d6d10603c5ad78518866e8d16ced8d Mon Sep 17 00:00:00 2001 From: itsHenry <2671230065@qq.com> Date: Wed, 7 Aug 2024 12:16:21 +0800 Subject: [PATCH 265/659] feat: persistant Task (#6925 close #5313) --- go.mod | 2 +- go.sum | 2 + internal/bootstrap/config.go | 12 +++--- internal/bootstrap/data/data.go | 1 + internal/bootstrap/data/task.go | 29 +++++++++++++ internal/bootstrap/task.go | 12 ++++-- internal/conf/config.go | 20 +++++---- internal/db/db.go | 2 +- internal/db/tasks.go | 48 +++++++++++++++++++++ internal/fs/copy.go | 50 +++++++++++++++------- internal/model/task.go | 6 +++ internal/offline_download/tool/add.go | 1 + internal/offline_download/tool/download.go | 30 ++++++++----- internal/offline_download/tool/transfer.go | 29 ++++++++----- internal/offline_download/tool/util.go | 13 ++++++ 15 files changed, 201 insertions(+), 56 deletions(-) create mode 100644 internal/bootstrap/data/task.go create mode 100644 internal/db/tasks.go create mode 100644 internal/model/task.go diff --git a/go.mod b/go.mod index 8bed00741b5..318da35057a 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 - github.com/xhofe/tache v0.1.1 + github.com/xhofe/tache v0.1.2 github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.25.0 diff --git a/go.sum b/go.sum index d82af70ecf2..9de941c5e85 100644 --- a/go.sum +++ b/go.sum @@ -506,6 +506,8 @@ github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3K github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.1 h1:O5QY4cVjIGELx3UGh6LbVAc18MWGXgRNQjMt72x6w/8= github.com/xhofe/tache v0.1.1/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/xhofe/tache v0.1.2 h1:pHrXlrWcbTb4G7hVUDW7Rc+YTUnLJvnLBrdktVE1Fqg= +github.com/xhofe/tache v0.1.2/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 2b7e9e13506..ff36509cf5f 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -68,11 +68,7 @@ func InitConfig() { } conf.Conf.TempDir = absPath } - err := os.RemoveAll(filepath.Join(conf.Conf.TempDir)) - if err != nil { - log.Errorln("failed delete temp file:", err) - } - err = os.MkdirAll(conf.Conf.TempDir, 0o777) + err := os.MkdirAll(conf.Conf.TempDir, 0o777) if err != nil { log.Fatalf("create temp dir error: %+v", err) } @@ -104,3 +100,9 @@ func initURL() { } conf.URL = u } + +func CleanTempDir() { + if err := os.RemoveAll(conf.Conf.TempDir); err != nil { + log.Errorln("failed delete temp file: ", err) + } +} diff --git a/internal/bootstrap/data/data.go b/internal/bootstrap/data/data.go index 6c77ebf2385..c2170d2f479 100644 --- a/internal/bootstrap/data/data.go +++ b/internal/bootstrap/data/data.go @@ -5,6 +5,7 @@ import "github.com/alist-org/alist/v3/cmd/flags" func InitData() { initUser() initSettings() + initTasks() if flags.Dev { initDevData() initDevDo() diff --git a/internal/bootstrap/data/task.go b/internal/bootstrap/data/task.go new file mode 100644 index 00000000000..7100e2e25c1 --- /dev/null +++ b/internal/bootstrap/data/task.go @@ -0,0 +1,29 @@ +package data + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" +) + +var initialTaskItems []model.TaskItem + +func initTasks() { + InitialTasks() + + for i := range initialTaskItems { + item := &initialTaskItems[i] + taskitem, _ := db.GetTaskDataByType(item.Key) + if taskitem == nil { + db.CreateTaskData(item) + } + } +} + +func InitialTasks() []model.TaskItem { + initialTaskItems = []model.TaskItem{ + {Key: "copy", PersistData: "[]"}, + {Key: "download", PersistData: "[]"}, + {Key: "transfer", PersistData: "[]"}, + } + return initialTaskItems +} diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go index 5d52e9d2ef8..3390235320c 100644 --- a/internal/bootstrap/task.go +++ b/internal/bootstrap/task.go @@ -2,14 +2,18 @@ package bootstrap import ( "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/xhofe/tache" ) func InitTaskManager() { - fs.UploadTaskManager = tache.NewManager[*fs.UploadTask](tache.WithWorks(conf.Conf.Tasks.Upload.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) - fs.CopyTaskManager = tache.NewManager[*fs.CopyTask](tache.WithWorks(conf.Conf.Tasks.Copy.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Copy.MaxRetry)) - tool.DownloadTaskManager = tache.NewManager[*tool.DownloadTask](tache.WithWorks(conf.Conf.Tasks.Download.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Download.MaxRetry)) - tool.TransferTaskManager = tache.NewManager[*tool.TransferTask](tache.WithWorks(conf.Conf.Tasks.Transfer.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Transfer.MaxRetry)) + fs.UploadTaskManager = tache.NewManager[*fs.UploadTask](tache.WithWorks(conf.Conf.Tasks.Upload.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //upload will not support persist + fs.CopyTaskManager = tache.NewManager[*fs.CopyTask](tache.WithWorks(conf.Conf.Tasks.Copy.Workers), tache.WithPersistFunction(db.GetTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant), db.UpdateTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Copy.MaxRetry)) + tool.DownloadTaskManager = tache.NewManager[*tool.DownloadTask](tache.WithWorks(conf.Conf.Tasks.Download.Workers), tache.WithPersistFunction(db.GetTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant), db.UpdateTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Download.MaxRetry)) + tool.TransferTaskManager = tache.NewManager[*tool.TransferTask](tache.WithWorks(conf.Conf.Tasks.Transfer.Workers), tache.WithPersistFunction(db.GetTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant), db.UpdateTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Transfer.MaxRetry)) + if len(tool.TransferTaskManager.GetAll()) == 0 { //prevent offline downloaded files from being deleted + CleanTempDir() + } } diff --git a/internal/conf/config.go b/internal/conf/config.go index 742aab1f265..c5dc9c521bf 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -47,8 +47,9 @@ type LogConfig struct { } type TaskConfig struct { - Workers int `json:"workers" env:"WORKERS"` - MaxRetry int `json:"max_retry" env:"MAX_RETRY"` + Workers int `json:"workers" env:"WORKERS"` + MaxRetry int `json:"max_retry" env:"MAX_RETRY"` + TaskPersistant bool `json:"task_persistant" env:"TASK_PERSISTANT"` } type TasksConfig struct { @@ -130,19 +131,22 @@ func DefaultConfig() *Config { TlsInsecureSkipVerify: true, Tasks: TasksConfig{ Download: TaskConfig{ - Workers: 5, - MaxRetry: 1, + Workers: 5, + MaxRetry: 1, + TaskPersistant: true, }, Transfer: TaskConfig{ - Workers: 5, - MaxRetry: 2, + Workers: 5, + MaxRetry: 2, + TaskPersistant: true, }, Upload: TaskConfig{ Workers: 5, }, Copy: TaskConfig{ - Workers: 5, - MaxRetry: 2, + Workers: 5, + MaxRetry: 2, + TaskPersistant: true, }, }, Cors: Cors{ diff --git a/internal/db/db.go b/internal/db/db.go index cd3905ffc92..2df58d3760b 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/tasks.go b/internal/db/tasks.go new file mode 100644 index 00000000000..9d2de1cff64 --- /dev/null +++ b/internal/db/tasks.go @@ -0,0 +1,48 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" +) + +func GetTaskDataByType(type_s string) (*model.TaskItem, error) { + task := model.TaskItem{Key: type_s} + if err := db.Where(task).First(&task).Error; err != nil { + return nil, errors.Wrapf(err, "failed find task") + } + return &task, nil +} + +func UpdateTaskData(t *model.TaskItem) error { + return errors.WithStack(db.Model(&model.TaskItem{}).Where("key = ?", t.Key).Update("persist_data", t.PersistData).Error) +} + +func CreateTaskData(t *model.TaskItem) error { + return errors.WithStack(db.Create(t).Error) +} + +func GetTaskDataFunc(type_s string, enabled bool) func() ([]byte, error) { + if !enabled { + return nil + } + task, err := GetTaskDataByType(type_s) + if err != nil { + return nil + } + return func() ([]byte, error) { + return []byte(task.PersistData), nil + } +} + +func UpdateTaskDataFunc(type_s string, enabled bool) func([]byte) error { + if !enabled { + return nil + } + return func(data []byte) error { + s := string(data) + if s == "null" || s == "" { + s = "[]" + } + return UpdateTaskData(&model.TaskItem{Key: type_s, PersistData: s}) + } +} diff --git a/internal/fs/copy.go b/internal/fs/copy.go index 25f068f0c40..38407c9a863 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -3,6 +3,9 @@ package fs import ( "context" "fmt" + "net/http" + stdpath "path" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" @@ -11,20 +14,21 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" "github.com/xhofe/tache" - "net/http" - stdpath "path" ) type CopyTask struct { tache.Base - Status string `json:"status"` - srcStorage, dstStorage driver.Driver - srcObjPath, dstDirPath string + Status string `json:"-"` //don't save status to save space + SrcObjPath string `json:"src_path"` + DstDirPath string `json:"dst_path"` + srcStorage driver.Driver `json:"-"` + dstStorage driver.Driver `json:"-"` + SrcStorageMp string `json:"src_storage_mp"` + DstStorageMp string `json:"dst_storage_mp"` } func (t *CopyTask) GetName() string { - return fmt.Sprintf("copy [%s](%s) to [%s](%s)", - t.srcStorage.GetStorage().MountPath, t.srcObjPath, t.dstStorage.GetStorage().MountPath, t.dstDirPath) + return fmt.Sprintf("copy [%s](%s) to [%s](%s)", t.SrcStorageMp, t.SrcObjPath, t.DstStorageMp, t.DstDirPath) } func (t *CopyTask) GetStatus() string { @@ -32,7 +36,17 @@ func (t *CopyTask) GetStatus() string { } func (t *CopyTask) Run() error { - return copyBetween2Storages(t, t.srcStorage, t.dstStorage, t.srcObjPath, t.dstDirPath) + var err error + if t.srcStorage == nil { + t.srcStorage, err = op.GetStorageByMountPath(t.SrcStorageMp) + } + if t.dstStorage == nil { + t.dstStorage, err = op.GetStorageByMountPath(t.DstStorageMp) + } + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + return copyBetween2Storages(t, t.srcStorage, t.dstStorage, t.SrcObjPath, t.DstDirPath) } var CopyTaskManager *tache.Manager[*CopyTask] @@ -79,10 +93,12 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool } // not in the same storage t := &CopyTask{ - srcStorage: srcStorage, - dstStorage: dstStorage, - srcObjPath: srcObjActualPath, - dstDirPath: dstDirActualPath, + srcStorage: srcStorage, + dstStorage: dstStorage, + SrcObjPath: srcObjActualPath, + DstDirPath: dstDirActualPath, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, } CopyTaskManager.Add(t) return t, nil @@ -107,10 +123,12 @@ func copyBetween2Storages(t *CopyTask, srcStorage, dstStorage driver.Driver, src srcObjPath := stdpath.Join(srcObjPath, obj.GetName()) dstObjPath := stdpath.Join(dstDirPath, srcObj.GetName()) CopyTaskManager.Add(&CopyTask{ - srcStorage: srcStorage, - dstStorage: dstStorage, - srcObjPath: srcObjPath, - dstDirPath: dstObjPath, + srcStorage: srcStorage, + dstStorage: dstStorage, + SrcObjPath: srcObjPath, + DstDirPath: dstObjPath, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, }) } t.Status = "src object is dir, added all copy tasks of objs" diff --git a/internal/model/task.go b/internal/model/task.go new file mode 100644 index 00000000000..8a87c5a5062 --- /dev/null +++ b/internal/model/task.go @@ -0,0 +1,6 @@ +package model + +type TaskItem struct { + Key string `json:"key"` + PersistData string `gorm:"type:text" json:"persist_data"` +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index e9bcdc50e52..372c73a6cc7 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -76,6 +76,7 @@ func AddURL(ctx context.Context, args *AddURLArgs) (tache.TaskWithInfo, error) { DstDirPath: args.DstDirPath, TempDir: tempDir, DeletePolicy: deletePolicy, + Toolname: args.Tool, tool: tool, } DownloadTaskManager.Add(t) diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 79a29ef0c9a..c778d93632e 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -14,19 +14,26 @@ import ( type DownloadTask struct { tache.Base - Url string `json:"url"` - DstDirPath string `json:"dst_dir_path"` - TempDir string `json:"temp_dir"` - DeletePolicy DeletePolicy `json:"delete_policy"` - - Status string `json:"status"` - Signal chan int `json:"-"` - GID string `json:"-"` + Url string `json:"url"` + DstDirPath string `json:"dst_dir_path"` + TempDir string `json:"temp_dir"` + DeletePolicy DeletePolicy `json:"delete_policy"` + Toolname string `json:"toolname"` + Status string `json:"-"` + Signal chan int `json:"-"` + GID string `json:"-"` tool Tool callStatusRetried int } func (t *DownloadTask) Run() error { + if t.tool == nil { + tool, err := Tools.Get(t.Toolname) + if err != nil { + return errors.WithMessage(err, "failed get tool") + } + t.tool = tool + } if err := t.tool.Run(t); !errs.IsNotSupportError(err) { if err == nil { return t.Complete() @@ -142,9 +149,10 @@ func (t *DownloadTask) Complete() error { file := files[i] TransferTaskManager.Add(&TransferTask{ file: file, - dstDirPath: t.DstDirPath, - tempDir: t.TempDir, - deletePolicy: t.DeletePolicy, + DstDirPath: t.DstDirPath, + TempDir: t.TempDir, + DeletePolicy: t.DeletePolicy, + FileDir: file.Path, }) } return nil diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 0ef58df5019..3744c7b500f 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -2,6 +2,9 @@ package tool import ( "fmt" + "os" + "path/filepath" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" @@ -9,21 +12,27 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" - "os" - "path/filepath" ) type TransferTask struct { tache.Base + FileDir string `json:"file_dir"` + DstDirPath string `json:"dst_dir_path"` + TempDir string `json:"temp_dir"` + DeletePolicy DeletePolicy `json:"delete_policy"` file File - dstDirPath string - tempDir string - deletePolicy DeletePolicy } func (t *TransferTask) Run() error { // check dstDir again - storage, dstDirActualPath, err := op.GetStorageAndActualPath(t.dstDirPath) + var err error + if (t.file == File{}) { + t.file, err = GetFile(t.FileDir) + if err != nil { + return errors.Wrapf(err, "failed to get file %s", t.FileDir) + } + } + storage, dstDirActualPath, err := op.GetStorageAndActualPath(t.DstDirPath) if err != nil { return errors.WithMessage(err, "failed get storage") } @@ -44,7 +53,7 @@ func (t *TransferTask) Run() error { Mimetype: mimetype, Closers: utils.NewClosers(rc), } - relDir, err := filepath.Rel(t.tempDir, filepath.Dir(t.file.Path)) + relDir, err := filepath.Rel(t.TempDir, filepath.Dir(t.file.Path)) if err != nil { log.Errorf("find relation directory error: %v", err) } @@ -53,7 +62,7 @@ func (t *TransferTask) Run() error { } func (t *TransferTask) GetName() string { - return fmt.Sprintf("transfer %s to [%s]", t.file.Path, t.dstDirPath) + return fmt.Sprintf("transfer %s to [%s]", t.file.Path, t.DstDirPath) } func (t *TransferTask) GetStatus() string { @@ -61,7 +70,7 @@ func (t *TransferTask) GetStatus() string { } func (t *TransferTask) OnSucceeded() { - if t.deletePolicy == DeleteOnUploadSucceed || t.deletePolicy == DeleteAlways { + if t.DeletePolicy == DeleteOnUploadSucceed || t.DeletePolicy == DeleteAlways { err := os.Remove(t.file.Path) if err != nil { log.Errorf("failed to delete file %s, error: %s", t.file.Path, err.Error()) @@ -70,7 +79,7 @@ func (t *TransferTask) OnSucceeded() { } func (t *TransferTask) OnFailed() { - if t.deletePolicy == DeleteOnUploadFailed || t.deletePolicy == DeleteAlways { + if t.DeletePolicy == DeleteOnUploadFailed || t.DeletePolicy == DeleteAlways { err := os.Remove(t.file.Path) if err != nil { log.Errorf("failed to delete file %s, error: %s", t.file.Path, err.Error()) diff --git a/internal/offline_download/tool/util.go b/internal/offline_download/tool/util.go index 4258eff61e0..b2c6ec02bfa 100644 --- a/internal/offline_download/tool/util.go +++ b/internal/offline_download/tool/util.go @@ -26,3 +26,16 @@ func GetFiles(dir string) ([]File, error) { } return files, nil } + +func GetFile(path string) (File, error) { + info, err := os.Stat(path) + if err != nil { + return File{}, err + } + return File{ + Name: info.Name(), + Size: info.Size(), + Path: path, + Modified: info.ModTime(), + }, nil +} From 2d77db6bc2bb8bea0c281144706e99b75310d2bc Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sat, 10 Aug 2024 20:58:10 +0800 Subject: [PATCH 266/659] fix(halalcloud): fix the timeout issue when logging in (#6960) --- drivers/halalcloud/driver.go | 2 +- drivers/halalcloud/util.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/halalcloud/driver.go b/drivers/halalcloud/driver.go index f99b5f6f412..08bb3808bfd 100644 --- a/drivers/halalcloud/driver.go +++ b/drivers/halalcloud/driver.go @@ -147,7 +147,7 @@ func (d *HalalCloud) IsLogin() bool { if err != nil { return false } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() result, err := pbPublicUser.NewPubUserClient(serv.GetGrpcConnection()).Get(ctx, &pbPublicUser.User{ Identity: "", diff --git a/drivers/halalcloud/util.go b/drivers/halalcloud/util.go index be1b9b36551..f3012a8c83c 100644 --- a/drivers/halalcloud/util.go +++ b/drivers/halalcloud/util.go @@ -62,7 +62,7 @@ func (d *HalalCloud) NewAuthServiceWithOauth(options ...HalalOption) (*AuthServi } defer grpcConnection.Close() userClient := pbPublicUser.NewPubUserClient(grpcConnection) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() stateString := uuid.New().String() // queryValues.Add("callback", oauthToken.Callback) From 29165d8e608e52af047c4adb1d4f05584b007a5f Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Sat, 10 Aug 2024 20:59:07 +0800 Subject: [PATCH 267/659] feat(115): add offline download tool (close #6888 in #6954) --- drivers/115/driver.go | 17 ++- go.mod | 2 +- go.sum | 6 +- internal/offline_download/115/client.go | 124 +++++++++++++++++++++ internal/offline_download/all.go | 1 + internal/offline_download/tool/add.go | 9 +- internal/offline_download/tool/download.go | 20 +++- 7 files changed, 166 insertions(+), 13 deletions(-) create mode 100644 internal/offline_download/115/client.go diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 57b6c45f9e7..2a1c8deef47 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -63,7 +63,7 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err := d.WaitLimit(ctx); err != nil { return nil, err } - var userAgent = args.Header.Get("User-Agent") + userAgent := args.Header.Get("User-Agent") downloadInfo, err := d. DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { @@ -179,7 +179,22 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } // 分片上传 return d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID) +} + +func (d *Pan115) OfflineList(ctx context.Context) ([]*driver115.OfflineTask, error) { + resp, err := d.client.ListOfflineTask(0) + if err != nil { + return nil, err + } + return resp.Tasks, nil +} + +func (d *Pan115) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return d.client.AddOfflineTaskURIs(uris, dstDir.GetID()) +} +func (d *Pan115) DeleteOfflineTasks(ctx context.Context, hashes []string, deleteFiles bool) error { + return d.client.DeleteOfflineTasks(hashes, deleteFiles) } var _ driver.Driver = (*Pan115)(nil) diff --git a/go.mod b/go.mod index 318da35057a..a6b042f1897 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/alist-org/alist/v3 go 1.22.4 require ( - github.com/SheltonZhu/115driver v1.0.25 + github.com/SheltonZhu/115driver v1.0.26 github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 diff --git a/go.sum b/go.sum index 9de941c5e85..ecc7e8974a8 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/SheltonZhu/115driver v1.0.25 h1:i101yanLKUwV1Pi7x+vgNOwgz7Hp9JbNmo6BCZ9/4wo= -github.com/SheltonZhu/115driver v1.0.25/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= +github.com/SheltonZhu/115driver v1.0.26 h1:UDUEZffJoQLFYs2nxnyxqvxwSaocxP4LNaOycVY6syU= +github.com/SheltonZhu/115driver v1.0.26/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= @@ -550,8 +550,6 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w= -golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go new file mode 100644 index 00000000000..0ebf38ffced --- /dev/null +++ b/internal/offline_download/115/client.go @@ -0,0 +1,124 @@ +package _115 + +import ( + "context" + "fmt" + + "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" +) + +type Cloud115 struct { + refreshTaskCache bool +} + +func (p *Cloud115) Name() string { + return "115 Cloud" +} + +func (p *Cloud115) Items() []model.SettingItem { + return nil +} + +func (p *Cloud115) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (p *Cloud115) Init() (string, error) { + p.refreshTaskCache = false + return "ok", nil +} + +func (p *Cloud115) IsReady() bool { + return true +} + +func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { + // 添加新任务刷新缓存 + p.refreshTaskCache = true + // args.TempDir 已经被修改为了 DstDirPath + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver115, ok := storage.(*_115.Pan115) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") + } + + ctx := context.Background() + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + + hashs, err := driver115.OfflineDownload(ctx, []string{args.Url}, parentDir) + if err != nil || len(hashs) < 1 { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + + return hashs[0], nil +} + +func (p *Cloud115) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + if err != nil { + return err + } + driver115, ok := storage.(*_115.Pan115) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") + } + + ctx := context.Background() + if err := driver115.DeleteOfflineTasks(ctx, []string{task.GID}, false); err != nil { + return err + } + return nil +} + +func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + if err != nil { + return nil, err + } + driver115, ok := storage.(*_115.Pan115) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") + } + + tasks, err := driver115.OfflineList(context.Background()) + if err != nil { + return nil, err + } + + s := &tool.Status{ + Progress: 0, + NewGID: "", + Completed: false, + Status: "the task has been deleted", + Err: nil, + } + for _, t := range tasks { + if t.InfoHash == task.GID { + s.Progress = t.Percent + s.Status = t.GetStatus() + s.Completed = t.IsDone() + if t.IsFailed() { + s.Err = fmt.Errorf(t.GetStatus()) + } + return s, nil + } + } + s.Err = fmt.Errorf("the task has been deleted") + return nil, nil +} + +var _ tool.Tool = (*Cloud115)(nil) + +func init() { + tool.Tools.Add(&Cloud115{}) +} diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 67869dee8d2..ee80b5a0b8f 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -1,6 +1,7 @@ package offline_download import ( + _ "github.com/alist-org/alist/v3/internal/offline_download/115" _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" _ "github.com/alist-org/alist/v3/internal/offline_download/http" _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 372c73a6cc7..c7c5c781f71 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -66,11 +66,18 @@ func AddURL(ctx context.Context, args *AddURLArgs) (tache.TaskWithInfo, error) { uid := uuid.NewString() tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) deletePolicy := args.DeletePolicy - if args.Tool == "pikpak" { + + switch args.Tool { + case "115 Cloud": + tempDir = args.DstDirPath + // 防止将下载好的文件删除 + deletePolicy = DeleteNever + case "pikpak": tempDir = args.DstDirPath // 防止将下载好的文件删除 deletePolicy = DeleteNever } + t := &DownloadTask{ Url: args.URL, DstDirPath: args.DstDirPath, diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index c778d93632e..4cc86a26124 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -54,9 +54,7 @@ func (t *DownloadTask) Run() error { return err } t.GID = gid - var ( - ok bool - ) + var ok bool outer: for { select { @@ -81,6 +79,15 @@ outer: if t.tool.Name() == "pikpak" { return nil } + if t.tool.Name() == "115 Cloud" { + // hack for 115 + <-time.After(time.Second * 1) + err := t.tool.Remove(t) + if err != nil { + log.Errorln(err.Error()) + } + return nil + } t.Status = "offline download completed, maybe transferring" // hack for qBittorrent if t.tool.Name() == "qBittorrent" { @@ -136,6 +143,9 @@ func (t *DownloadTask) Complete() error { if t.tool.Name() == "pikpak" { return nil } + if t.tool.Name() == "115 Cloud" { + return nil + } if getFileser, ok := t.tool.(GetFileser); ok { files = getFileser.GetFiles(t) } else { @@ -166,6 +176,4 @@ func (t *DownloadTask) GetStatus() string { return t.Status } -var ( - DownloadTaskManager *tache.Manager[*DownloadTask] -) +var DownloadTaskManager *tache.Manager[*DownloadTask] From 979d0cfeeed8990319bf4d6b3dd77963030fee4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8A=B1=E7=94=9F=E7=93=9C=E5=AD=90=E5=A4=A7=E6=9D=8F?= =?UTF-8?q?=E4=BB=81?= Date: Sat, 10 Aug 2024 20:59:49 +0800 Subject: [PATCH 268/659] fix(chaoxing): upload to ChaoxingxingGroupCloud failed (#6953) change the data type on deserializing json --- drivers/chaoxing/types.go | 108 +++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go index c3074bbb721..71a59e15be6 100644 --- a/drivers/chaoxing/types.go +++ b/drivers/chaoxing/types.go @@ -191,33 +191,33 @@ type UploadFileDataRsp struct { Resid int64 `json:"resid"` Puid int `json:"puid"` Data struct { - DisableOpt bool `json:"disableOpt"` - Resid int64 `json:"resid"` - Crc string `json:"crc"` - Puid int `json:"puid"` - Isfile bool `json:"isfile"` - Pantype string `json:"pantype"` - Size int `json:"size"` - Name string `json:"name"` - ObjectID string `json:"objectId"` - Restype string `json:"restype"` - UploadDate time.Time `json:"uploadDate"` - ModifyDate time.Time `json:"modifyDate"` - UploadDateFormat string `json:"uploadDateFormat"` - Residstr string `json:"residstr"` - Suffix string `json:"suffix"` - Preview string `json:"preview"` - Thumbnail string `json:"thumbnail"` - Creator int `json:"creator"` - Duration int `json:"duration"` - IsImg bool `json:"isImg"` - PreviewURL string `json:"previewUrl"` - Filetype string `json:"filetype"` - Filepath string `json:"filepath"` - Sort int `json:"sort"` - Topsort int `json:"topsort"` - ResTypeValue int `json:"resTypeValue"` - Extinfo string `json:"extinfo"` + DisableOpt bool `json:"disableOpt"` + Resid int64 `json:"resid"` + Crc string `json:"crc"` + Puid int `json:"puid"` + Isfile bool `json:"isfile"` + Pantype string `json:"pantype"` + Size int `json:"size"` + Name string `json:"name"` + ObjectID string `json:"objectId"` + Restype string `json:"restype"` + UploadDate int64 `json:"uploadDate"` + ModifyDate int64 `json:"modifyDate"` + UploadDateFormat string `json:"uploadDateFormat"` + Residstr string `json:"residstr"` + Suffix string `json:"suffix"` + Preview string `json:"preview"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + Duration int `json:"duration"` + IsImg bool `json:"isImg"` + PreviewURL string `json:"previewUrl"` + Filetype string `json:"filetype"` + Filepath string `json:"filepath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + ResTypeValue int `json:"resTypeValue"` + Extinfo string `json:"extinfo"` } `json:"data"` } @@ -225,33 +225,33 @@ type UploadDoneParam struct { Cataid string `json:"cataid"` Key string `json:"key"` Param struct { - DisableOpt bool `json:"disableOpt"` - Resid int64 `json:"resid"` - Crc string `json:"crc"` - Puid int `json:"puid"` - Isfile bool `json:"isfile"` - Pantype string `json:"pantype"` - Size int `json:"size"` - Name string `json:"name"` - ObjectID string `json:"objectId"` - Restype string `json:"restype"` - UploadDate time.Time `json:"uploadDate"` - ModifyDate time.Time `json:"modifyDate"` - UploadDateFormat string `json:"uploadDateFormat"` - Residstr string `json:"residstr"` - Suffix string `json:"suffix"` - Preview string `json:"preview"` - Thumbnail string `json:"thumbnail"` - Creator int `json:"creator"` - Duration int `json:"duration"` - IsImg bool `json:"isImg"` - PreviewURL string `json:"previewUrl"` - Filetype string `json:"filetype"` - Filepath string `json:"filepath"` - Sort int `json:"sort"` - Topsort int `json:"topsort"` - ResTypeValue int `json:"resTypeValue"` - Extinfo string `json:"extinfo"` + DisableOpt bool `json:"disableOpt"` + Resid int64 `json:"resid"` + Crc string `json:"crc"` + Puid int `json:"puid"` + Isfile bool `json:"isfile"` + Pantype string `json:"pantype"` + Size int `json:"size"` + Name string `json:"name"` + ObjectID string `json:"objectId"` + Restype string `json:"restype"` + UploadDate int64 `json:"uploadDate"` + ModifyDate int64 `json:"modifyDate"` + UploadDateFormat string `json:"uploadDateFormat"` + Residstr string `json:"residstr"` + Suffix string `json:"suffix"` + Preview string `json:"preview"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + Duration int `json:"duration"` + IsImg bool `json:"isImg"` + PreviewURL string `json:"previewUrl"` + Filetype string `json:"filetype"` + Filepath string `json:"filepath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + ResTypeValue int `json:"resTypeValue"` + Extinfo string `json:"extinfo"` } `json:"param"` } From 62ed169a397cee3ae8278bf76b62ec01c2b7f0ba Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sat, 10 Aug 2024 21:00:43 +0800 Subject: [PATCH 269/659] feat: add support for quark tv driver and uc tv driver (#6959) --- drivers/all.go | 1 + drivers/quark_uc_tv/driver.go | 174 ++++++++++++++++++++++++++++ drivers/quark_uc_tv/meta.go | 67 +++++++++++ drivers/quark_uc_tv/types.go | 102 ++++++++++++++++ drivers/quark_uc_tv/util.go | 211 ++++++++++++++++++++++++++++++++++ 5 files changed, 555 insertions(+) create mode 100644 drivers/quark_uc_tv/driver.go create mode 100644 drivers/quark_uc_tv/meta.go create mode 100644 drivers/quark_uc_tv/types.go create mode 100644 drivers/quark_uc_tv/util.go diff --git a/drivers/all.go b/drivers/all.go index b976f92f2a6..1f015ef7d61 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -41,6 +41,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" _ "github.com/alist-org/alist/v3/drivers/quark_uc" + _ "github.com/alist-org/alist/v3/drivers/quark_uc_tv" _ "github.com/alist-org/alist/v3/drivers/quqi" _ "github.com/alist-org/alist/v3/drivers/s3" _ "github.com/alist-org/alist/v3/drivers/seafile" diff --git a/drivers/quark_uc_tv/driver.go b/drivers/quark_uc_tv/driver.go new file mode 100644 index 00000000000..ff7ccf20f7a --- /dev/null +++ b/drivers/quark_uc_tv/driver.go @@ -0,0 +1,174 @@ +package quark_uc_tv + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type QuarkUCTV struct { + *QuarkUCTVCommon + model.Storage + Addition + config driver.Config + conf Conf +} + +func (d *QuarkUCTV) Config() driver.Config { + return d.config +} + +func (d *QuarkUCTV) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *QuarkUCTV) Init(ctx context.Context) error { + + if d.Addition.DeviceID == "" { + d.Addition.DeviceID = utils.GetMD5EncodeStr(time.Now().String()) + } + op.MustSaveDriverStorage(d) + + if d.QuarkUCTVCommon == nil { + d.QuarkUCTVCommon = &QuarkUCTVCommon{ + AccessToken: "", + } + } + ctx1, cancelFunc := context.WithTimeout(ctx, 5*time.Second) + defer cancelFunc() + if d.Addition.RefreshToken == "" { + if d.Addition.QueryToken == "" { + qrData, err := d.getLoginCode(ctx1) + if err != nil { + return err + } + // 展示二维码 + qrTemplate := ` + + ` + qrPage := fmt.Sprintf(qrTemplate, qrData) + return fmt.Errorf("need verify: \n%s", qrPage) + } else { + // 通过query token获取code -> refresh token + code, err := d.getCode(ctx1) + if err != nil { + return err + } + // 通过code获取refresh token + err = d.getRefreshTokenByTV(ctx1, code, false) + if err != nil { + return err + } + } + } + // 通过refresh token获取access token + if d.QuarkUCTVCommon.AccessToken == "" { + err := d.getRefreshTokenByTV(ctx1, d.Addition.RefreshToken, true) + if err != nil { + return err + } + } + + // 验证 access token 是否有效 + _, err := d.isLogin(ctx1) + if err != nil { + return err + } + return nil +} + +func (d *QuarkUCTV) Drop(ctx context.Context) error { + return nil +} + +func (d *QuarkUCTV) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]model.Obj, 0) + pageIndex := int64(0) + pageSize := int64(100) + for { + var filesData FilesData + _, err := d.request(ctx, "/file", "GET", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "method": "list", + "parent_fid": dir.GetID(), + "order_by": "3", + "desc": "1", + "category": "", + "source": "", + "ex_source": "", + "list_all": "0", + "page_size": strconv.FormatInt(pageSize, 10), + "page_index": strconv.FormatInt(pageIndex, 10), + }) + }, &filesData) + if err != nil { + return nil, err + } + for i := range filesData.Data.Files { + files = append(files, &filesData.Data.Files[i]) + } + if pageIndex*pageSize >= filesData.Data.TotalCount { + break + } else { + pageIndex++ + } + } + return files, nil +} + +func (d *QuarkUCTV) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + files := &model.Link{} + var fileLink FileLink + _, err := d.request(ctx, "/file", "GET", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "method": "download", + "group_by": "source", + "fid": file.GetID(), + "resolution": "low,normal,high,super,2k,4k", + "support": "dolby_vision", + }) + }, &fileLink) + if err != nil { + return nil, err + } + files.URL = fileLink.Data.DownloadURL + return files, nil +} + +func (d *QuarkUCTV) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *QuarkUCTV) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *QuarkUCTV) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *QuarkUCTV) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *QuarkUCTV) Remove(ctx context.Context, obj model.Obj) error { + return errs.NotImplement +} + +func (d *QuarkUCTV) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return nil, errs.NotImplement +} + +type QuarkUCTVCommon struct { + AccessToken string +} + +var _ driver.Driver = (*QuarkUCTV)(nil) diff --git a/drivers/quark_uc_tv/meta.go b/drivers/quark_uc_tv/meta.go new file mode 100644 index 00000000000..cf7e478566e --- /dev/null +++ b/drivers/quark_uc_tv/meta.go @@ -0,0 +1,67 @@ +package quark_uc_tv + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + RefreshToken string `json:"refresh_token" required:"false" default:""` + // 必要且影响登录,由签名决定 + DeviceID string `json:"device_id" required:"false" default:""` + // 登陆所用的数据 无需手动填写 + QueryToken string `json:"query_token" required:"false" default:"" help:"don't edit'"` +} + +type Conf struct { + api string + clientID string + signKey string + appVer string + channel string + codeApi string +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &QuarkUCTV{ + config: driver.Config{ + Name: "QuarkTV", + OnlyLocal: false, + DefaultRoot: "0", + NoOverwriteUpload: true, + NoUpload: true, + }, + conf: Conf{ + api: "https://open-api-drive.quark.cn", + clientID: "d3194e61504e493eb6222857bccfed94", + signKey: "kw2dvtd7p4t3pjl2d9ed9yc8yej8kw2d", + appVer: "1.5.6", + channel: "CP", + codeApi: "http://api.extscreen.com/quarkdrive", + }, + } + }) + op.RegisterDriver(func() driver.Driver { + return &QuarkUCTV{ + config: driver.Config{ + Name: "UCTV", + OnlyLocal: false, + DefaultRoot: "0", + NoOverwriteUpload: true, + NoUpload: true, + }, + conf: Conf{ + api: "https://open-api-drive.uc.cn", + clientID: "5acf882d27b74502b7040b0c65519aa7", + signKey: "l3srvtd7p42l0d0x1u8d7yc8ye9kki4d", + appVer: "1.6.5", + channel: "UCTVOFFICIALWEB", + codeApi: "http://api.extscreen.com/ucdrive", + }, + } + }) +} diff --git a/drivers/quark_uc_tv/types.go b/drivers/quark_uc_tv/types.go new file mode 100644 index 00000000000..fb35b8b2d6d --- /dev/null +++ b/drivers/quark_uc_tv/types.go @@ -0,0 +1,102 @@ +package quark_uc_tv + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "time" +) + +type Resp struct { + CommonRsp + Errno int `json:"errno"` + ErrorInfo string `json:"error_info"` +} + +type CommonRsp struct { + Status int `json:"status"` + ReqID string `json:"req_id"` +} + +type RefreshTokenAuthResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Status int `json:"status"` + Errno int `json:"errno"` + ErrorInfo string `json:"error_info"` + ReqID string `json:"req_id"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` + Scope string `json:"scope"` + } `json:"data"` +} +type Files struct { + Fid string `json:"fid"` + ParentFid string `json:"parent_fid"` + Category int `json:"category"` + Filename string `json:"filename"` + Size int64 `json:"size"` + FileType string `json:"file_type"` + SubItems int `json:"sub_items,omitempty"` + Isdir int `json:"isdir"` + Duration int `json:"duration"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + IsBackup int `json:"is_backup"` + ThumbnailURL string `json:"thumbnail_url,omitempty"` +} + +func (f *Files) GetSize() int64 { + return f.Size +} + +func (f *Files) GetName() string { + return f.Filename +} + +func (f *Files) ModTime() time.Time { + //return time.Unix(f.UpdatedAt, 0) + return time.Unix(0, f.UpdatedAt*int64(time.Millisecond)) +} + +func (f *Files) CreateTime() time.Time { + //return time.Unix(f.CreatedAt, 0) + return time.Unix(0, f.CreatedAt*int64(time.Millisecond)) +} + +func (f *Files) IsDir() bool { + return f.Isdir == 1 +} + +func (f *Files) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f *Files) GetID() string { + return f.Fid +} + +func (f *Files) GetPath() string { + return "" +} + +var _ model.Obj = (*Files)(nil) + +type FilesData struct { + CommonRsp + Data struct { + TotalCount int64 `json:"total_count"` + Files []Files `json:"files"` + } `json:"data"` +} + +type FileLink struct { + CommonRsp + Data struct { + Fid string `json:"fid"` + FileName string `json:"file_name"` + Size int64 `json:"size"` + DownloadURL string `json:"download_url"` + } `json:"data"` +} diff --git a/drivers/quark_uc_tv/util.go b/drivers/quark_uc_tv/util.go new file mode 100644 index 00000000000..fefbb0361fb --- /dev/null +++ b/drivers/quark_uc_tv/util.go @@ -0,0 +1,211 @@ +package quark_uc_tv + +import ( + "context" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "errors" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "net/http" + "strconv" + "time" +) + +const ( + UserAgent = "Mozilla/5.0 (Linux; U; Android 13; zh-cn; M2004J7AC Build/UKQ1.231108.001) AppleWebKit/533.1 (KHTML, like Gecko) Mobile Safari/533.1" + DeviceBrand = "Xiaomi" + Platform = "tv" + DeviceName = "M2004J7AC" + DeviceModel = "M2004J7AC" + BuildDevice = "M2004J7AC" + BuildProduct = "M2004J7AC" + DeviceGpu = "Adreno (TM) 550" + ActivityRect = "{}" +) + +func (d *QuarkUCTV) request(ctx context.Context, pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + u := d.conf.api + pathname + tm, token, reqID := d.generateReqSign(method, pathname, d.conf.signKey) + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "User-Agent": UserAgent, + "x-pan-tm": tm, + "x-pan-token": token, + "x-pan-client-id": d.conf.clientID, + }) + req.SetQueryParams(map[string]string{ + "req_id": reqID, + "access_token": d.QuarkUCTVCommon.AccessToken, + "app_ver": d.conf.appVer, + "device_id": d.Addition.DeviceID, + "device_brand": DeviceBrand, + "platform": Platform, + "device_name": DeviceName, + "device_model": DeviceModel, + "build_device": BuildDevice, + "build_product": BuildProduct, + "device_gpu": DeviceGpu, + "activity_rect": ActivityRect, + "channel": d.conf.channel, + }) + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Resp + req.SetError(&e) + res, err := req.Execute(method, u) + if err != nil { + return nil, err + } + // 判断 是否需要 刷新 access_token + if e.Status == -1 && e.Errno == 10001 { + // token 过期 + err = d.getRefreshTokenByTV(ctx, d.Addition.RefreshToken, true) + if err != nil { + return nil, err + } + ctx1, cancelFunc := context.WithTimeout(ctx, 10*time.Second) + defer cancelFunc() + return d.request(ctx1, pathname, method, callback, resp) + } + + if e.Status >= 400 || e.Errno != 0 { + return nil, errors.New(e.ErrorInfo) + } + return res.Body(), nil +} + +func (d *QuarkUCTV) getLoginCode(ctx context.Context) (string, error) { + // 获取登录二维码 + pathname := "/oauth/authorize" + var resp struct { + CommonRsp + QrData string `json:"qr_data"` + QueryToken string `json:"query_token"` + } + _, err := d.request(ctx, pathname, "GET", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "auth_type": "code", + "client_id": d.conf.clientID, + "scope": "netdisk", + "qrcode": "1", + "qr_width": "460", + "qr_height": "460", + }) + }, &resp) + if err != nil { + return "", err + } + // 保存query_token 用于后续登录 + if resp.QueryToken != "" { + d.Addition.QueryToken = resp.QueryToken + op.MustSaveDriverStorage(d) + } + return resp.QrData, nil +} + +func (d *QuarkUCTV) getCode(ctx context.Context) (string, error) { + // 通过query token获取code + pathname := "/oauth/code" + var resp struct { + CommonRsp + Code string `json:"code"` + } + _, err := d.request(ctx, pathname, "GET", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "client_id": d.conf.clientID, + "scope": "netdisk", + "query_token": d.Addition.QueryToken, + }) + }, &resp) + if err != nil { + return "", err + } + return resp.Code, nil +} + +func (d *QuarkUCTV) getRefreshTokenByTV(ctx context.Context, code string, isRefresh bool) error { + pathname := "/token" + _, _, reqID := d.generateReqSign("POST", pathname, d.conf.signKey) + u := d.conf.codeApi + pathname + var resp RefreshTokenAuthResp + body := map[string]string{ + "req_id": reqID, + "app_ver": d.conf.appVer, + "device_id": d.Addition.DeviceID, + "device_brand": DeviceBrand, + "platform": Platform, + "device_name": DeviceName, + "device_model": DeviceModel, + "build_device": BuildDevice, + "build_product": BuildProduct, + "device_gpu": DeviceGpu, + "activity_rect": ActivityRect, + "channel": d.conf.channel, + } + if isRefresh { + body["refresh_token"] = code + } else { + body["code"] = code + } + + _, err := base.RestyClient.R(). + SetHeader("Content-Type", "application/json"). + SetBody(body). + SetResult(&resp). + SetContext(ctx). + Post(u) + if err != nil { + return err + } + if resp.Code != 200 { + return errors.New(resp.Message) + } + if resp.Data.RefreshToken != "" { + d.Addition.RefreshToken = resp.Data.RefreshToken + op.MustSaveDriverStorage(d) + d.QuarkUCTVCommon.AccessToken = resp.Data.AccessToken + } else { + return errors.New("refresh token is empty") + } + return nil +} + +func (d *QuarkUCTV) isLogin(ctx context.Context) (bool, error) { + _, err := d.request(ctx, "/user", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "method": "user_info", + }) + }, nil) + return err == nil, err +} + +func (d *QuarkUCTV) generateReqSign(method string, pathname string, key string) (string, string, string) { + //timestamp 13位时间戳 + timestamp := strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10) + deviceID := d.Addition.DeviceID + if deviceID == "" { + deviceID = utils.GetMD5EncodeStr(timestamp) + d.Addition.DeviceID = deviceID + op.MustSaveDriverStorage(d) + } + // 生成req_id + reqID := md5.Sum([]byte(deviceID + timestamp)) + reqIDHex := hex.EncodeToString(reqID[:]) + + // 生成x_pan_token + tokenData := method + "&" + pathname + "&" + timestamp + "&" + key + xPanToken := sha256.Sum256([]byte(tokenData)) + xPanTokenHex := hex.EncodeToString(xPanToken[:]) + + return timestamp, xPanTokenHex, reqIDHex +} From d3bc8993ee8aad40c0bf7b9a2099b4a9e993482c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 21:01:05 +0800 Subject: [PATCH 270/659] fix(deps): update module github.com/dlclark/regexp2 to v1.11.4 (#6958) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a6b042f1897..adefae4ab3a 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/deckarep/golang-set/v2 v2.6.0 github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 github.com/disintegration/imaging v1.6.2 - github.com/dlclark/regexp2 v1.11.2 + github.com/dlclark/regexp2 v1.11.4 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 diff --git a/go.sum b/go.sum index ecc7e8974a8..2443d0b9b49 100644 --- a/go.sum +++ b/go.sum @@ -140,6 +140,8 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dlclark/regexp2 v1.11.2 h1:/u628IuisSTwri5/UKloiIsH8+qF2Pu7xEQX+yIKg68= github.com/dlclark/regexp2 v1.11.2/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= From 8032d0afb6fd5aa2638d3dc9e178771f8376f746 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 21:01:41 +0800 Subject: [PATCH 271/659] fix(deps): update module golang.org/x/oauth2 to v0.22.0 (#6943) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index adefae4ab3a..e5fd55dd84f 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 golang.org/x/image v0.18.0 golang.org/x/net v0.27.0 - golang.org/x/oauth2 v0.21.0 + golang.org/x/oauth2 v0.22.0 golang.org/x/time v0.5.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 diff --git a/go.sum b/go.sum index 2443d0b9b49..8db11f1fcab 100644 --- a/go.sum +++ b/go.sum @@ -580,6 +580,8 @@ golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 34e34ef564b060232b0412af27b7e07fa9e6948b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:38:32 +0800 Subject: [PATCH 272/659] fix(deps): update module golang.org/x/time to v0.6.0 (#6944) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e5fd55dd84f..f23d32f7cc1 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( golang.org/x/image v0.18.0 golang.org/x/net v0.27.0 golang.org/x/oauth2 v0.22.0 - golang.org/x/time v0.5.0 + golang.org/x/time v0.6.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 diff --git a/go.sum b/go.sum index 8db11f1fcab..135739a3907 100644 --- a/go.sum +++ b/go.sum @@ -646,6 +646,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From d596ef5c381044d105f13aba8b0185c0eeadb96a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:38:56 +0800 Subject: [PATCH 273/659] fix(deps): update module github.com/blevesearch/bleve/v2 to v2.4.2 (#6892) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index f23d32f7cc1..f5185c41af4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible github.com/aws/aws-sdk-go v1.54.19 - github.com/blevesearch/bleve/v2 v2.4.1 + github.com/blevesearch/bleve/v2 v2.4.2 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.26.6 @@ -74,8 +74,8 @@ require ( require ( github.com/BurntSushi/toml v0.3.1 // indirect - github.com/blevesearch/go-faiss v1.0.19 // indirect - github.com/blevesearch/zapx/v16 v16.1.4 // indirect + github.com/blevesearch/go-faiss v1.0.20 // indirect + github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/charmbracelet/x/ansi v0.1.4 // indirect github.com/charmbracelet/x/input v0.1.0 // indirect @@ -102,12 +102,12 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/blevesearch/bleve_index_api v1.1.9 // indirect + github.com/blevesearch/bleve_index_api v1.1.10 // indirect github.com/blevesearch/geo v0.1.20 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.2.14 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.2.15 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect diff --git a/go.sum b/go.sum index 135739a3907..906eff40912 100644 --- a/go.sum +++ b/go.sum @@ -48,12 +48,18 @@ github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/blevesearch/bleve/v2 v2.4.1 h1:8QWqsifq693mN3h6cSigKqkKUsUfv5hu0FDgz/4bFuA= github.com/blevesearch/bleve/v2 v2.4.1/go.mod h1:Ezmvsouspi+uVwnDzjIsCeUIT0WuBKlicP5JZnExWzo= +github.com/blevesearch/bleve/v2 v2.4.2 h1:NooYP1mb3c0StkiY9/xviiq2LGSaE8BQBCc/pirMx0U= +github.com/blevesearch/bleve/v2 v2.4.2/go.mod h1:ATNKj7Yl2oJv/lGuF4kx39bST2dveX6w0th2FFYLkc8= github.com/blevesearch/bleve_index_api v1.1.9 h1:Cpq0Lp3As0Gfk3+PmcoNDRKeI50C5yuFNpj0YlN/bOE= github.com/blevesearch/bleve_index_api v1.1.9/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= +github.com/blevesearch/bleve_index_api v1.1.10 h1:PDLFhVjrjQWr6jCuU7TwlmByQVCSEURADHdCqVS9+g0= +github.com/blevesearch/bleve_index_api v1.1.10/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= github.com/blevesearch/go-faiss v1.0.19 h1:UKoP8hS7DVsVSRRloNJb4qPfe2UQ99pP4D3oXd23g2A= github.com/blevesearch/go-faiss v1.0.19/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= +github.com/blevesearch/go-faiss v1.0.20 h1:AIkdTQFWuZ5LQmKQSebgMR4RynGNw8ZseJXaan5kvtI= +github.com/blevesearch/go-faiss v1.0.20/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= @@ -62,6 +68,8 @@ github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCD github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= github.com/blevesearch/scorch_segment_api/v2 v2.2.14 h1:fgMLMpGWR7u2TdRm7XSZVWhPvMAcdYHh25Lq1fQ6Fjo= github.com/blevesearch/scorch_segment_api/v2 v2.2.14/go.mod h1:B7+a7vfpY4NsjuTkpv/eY7RZ91Xr90VaJzT2t7upZN8= +github.com/blevesearch/scorch_segment_api/v2 v2.2.15 h1:prV17iU/o+A8FiZi9MXmqbagd8I0bCqM7OKUYPbnb5Y= +github.com/blevesearch/scorch_segment_api/v2 v2.2.15/go.mod h1:db0cmP03bPNadXrCDuVkKLV6ywFSiRgPFT1YVrestBc= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= @@ -82,6 +90,8 @@ github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wy github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= github.com/blevesearch/zapx/v16 v16.1.4 h1:TBQfG77g2UUXwfjOVcEtB9pXkg6JBmGXkeZKI67+TiA= github.com/blevesearch/zapx/v16 v16.1.4/go.mod h1:+Q+Z89Iv7ewhdX2jyE6Qs/RUnN4tZuokaQ0xvTaFmx8= +github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi/AUHjU= +github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= From a7efa3a676015a1ec49db370267e1a0d4b3505f2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:39:13 +0800 Subject: [PATCH 274/659] fix(deps): update golang.org/x/exp digest to 0cdaa3a (#6977) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 16 ++++++++-------- go.sum | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index f5185c41af4..ba25b523cbf 100644 --- a/go.mod +++ b/go.mod @@ -58,10 +58,10 @@ require ( github.com/xhofe/tache v0.1.2 github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.25.0 - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 + golang.org/x/crypto v0.26.0 + golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa golang.org/x/image v0.18.0 - golang.org/x/net v0.27.0 + golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 golang.org/x/time v0.6.0 google.golang.org/appengine v1.6.8 @@ -219,11 +219,11 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/term v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect - golang.org/x/tools v0.23.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.23.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect + golang.org/x/tools v0.24.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/grpc v1.65.0 diff --git a/go.sum b/go.sum index 906eff40912..30a88126643 100644 --- a/go.sum +++ b/go.sum @@ -562,8 +562,12 @@ golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDf golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= @@ -588,6 +592,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= @@ -599,6 +605,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -630,6 +638,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -639,6 +649,8 @@ golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -652,6 +664,8 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= @@ -668,6 +682,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From a26185fe055efb9bae80ffc1e8a6fe2cb0733040 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:40:05 +0800 Subject: [PATCH 275/659] fix(deps): update github.com/xhofe/go-cache digest to b1a7192 (#6939) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ba25b523cbf..6cd61aeae31 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 require ( github.com/SheltonZhu/115driver v1.0.26 - github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a + github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 diff --git a/go.sum b/go.sum index 30a88126643..177ca4eca68 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4 github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= +github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= +github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaVLk4gI/Co0paF92FmxSR6U1zY= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= From 285125d06add8408b7806aaeda230f8ecd655b33 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:40:26 +0800 Subject: [PATCH 276/659] fix(deps): update module github.com/larksuite/oapi-sdk-go/v3 to v3.3.1 (#6978) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6cd61aeae31..9a5185f372b 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 - github.com/larksuite/oapi-sdk-go/v3 v3.3.0 + github.com/larksuite/oapi-sdk-go/v3 v3.3.1 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.0 github.com/minio/sio v0.4.0 diff --git a/go.sum b/go.sum index 177ca4eca68..a4e8eceffe7 100644 --- a/go.sum +++ b/go.sum @@ -320,6 +320,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.3.0 h1:aCtFUiYgoRUW+aaWzVYw8jSzMe4A71rPEIn1DyHcNrY= github.com/larksuite/oapi-sdk-go/v3 v3.3.0/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/larksuite/oapi-sdk-go/v3 v3.3.1 h1:DLQQEgHUAGZB6RVlceB1f6A94O206exxW2RIMH+gMUc= +github.com/larksuite/oapi-sdk-go/v3 v3.3.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= From 8f3c5b158753ea34aafb885c9841b4095bd82d66 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 11 Aug 2024 11:40:47 +0800 Subject: [PATCH 277/659] fix(deps): update module github.com/meilisearch/meilisearch-go to v0.27.2 (#6907) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 9a5185f372b..4a32d700ffe 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/larksuite/oapi-sdk-go/v3 v3.3.1 github.com/maruel/natural v1.1.1 - github.com/meilisearch/meilisearch-go v0.27.0 + github.com/meilisearch/meilisearch-go v0.27.2 github.com/minio/sio v0.4.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.2 diff --git a/go.sum b/go.sum index a4e8eceffe7..37ecb01789d 100644 --- a/go.sum +++ b/go.sum @@ -354,6 +354,8 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.27.0 h1:lDFq8WzbsZCtt3/byr7GFqfOygWF5iy9TtDgzJo0Ds8= github.com/meilisearch/meilisearch-go v0.27.0/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= +github.com/meilisearch/meilisearch-go v0.27.2 h1:3G21dJ5i208shnLPDsIEZ0L0Geg/5oeXABFV7nlK94k= +github.com/meilisearch/meilisearch-go v0.27.2/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= From 95607991755ddd5dfc741a780f157467e0db3302 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Wed, 14 Aug 2024 19:33:15 +0800 Subject: [PATCH 278/659] fix(189pc): InvalidSessionKey (#6994 close #6992) --- drivers/189pc/utils.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index a000a84e005..f5a44455d2e 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -114,17 +114,19 @@ func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, para if err = y.refreshSession(); err != nil { return nil, err } - return y.request(url, method, callback, params, resp) + return y.request(url, method, callback, params, resp, isFamily...) + } + + // if erron.ErrorCode == "InvalidSessionKey" || erron.Code == "InvalidSessionKey" { + if strings.Contains(res.String(), "InvalidSessionKey") { + if err = y.refreshSession(); err != nil { + return nil, err + } + return y.request(url, method, callback, params, resp, isFamily...) } // 处理错误 if erron.HasError() { - if erron.ErrorCode == "InvalidSessionKey" { - if err = y.refreshSession(); err != nil { - return nil, err - } - return y.request(url, method, callback, params, resp) - } return nil, &erron } return res.Body(), nil From 3dc250cc37f8aee1fc9df9929de34f732ee4402d Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Wed, 14 Aug 2024 19:34:11 +0800 Subject: [PATCH 279/659] feat(115): update qrcode source list (#6996) * remove mac, linux, window (disabled) * add alipaymini, wechatmini, qandroid --- drivers/115/meta.go | 8 ++++---- drivers/115/util.go | 22 +++++++++++--------- drivers/115_share/meta.go | 2 +- go.mod | 2 +- go.sum | 43 ++------------------------------------- 5 files changed, 20 insertions(+), 57 deletions(-) diff --git a/drivers/115/meta.go b/drivers/115/meta.go index 5791f1bd140..38c1742a741 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -8,7 +8,7 @@ import ( type Addition struct { Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` - QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,linux,mac,windows,tv" default:"linux" help:"select the QR code device, default linux"` + QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` driver.RootID @@ -17,9 +17,9 @@ type Addition struct { var config = driver.Config{ Name: "115 Cloud", DefaultRoot: "0", - //OnlyProxy: true, - //OnlyLocal: true, - //NoOverwriteUpload: true, + // OnlyProxy: true, + // OnlyLocal: true, + // NoOverwriteUpload: true, } func init() { diff --git a/drivers/115/util.go b/drivers/115/util.go index cb28fff43fe..d88a9ce6381 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -38,17 +38,17 @@ func (d *Pan115) login() error { } d.client = driver115.New(opts...) cr := &driver115.Credential{} - if d.Addition.QRCodeToken != "" { + if d.QRCodeToken != "" { s := &driver115.QRCodeSession{ - UID: d.Addition.QRCodeToken, + UID: d.QRCodeToken, } if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { return errors.Wrap(err, "failed to login by qrcode") } - d.Addition.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s", cr.UID, cr.CID, cr.SEID) - d.Addition.QRCodeToken = "" - } else if d.Addition.Cookie != "" { - if err = cr.FromCookie(d.Addition.Cookie); err != nil { + d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s", cr.UID, cr.CID, cr.SEID) + d.QRCodeToken = "" + } else if d.Cookie != "" { + if err = cr.FromCookie(d.Cookie); err != nil { return errors.Wrap(err, "failed to login by cookies") } d.client.ImportCredential(cr) @@ -370,11 +370,13 @@ LOOP: } return d.checkUploadStatus(dirID, params.SHA1) } + func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { for _, chunk := range chunks { ch <- chunk } } + func (d *Pan115) checkUploadStatus(dirID, sha1 string) error { // 验证上传是否成功 req := d.client.NewRequest().ForceContentType("application/json;charset=UTF-8") @@ -431,8 +433,8 @@ func SplitFileByPartNum(fileSize int64, chunkNum int) ([]oss.FileChunk, error) { } var chunks []oss.FileChunk - var chunk = oss.FileChunk{} - var chunkN = (int64)(chunkNum) + chunk := oss.FileChunk{} + chunkN := (int64)(chunkNum) for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * (fileSize / chunkN) @@ -454,13 +456,13 @@ func SplitFileByPartSize(fileSize int64, chunkSize int64) ([]oss.FileChunk, erro return nil, errors.New("chunkSize invalid") } - var chunkN = fileSize / chunkSize + chunkN := fileSize / chunkSize if chunkN >= 10000 { return nil, errors.New("Too many parts, please increase part size") } var chunks []oss.FileChunk - var chunk = oss.FileChunk{} + chunk := oss.FileChunk{} for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * chunkSize diff --git a/drivers/115_share/meta.go b/drivers/115_share/meta.go index 90dd7d8f170..1d203b24c2b 100644 --- a/drivers/115_share/meta.go +++ b/drivers/115_share/meta.go @@ -8,7 +8,7 @@ import ( type Addition struct { Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` - QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,linux,mac,windows,tv" default:"linux" help:"select the QR code device, default linux"` + QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` PageSize int64 `json:"page_size" type:"number" default:"20" help:"list api per page size of 115 driver"` LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"` diff --git a/go.mod b/go.mod index 4a32d700ffe..5e04de91c8b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/alist-org/alist/v3 go 1.22.4 require ( - github.com/SheltonZhu/115driver v1.0.26 + github.com/SheltonZhu/115driver v1.0.27 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 diff --git a/go.sum b/go.sum index 37ecb01789d..4931aee2302 100644 --- a/go.sum +++ b/go.sum @@ -7,12 +7,10 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/SheltonZhu/115driver v1.0.26 h1:UDUEZffJoQLFYs2nxnyxqvxwSaocxP4LNaOycVY6syU= -github.com/SheltonZhu/115driver v1.0.26/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= +github.com/SheltonZhu/115driver v1.0.27 h1:Ya1HYHYXFmi7JnqQ/+Vy6xZvq3leto+E+PxTm6UChj8= +github.com/SheltonZhu/115driver v1.0.27/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= -github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= -github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaVLk4gI/Co0paF92FmxSR6U1zY= @@ -48,18 +46,12 @@ github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZ github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/blevesearch/bleve/v2 v2.4.1 h1:8QWqsifq693mN3h6cSigKqkKUsUfv5hu0FDgz/4bFuA= -github.com/blevesearch/bleve/v2 v2.4.1/go.mod h1:Ezmvsouspi+uVwnDzjIsCeUIT0WuBKlicP5JZnExWzo= github.com/blevesearch/bleve/v2 v2.4.2 h1:NooYP1mb3c0StkiY9/xviiq2LGSaE8BQBCc/pirMx0U= github.com/blevesearch/bleve/v2 v2.4.2/go.mod h1:ATNKj7Yl2oJv/lGuF4kx39bST2dveX6w0th2FFYLkc8= -github.com/blevesearch/bleve_index_api v1.1.9 h1:Cpq0Lp3As0Gfk3+PmcoNDRKeI50C5yuFNpj0YlN/bOE= -github.com/blevesearch/bleve_index_api v1.1.9/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/bleve_index_api v1.1.10 h1:PDLFhVjrjQWr6jCuU7TwlmByQVCSEURADHdCqVS9+g0= github.com/blevesearch/bleve_index_api v1.1.10/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= -github.com/blevesearch/go-faiss v1.0.19 h1:UKoP8hS7DVsVSRRloNJb4qPfe2UQ99pP4D3oXd23g2A= -github.com/blevesearch/go-faiss v1.0.19/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/go-faiss v1.0.20 h1:AIkdTQFWuZ5LQmKQSebgMR4RynGNw8ZseJXaan5kvtI= github.com/blevesearch/go-faiss v1.0.20/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= @@ -68,8 +60,6 @@ github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZG github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.2.14 h1:fgMLMpGWR7u2TdRm7XSZVWhPvMAcdYHh25Lq1fQ6Fjo= -github.com/blevesearch/scorch_segment_api/v2 v2.2.14/go.mod h1:B7+a7vfpY4NsjuTkpv/eY7RZ91Xr90VaJzT2t7upZN8= github.com/blevesearch/scorch_segment_api/v2 v2.2.15 h1:prV17iU/o+A8FiZi9MXmqbagd8I0bCqM7OKUYPbnb5Y= github.com/blevesearch/scorch_segment_api/v2 v2.2.15/go.mod h1:db0cmP03bPNadXrCDuVkKLV6ywFSiRgPFT1YVrestBc= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= @@ -90,8 +80,6 @@ github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz7 github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= -github.com/blevesearch/zapx/v16 v16.1.4 h1:TBQfG77g2UUXwfjOVcEtB9pXkg6JBmGXkeZKI67+TiA= -github.com/blevesearch/zapx/v16 v16.1.4/go.mod h1:+Q+Z89Iv7ewhdX2jyE6Qs/RUnN4tZuokaQ0xvTaFmx8= github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi/AUHjU= github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= @@ -150,8 +138,6 @@ github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 h1:OtSeLS5y0Uy01jaKK4m github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8/go.mod h1:apkPC/CR3s48O2D7Y++n1XWEpgPNNCjXYga3PPbJe2E= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= -github.com/dlclark/regexp2 v1.11.2 h1:/u628IuisSTwri5/UKloiIsH8+qF2Pu7xEQX+yIKg68= -github.com/dlclark/regexp2 v1.11.2/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= @@ -318,8 +304,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/larksuite/oapi-sdk-go/v3 v3.3.0 h1:aCtFUiYgoRUW+aaWzVYw8jSzMe4A71rPEIn1DyHcNrY= -github.com/larksuite/oapi-sdk-go/v3 v3.3.0/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/larksuite/oapi-sdk-go/v3 v3.3.1 h1:DLQQEgHUAGZB6RVlceB1f6A94O206exxW2RIMH+gMUc= github.com/larksuite/oapi-sdk-go/v3 v3.3.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -352,8 +336,6 @@ github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= -github.com/meilisearch/meilisearch-go v0.27.0 h1:lDFq8WzbsZCtt3/byr7GFqfOygWF5iy9TtDgzJo0Ds8= -github.com/meilisearch/meilisearch-go v0.27.0/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/meilisearch/meilisearch-go v0.27.2 h1:3G21dJ5i208shnLPDsIEZ0L0Geg/5oeXABFV7nlK94k= github.com/meilisearch/meilisearch-go v0.27.2/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= @@ -522,8 +504,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= -github.com/xhofe/tache v0.1.1 h1:O5QY4cVjIGELx3UGh6LbVAc18MWGXgRNQjMt72x6w/8= -github.com/xhofe/tache v0.1.1/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/tache v0.1.2 h1:pHrXlrWcbTb4G7hVUDW7Rc+YTUnLJvnLBrdktVE1Fqg= github.com/xhofe/tache v0.1.2/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= @@ -566,12 +546,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= -golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -596,12 +572,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= -golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -609,8 +581,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -642,8 +612,6 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -653,8 +621,6 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= -golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= -golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -668,13 +634,10 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -686,8 +649,6 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg= -golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 94937db4915ceea4dad19830bc03a4e729279f4f Mon Sep 17 00:00:00 2001 From: Mmx Date: Wed, 14 Aug 2024 19:34:48 +0800 Subject: [PATCH 280/659] feat(s3): using internal download method in proxy (#6988) --- drivers/s3/driver.go | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 728c642038c..2b72d78980f 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/alist-org/alist/v3/server/common" "io" "net/url" stdpath "path" @@ -95,23 +96,27 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo input.ResponseContentDisposition = &disposition } req, _ := d.linkClient.GetObjectRequest(input) - var link string + var link model.Link var err error if d.CustomHost != "" { err = req.Build() - link = req.HTTPRequest.URL.String() + link.URL = req.HTTPRequest.URL.String() if d.RemoveBucket { - link = strings.Replace(link, "/"+d.Bucket, "", 1) + link.URL = strings.Replace(link.URL, "/"+d.Bucket, "", 1) } } else { - link, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + if common.ShouldProxy(d, filename) { + err = req.Sign() + link.URL = req.HTTPRequest.URL.String() + link.Header = req.HTTPRequest.Header + } else { + link.URL, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + } } if err != nil { return nil, err } - return &model.Link{ - URL: link, - }, nil + return &link, nil } func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { From 6bff5b610731139b558e23ffca5acfff50196c4a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 14 Aug 2024 19:35:33 +0800 Subject: [PATCH 281/659] fix(deps): update module golang.org/x/image to v0.19.0 (#6982) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5e04de91c8b..ae7c36c89cf 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,7 @@ require ( github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.26.0 golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa - golang.org/x/image v0.18.0 + golang.org/x/image v0.19.0 golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 golang.org/x/time v0.6.0 diff --git a/go.sum b/go.sum index 4931aee2302..d0cfa9fc926 100644 --- a/go.sum +++ b/go.sum @@ -553,6 +553,8 @@ golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeId golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= +golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= +golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= From 8e6c1aa78d1f43e7e6f329a7d27b82753ea165da Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Thu, 15 Aug 2024 21:46:55 +0800 Subject: [PATCH 282/659] fix(pikpak): refresh_token cannot be obtained (#7017) --- drivers/pikpak/driver.go | 57 +++++--------- drivers/pikpak/meta.go | 2 + drivers/pikpak/util.go | 156 +++++++++++++++++++++++++-------------- 3 files changed, 124 insertions(+), 91 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 1d086e0a5cf..f0fc57f7d29 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -20,14 +20,14 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" - "golang.org/x/oauth2" ) type PikPak struct { model.Storage Addition *Common - oauth2Token oauth2.TokenSource + RefreshToken string + AccessToken string } func (d *PikPak) Config() driver.Config { @@ -58,29 +58,27 @@ func (d *PikPak) Init(ctx context.Context) (err error) { } } - oauth2Config := &oauth2.Config{ - ClientID: d.ClientID, - ClientSecret: d.ClientSecret, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://user.mypikpak.com/v1/auth/signin", - TokenURL: "https://user.mypikpak.com/v1/auth/token", - AuthStyle: oauth2.AuthStyleInParams, - }, + if d.Addition.CaptchaToken != "" && d.Addition.RefreshToken == "" { + d.SetCaptchaToken(d.Addition.CaptchaToken) } - d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { - return oauth2Config.PasswordCredentialsToken( - context.WithValue(context.Background(), oauth2.HTTPClient, base.HttpClient), - d.Username, - d.Password, - ) - })) - - // 获取用户ID - _ = d.GetUserID() + // 如果已经有RefreshToken，直接刷新AccessToken + if d.Addition.RefreshToken != "" { + d.RefreshToken = d.Addition.RefreshToken + if err := d.refreshToken(); err != nil { + return err + } + } else { + if err := d.login(); err != nil { + return err + } + } // 获取CaptchaToken - _ = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Common.UserID) + err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Common.UserID) + if err != nil { + return err + } // 更新UserAgent d.Common.UserAgent = BuildCustomUserAgent(d.Common.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, d.Common.UserID) return nil @@ -102,7 +100,7 @@ func (d *PikPak) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp File - _, err := d.requestWithCaptchaToken(fmt.Sprintf("https://api-drive.mypikpak.com/drive/v1/files/%s?_magic=2021&thumbnail_size=SIZE_LARGE", file.GetID()), + _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.com/drive/v1/files/%s?_magic=2021&thumbnail_size=SIZE_LARGE", file.GetID()), http.MethodGet, nil, &resp) if err != nil { return nil, err @@ -320,19 +318,4 @@ func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, delet return nil } -func (d *PikPak) GetUserID() error { - - token, err := d.oauth2Token.Token() - if err != nil { - return err - } - - userID := token.Extra("sub").(string) - - if userID != "" { - d.Common.SetUserID(userID) - } - return nil -} - var _ driver.Driver = (*PikPak)(nil) diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index c462ed13d6b..51ba5c46937 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -11,6 +11,8 @@ type Addition struct { Password string `json:"password" required:"true"` ClientID string `json:"client_id" required:"true" default:"YNxT9w7GMdWvEOKa"` ClientSecret string `json:"client_secret" required:"true" default:"dbw2OtmVEeuUvIptb1Coyg"` + RefreshToken string `json:"refresh_token" required:"true" default:""` + CaptchaToken string `json:"captcha_token" default:""` DisableMediaLink bool `json:"disable_media_link"` } diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index d4cf64f05a1..7eb2b803162 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -4,8 +4,11 @@ import ( "crypto/md5" "crypto/sha1" "encoding/hex" + "errors" "fmt" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" + jsoniter "github.com/json-iterator/go" "net/http" "regexp" "strings" @@ -18,34 +21,101 @@ import ( // do others that not defined in Driver interface var Algorithms = []string{ - "PAe56I7WZ6FCSkFy77A96jHWcQA27ui80Qy4", - "SUbmk67TfdToBAEe2cZyP8vYVeN", - "1y3yFSZVWiGN95fw/2FQlRuH/Oy6WnO", - "8amLtHJpGzHPz4m9hGz7r+i+8dqQiAk", - "tmIEq5yl2g/XWwM3sKZkY4SbL8YUezrvxPksNabUJ", - "4QvudeJwgJuSf/qb9/wjC21L5aib", - "D1RJd+FZ+LBbt+dAmaIyYrT9gxJm0BB", - "1If", - "iGZr/SJPUFRkwvC174eelKy", + "Gez0T9ijiI9WCeTsKSg3SMlx", + "zQdbalsolyb1R/", + "ftOjr52zt51JD68C3s", + "yeOBMH0JkbQdEFNNwQ0RI9T3wU/v", + "BRJrQZiTQ65WtMvwO", + "je8fqxKPdQVJiy1DM6Bc9Nb1", + "niV", + "9hFCW2R1", + "sHKHpe2i96", + "p7c5E6AcXQ/IJUuAEC9W6", + "", + "aRv9hjc9P+Pbn+u3krN6", + "BzStcgE8qVdqjEH16l4", + "SqgeZvL5j9zoHP95xWHt", + "zVof5yaJkPe3VFpadPof", } const ( ClientID = "YNxT9w7GMdWvEOKa" ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - ClientVersion = "1.46.2" + ClientVersion = "1.47.1" PackageName = "com.pikcloud.pikpak" SdkVersion = "2.0.4.204000 " ) -func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - req := base.RestyClient.R() +func (d *PikPak) login() error { + url := "https://user.mypikpak.com/v1/auth/signin" + if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { + return err + } + var e ErrResp + res, err := base.RestyClient.R().SetError(&e).SetBody(base.Json{ + "captcha_token": d.GetCaptchaToken(), + "client_id": ClientID, + "client_secret": ClientSecret, + "username": d.Username, + "password": d.Password, + }).SetQueryParam("client_id", ClientID).Post(url) + if err != nil { + return err + } + if e.ErrorCode != 0 { + return &e + } + data := res.Body() + d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() + d.AccessToken = jsoniter.Get(data, "access_token").ToString() + d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) + d.Addition.RefreshToken = d.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} - token, err := d.oauth2Token.Token() +func (d *PikPak) refreshToken() error { + url := "https://user.mypikpak.com/v1/auth/token" + var e ErrResp + res, err := base.RestyClient.R().SetError(&e). + SetHeader("user-agent", "").SetBody(base.Json{ + "client_id": ClientID, + "client_secret": ClientSecret, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }).SetQueryParam("client_id", ClientID).Post(url) if err != nil { - return nil, err + d.Status = err.Error() + op.MustSaveDriverStorage(d) + return err + } + if e.ErrorCode != 0 { + if e.ErrorCode == 4126 { + // refresh_token invalid, re-login + return d.login() + } + d.Status = e.Error() + op.MustSaveDriverStorage(d) + return errors.New(e.Error()) } - req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) + data := res.Body() + d.Status = "work" + d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() + d.AccessToken = jsoniter.Get(data, "access_token").ToString() + d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) + d.Addition.RefreshToken = d.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} +func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Authorization": "Bearer " + d.AccessToken, + "User-Agent": d.GetUserAgent(), + "X-Device-ID": d.GetDeviceID(), + "X-Captcha-Token": d.GetCaptchaToken(), + }) if callback != nil { callback(req) } @@ -59,48 +129,22 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r return nil, err } - if e.IsError() { - return nil, &e - } - return res.Body(), nil -} - -func (d *PikPak) requestWithCaptchaToken(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - - data, err := d.request(url, method, func(req *resty.Request) { - req.SetHeaders(map[string]string{ - "User-Agent": d.GetUserAgent(), - "X-Device-ID": d.GetDeviceID(), - "X-Captcha-Token": d.GetCaptchaToken(), - }) - if callback != nil { - callback(req) - } - }, resp) - - errResp, ok := err.(*ErrResp) - if !ok { - return nil, err - } - - switch errResp.ErrorCode { + switch e.ErrorCode { case 0: - return data, nil - //case 4122, 4121, 10, 16: - // if d.refreshTokenFunc != nil { - // if err = xc.refreshTokenFunc(); err == nil { - // break - // } - // } - // return nil, err + return res.Body(), nil + case 4122, 4121, 10, 16: + if err1 := d.refreshToken(); err1 != nil { + return nil, err1 + } + return d.request(url, method, callback, resp) case 9: // 验证码token过期 if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.Common.UserID); err != nil { return nil, err } + return d.request(url, method, callback, resp) default: return nil, err } - return d.requestWithCaptchaToken(url, method, callback, resp) } func (d *PikPak) getFiles(id string) ([]File, error) { @@ -276,7 +320,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err var e ErrResp var resp CaptchaTokenResponse _, err := d.request("https://user.mypikpak.com/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { - req.SetError(&e).SetBody(param) + req.SetError(&e).SetBody(param).SetQueryParam("client_id", ClientID) }, &resp) if err != nil { @@ -287,12 +331,16 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err return &e } - if resp.Url != "" { - return fmt.Errorf(`need verify: Click Here`, resp.Url) - } - if resp.CaptchaToken == "" { return fmt.Errorf("empty captchaToken") + } else { + // 对 被风控的情况 进行处理 + d.Addition.CaptchaToken = resp.CaptchaToken + op.MustSaveDriverStorage(d) + } + + if resp.Url != "" { + return fmt.Errorf(`need verify: Click Here`, resp.Url) } if d.Common.RefreshCTokenCk != nil { From 1f652e2e7d2b9a70d7c601ebcbdf0e7191ffe3b6 Mon Sep 17 00:00:00 2001 From: Mmx Date: Thu, 15 Aug 2024 21:48:48 +0800 Subject: [PATCH 283/659] ci(docker): using docker build args instead of extra dockerfile for ffmpeg (#6989) * build: using docker build arg to determine install ffmpeg or not * ci: pass build-args to ffmpeg image build step --- .github/workflows/build_docker.yml | 7 ++----- .github/workflows/release_docker.yml | 3 ++- Dockerfile | 15 +++++++++++---- Dockerfile.ci | 18 ++++++++++++------ Dockerfile.ffmpeg | 4 ---- 5 files changed, 27 insertions(+), 20 deletions(-) delete mode 100644 Dockerfile.ffmpeg diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index c6c301f3ba8..cfd045117e1 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -74,19 +74,16 @@ jobs: labels: ${{ steps.meta.outputs.labels }} platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x - - name: Replace dockerfile tag - run: | - sed -i -e "s/latest/main/g" Dockerfile.ffmpeg - - name: Build and push with ffmpeg id: docker_build_ffmpeg uses: docker/build-push-action@v6 with: context: . - file: Dockerfile.ffmpeg + file: Dockerfile.ci push: ${{ github.event_name == 'push' }} tags: ${{ steps.meta-ffmpeg.outputs.tags }} labels: ${{ steps.meta-ffmpeg.outputs.labels }} + build-args: INSTALL_FFMPEG=true platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x build_docker_with_aria2: diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 3078d0c3094..4e039ba0780 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -74,10 +74,11 @@ jobs: uses: docker/build-push-action@v6 with: context: . - file: Dockerfile.ffmpeg + file: Dockerfile.ci push: true tags: ${{ steps.meta-ffmpeg.outputs.tags }} labels: ${{ steps.meta-ffmpeg.outputs.labels }} + build-args: INSTALL_FFMPEG=true platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x release_docker_with_aria2: diff --git a/Dockerfile b/Dockerfile index 23ca42da201..74fa2165482 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,16 +8,23 @@ COPY ./ ./ RUN bash build.sh release docker FROM alpine:edge + +ARG INSTALL_FFMPEG=false LABEL MAINTAINER="i@nn.ci" -VOLUME /opt/alist/data/ + WORKDIR /opt/alist/ -COPY --from=builder /app/bin/alist ./ -COPY entrypoint.sh /entrypoint.sh + RUN apk update && \ apk upgrade --no-cache && \ apk add --no-cache bash ca-certificates su-exec tzdata; \ - chmod +x /entrypoint.sh && \ + [ "$INSTALL_FFMPEG" = "true" ] && apk add --no-cache ffmpeg; \ rm -rf /var/cache/apk/* + +COPY --from=builder /app/bin/alist ./ +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh && /entrypoint.sh version + ENV PUID=0 PGID=0 UMASK=022 +VOLUME /opt/alist/data/ EXPOSE 5244 5245 CMD [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/Dockerfile.ci b/Dockerfile.ci index c25e2471b16..3f437f16dfe 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,16 +1,22 @@ FROM alpine:edge + ARG TARGETPLATFORM +ARG INSTALL_FFMPEG=false LABEL MAINTAINER="i@nn.ci" -VOLUME /opt/alist/data/ + WORKDIR /opt/alist/ -COPY /build/${TARGETPLATFORM}/alist ./ -COPY entrypoint.sh /entrypoint.sh + RUN apk update && \ apk upgrade --no-cache && \ apk add --no-cache bash ca-certificates su-exec tzdata; \ - chmod +x /entrypoint.sh && \ - rm -rf /var/cache/apk/* && \ - /entrypoint.sh version + [ "$INSTALL_FFMPEG" = "true" ] && apk add --no-cache ffmpeg; \ + rm -rf /var/cache/apk/* + +COPY /build/${TARGETPLATFORM}/alist ./ +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh && /entrypoint.sh version + ENV PUID=0 PGID=0 UMASK=022 +VOLUME /opt/alist/data/ EXPOSE 5244 5245 CMD [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/Dockerfile.ffmpeg b/Dockerfile.ffmpeg deleted file mode 100644 index 9799d777edf..00000000000 --- a/Dockerfile.ffmpeg +++ /dev/null @@ -1,4 +0,0 @@ -FROM xhofe/alist:latest -RUN apk update && \ - apk add --no-cache ffmpeg \ - rm -rf /var/cache/apk/* \ No newline at end of file From 51c95ee1173a4c84e2d14ecb39a89be2bc6c2983 Mon Sep 17 00:00:00 2001 From: 1-1-2 <50569812+1-1-2@users.noreply.github.com> Date: Thu, 15 Aug 2024 22:25:53 +0800 Subject: [PATCH 284/659] fix: decode body if enable gzip (#7003) --- internal/net/serve.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/net/serve.go b/internal/net/serve.go index adee75ae1d6..e58d7eb9f46 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -1,6 +1,7 @@ package net import ( + "compress/gzip" "context" "fmt" "io" @@ -222,8 +223,19 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea } // TODO clean header with blocklist or passlist res.Header.Del("set-cookie") + var reader io.Reader if res.StatusCode >= 400 { - all, _ := io.ReadAll(res.Body) + // 根据 Content-Encoding 判断 Body 是否压缩 + switch res.Header.Get("Content-Encoding") { + case "gzip": + // 使用gzip.NewReader解压缩 + reader, _ = gzip.NewReader(res.Body) + defer reader.(*gzip.Reader).Close() + default: + // 没有Content-Encoding，直接读取 + reader = res.Body + } + all, _ := io.ReadAll(reader) _ = res.Body.Close() msg := string(all) log.Debugln(msg) From e1906c9312822608877460c52fb6bf06fed276b5 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 17 Aug 2024 22:06:08 +0800 Subject: [PATCH 285/659] ci: only release on tag with `v` prefix --- .github/workflows/changelog.yml | 3 ++- .github/workflows/release_docker.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 8c1bfb67d03..55efd9a8984 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -3,7 +3,7 @@ name: auto changelog on: push: tags: - - '*' + - 'v*' jobs: changelog: @@ -14,6 +14,7 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 + - run: npx changelogithub # or changelogithub@0.12 if ensure the stable result env: GITHUB_TOKEN: ${{secrets.MY_TOKEN}} diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 4e039ba0780..95a686b2fce 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -3,7 +3,7 @@ name: release_docker on: push: tags: - - '*' + - 'v*' jobs: release_docker: From e5fe9ea5f62902f6c23e0fb67b43141b58f4b2cd Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 17 Aug 2024 22:20:38 +0800 Subject: [PATCH 286/659] ci: set changelog for beta release --- .github/workflows/beta_release.yml | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/beta_release.yml diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml new file mode 100644 index 00000000000..55abb02ea7f --- /dev/null +++ b/.github/workflows/beta_release.yml @@ -0,0 +1,36 @@ +name: beta release + +on: + push: + branches: [ 'main' ] + +jobs: + release: + strategy: + matrix: + platform: [ ubuntu-latest ] + go-version: [ '1.21' ] + name: Beta Release Changelog + runs-on: ${{ matrix.platform }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: changelog # or changelogithub@0.12 if ensure the stable result + id: changelog + run: | + git tag -l + npx changelogen@latest --output CHANGELOG.md +# env: +# GITHUB_TOKEN: ${{secrets.MY_TOKEN}} + + - name: Prerelease + uses: tubone24/update_release@master + env: + GITHUB_TOKEN: ${{ secrets.MY_TOKEN }} + TAG_NAME: beta + with: + prerelease: true + body_path: CHANGELOG.md \ No newline at end of file From 4ba476e25c7f42a2ecddcb4d12f2a2d381e3278b Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 18 Aug 2024 00:21:48 +0800 Subject: [PATCH 287/659] ci: build beta release --- .github/workflows/beta_release.yml | 66 +++++++++++++++++++++++++----- build.sh | 6 +++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 55abb02ea7f..d83d5026c19 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -5,7 +5,7 @@ on: branches: [ 'main' ] jobs: - release: + changelog: strategy: matrix: platform: [ ubuntu-latest ] @@ -23,14 +23,60 @@ jobs: run: | git tag -l npx changelogen@latest --output CHANGELOG.md -# env: -# GITHUB_TOKEN: ${{secrets.MY_TOKEN}} - - - name: Prerelease - uses: tubone24/update_release@master - env: - GITHUB_TOKEN: ${{ secrets.MY_TOKEN }} - TAG_NAME: beta + + - name: Upload assets + uses: softprops/action-gh-release@v2 + with: + body_path: CHANGELOG.md + files: CHANGELOG.md + prerelease: true + tag_name: beta + + release: + needs: + - changelog + strategy: + matrix: + include: + - target: '!(*musl*|*windows-arm64*|*android*)' # xgo + hash: "md5" + - target: 'linux-*-musl*' #musl + hash: "md5-linux-musl" + - target: 'windows-arm64' #win-arm64 + hash: "md5-windows-arm64" + - target: 'android-*' #android + hash: "md5-android" + name: Beta Release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Setup web + run: bash build.sh dev web + + - name: Build + id: test-action + uses: go-cross/cgo-actions@v1 + with: + targets: ${{ matrix.target }} + musl-target-format: $os-$musl-$arch + out-dir: build + + - name: Compress + run: | + bash build.sh zip ${{ matrix.hash }} + + - name: Upload assets + uses: softprops/action-gh-release@v2 with: + files: build/compress/* prerelease: true - body_path: CHANGELOG.md \ No newline at end of file + tag_name: beta \ No newline at end of file diff --git a/build.sh b/build.sh index 9d0f4174432..5a7d0d666ef 100644 --- a/build.sh +++ b/build.sh @@ -267,6 +267,8 @@ if [ "$1" = "dev" ]; then BuildDocker elif [ "$2" = "docker-multiplatform" ]; then BuildDockerMultiplatform + elif [ "$2" = "web" ]; then + echo "web only" else BuildDev fi @@ -285,6 +287,8 @@ elif [ "$1" = "release" ]; then elif [ "$2" = "android" ]; then BuildReleaseAndroid MakeRelease "md5-android.txt" + elif [ "$2" = "web" ]; then + echo "web only" else BuildRelease MakeRelease "md5.txt" @@ -293,6 +297,8 @@ elif [ "$1" = "prepare" ]; then if [ "$2" = "docker-multiplatform" ]; then PrepareBuildDockerMusl fi +elif [ "$1" = "zip" ]; then + MakeRelease "$2".txt else echo -e "Parameter error" fi From e8e6d71c415d26a6e36372957867bf858264522f Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 18 Aug 2024 00:38:27 +0800 Subject: [PATCH 288/659] ci: only one beta release action concurrency [skip ci] --- .github/workflows/beta_release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index d83d5026c19..37a5f5745a1 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -4,6 +4,10 @@ on: push: branches: [ 'main' ] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: changelog: strategy: From 69e5b66b5002bc8f89fdf1204f6e0e9498b277d2 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 18 Aug 2024 13:57:44 +0800 Subject: [PATCH 289/659] ci: use changelogithub to generate changelog --- .github/workflows/beta_release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 37a5f5745a1..4252b1309e5 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -26,7 +26,9 @@ jobs: id: changelog run: | git tag -l - npx changelogen@latest --output CHANGELOG.md + npx changelogithub --output CHANGELOG.md +# npx changelogen@latest --output CHANGELOG.md + - name: Upload assets uses: softprops/action-gh-release@v2 From e238b90836d4780302ed7b2c40e37f3dfc9ee5f2 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sun, 18 Aug 2024 23:26:29 +0800 Subject: [PATCH 290/659] fix(pikpak): modify the processing logic of CaptchaToken (#7024) --- drivers/pikpak/driver.go | 92 ++++++++++++++++--- drivers/pikpak/meta.go | 6 +- drivers/pikpak/util.go | 186 ++++++++++++++++++++++++--------------- 3 files changed, 196 insertions(+), 88 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index f0fc57f7d29..13b9843015d 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -5,6 +5,8 @@ import ( "encoding/json" "fmt" "github.com/alist-org/alist/v3/internal/op" + "golang.org/x/oauth2" + "io" "net/http" "strconv" "strings" @@ -28,6 +30,7 @@ type PikPak struct { *Common RefreshToken string AccessToken string + oauth2Token oauth2.TokenSource } func (d *PikPak) Config() driver.Config { @@ -39,10 +42,6 @@ func (d *PikPak) GetAddition() driver.Additional { } func (d *PikPak) Init(ctx context.Context) (err error) { - if d.ClientID == "" || d.ClientSecret == "" { - d.ClientID = "YNxT9w7GMdWvEOKa" - d.ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - } if d.Common == nil { d.Common = &Common{ @@ -50,7 +49,7 @@ func (d *PikPak) Init(ctx context.Context) (err error) { CaptchaToken: "", UserID: "", DeviceID: utils.GetMD5EncodeStr(d.Username + d.Password), - UserAgent: BuildCustomUserAgent(utils.GetMD5EncodeStr(d.Username+d.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, ""), + UserAgent: "", RefreshCTokenCk: func(token string) { d.Common.CaptchaToken = token op.MustSaveDriverStorage(d) @@ -58,21 +57,70 @@ func (d *PikPak) Init(ctx context.Context) (err error) { } } + if d.Platform == "android" { + d.ClientID = AndroidClientID + d.ClientSecret = AndroidClientSecret + d.ClientVersion = AndroidClientVersion + d.PackageName = AndroidPackageName + d.Algorithms = AndroidAlgorithms + d.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(d.Username+d.Password), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, "") + } else if d.Platform == "web" { + d.ClientID = WebClientID + d.ClientSecret = WebClientSecret + d.ClientVersion = WebClientVersion + d.PackageName = WebPackageName + d.Algorithms = WebAlgorithms + d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" + } + if d.Addition.CaptchaToken != "" && d.Addition.RefreshToken == "" { d.SetCaptchaToken(d.Addition.CaptchaToken) } - // 如果已经有RefreshToken，直接刷新AccessToken + if d.Addition.DeviceID != "" { + d.SetDeviceID(d.Addition.DeviceID) + } else { + d.Addition.DeviceID = d.Common.DeviceID + op.MustSaveDriverStorage(d) + } + // 初始化 oauth2Config + oauth2Config := &oauth2.Config{ + ClientID: d.ClientID, + ClientSecret: d.ClientSecret, + Endpoint: oauth2.Endpoint{ + AuthURL: "https://user.mypikpak.com/v1/auth/signin", + TokenURL: "https://user.mypikpak.com/v1/auth/token", + AuthStyle: oauth2.AuthStyleInParams, + }, + } + + // 如果已经有RefreshToken，直接获取AccessToken if d.Addition.RefreshToken != "" { - d.RefreshToken = d.Addition.RefreshToken - if err := d.refreshToken(); err != nil { - return err - } + // 使用 oauth2 刷新令牌 + // 初始化 oauth2Token + d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { + return oauth2Config.TokenSource(ctx, &oauth2.Token{ + RefreshToken: d.Addition.RefreshToken, + }).Token() + })) } else { + // 如果没有填写RefreshToken，尝试登录 获取 refreshToken if err := d.login(); err != nil { return err } + d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { + return oauth2Config.TokenSource(ctx, &oauth2.Token{ + RefreshToken: d.RefreshToken, + }).Token() + })) + } + + token, err := d.oauth2Token.Token() + if err != nil { + return err } + d.RefreshToken = token.RefreshToken + d.AccessToken = token.AccessToken // 获取CaptchaToken err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Common.UserID) @@ -80,7 +128,13 @@ func (d *PikPak) Init(ctx context.Context) (err error) { return err } // 更新UserAgent - d.Common.UserAgent = BuildCustomUserAgent(d.Common.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, d.Common.UserID) + if d.Platform == "android" { + d.Common.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(d.Username+d.Password), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, d.Common.UserID) + } + + // 保存 有效的 RefreshToken + d.Addition.RefreshToken = d.RefreshToken + op.MustSaveDriverStorage(d) return nil } @@ -100,8 +154,18 @@ func (d *PikPak) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp File - _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.com/drive/v1/files/%s?_magic=2021&thumbnail_size=SIZE_LARGE", file.GetID()), - http.MethodGet, nil, &resp) + queryParams := map[string]string{ + "_magic": "2021", + "usage": "FETCH", + "thumbnail_size": "SIZE_LARGE", + } + if !d.DisableMediaLink { + queryParams["usage"] = "CACHE" + } + _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.com/drive/v1/files/%s", file.GetID()), + http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(queryParams) + }, &resp) if err != nil { return nil, err } @@ -224,7 +288,7 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr input := &s3manager.UploadInput{ Bucket: ¶ms.Bucket, Key: ¶ms.Key, - Body: stream, + Body: io.TeeReader(stream, driver.NewProgress(stream.GetSize(), up)), } _, err = uploader.UploadWithContext(ctx, input) return err diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index 51ba5c46937..d27cee32c74 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -9,11 +9,11 @@ type Addition struct { driver.RootID Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` - ClientID string `json:"client_id" required:"true" default:"YNxT9w7GMdWvEOKa"` - ClientSecret string `json:"client_secret" required:"true" default:"dbw2OtmVEeuUvIptb1Coyg"` + Platform string `json:"platform" required:"true" type:"select" options:"android,web"` RefreshToken string `json:"refresh_token" required:"true" default:""` CaptchaToken string `json:"captcha_token" default:""` - DisableMediaLink bool `json:"disable_media_link"` + DeviceID string `json:"device_id" required:"false" default:""` + DisableMediaLink bool `json:"disable_media_link" default:"true"` } var config = driver.Config{ diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 7eb2b803162..902a9d32191 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -20,7 +20,7 @@ import ( // do others that not defined in Driver interface -var Algorithms = []string{ +var AndroidAlgorithms = []string{ "Gez0T9ijiI9WCeTsKSg3SMlx", "zQdbalsolyb1R/", "ftOjr52zt51JD68C3s", @@ -38,27 +38,54 @@ var Algorithms = []string{ "zVof5yaJkPe3VFpadPof", } +var WebAlgorithms = []string{ + "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", + "+r6CQVxjzJV6LCV", + "F", + "pFJRC", + "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", + "/750aCr4lm/Sly/c", + "RB+DT/gZCrbV", + "", + "CyLsf7hdkIRxRm215hl", + "7xHvLi2tOYP0Y92b", + "ZGTXXxu8E/MIWaEDB+Sm/", + "1UI3", + "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", + "ihtqpG6FMt65+Xk+tWUH2", + "NhXXU9rg4XXdzo7u5o", +} + const ( - ClientID = "YNxT9w7GMdWvEOKa" - ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - ClientVersion = "1.47.1" - PackageName = "com.pikcloud.pikpak" - SdkVersion = "2.0.4.204000 " + AndroidClientID = "YNxT9w7GMdWvEOKa" + AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + AndroidClientVersion = "1.47.1" + AndroidPackageName = "com.pikcloud.pikpak" + AndroidSdkVersion = "2.0.4.204000" + WebClientID = "YUMx5nI8ZU8Ap8pm" + WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + WebClientVersion = "2.0.0" + WebPackageName = "mypikpak.com" + WebSdkVersion = "8.0.3" ) func (d *PikPak) login() error { url := "https://user.mypikpak.com/v1/auth/signin" - if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { - return err + // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) + if d.GetCaptchaToken() == "" { + if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { + return err + } } + var e ErrResp - res, err := base.RestyClient.R().SetError(&e).SetBody(base.Json{ + res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e).SetBody(base.Json{ "captcha_token": d.GetCaptchaToken(), - "client_id": ClientID, - "client_secret": ClientSecret, + "client_id": d.ClientID, + "client_secret": d.ClientSecret, "username": d.Username, "password": d.Password, - }).SetQueryParam("client_id", ClientID).Post(url) + }).SetQueryParam("client_id", d.ClientID).Post(url) if err != nil { return err } @@ -69,53 +96,60 @@ func (d *PikPak) login() error { d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() d.AccessToken = jsoniter.Get(data, "access_token").ToString() d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) - d.Addition.RefreshToken = d.RefreshToken - op.MustSaveDriverStorage(d) return nil } -func (d *PikPak) refreshToken() error { - url := "https://user.mypikpak.com/v1/auth/token" - var e ErrResp - res, err := base.RestyClient.R().SetError(&e). - SetHeader("user-agent", "").SetBody(base.Json{ - "client_id": ClientID, - "client_secret": ClientSecret, - "grant_type": "refresh_token", - "refresh_token": d.RefreshToken, - }).SetQueryParam("client_id", ClientID).Post(url) - if err != nil { - d.Status = err.Error() - op.MustSaveDriverStorage(d) - return err - } - if e.ErrorCode != 0 { - if e.ErrorCode == 4126 { - // refresh_token invalid, re-login - return d.login() - } - d.Status = e.Error() - op.MustSaveDriverStorage(d) - return errors.New(e.Error()) - } - data := res.Body() - d.Status = "work" - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() - d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) - d.Addition.RefreshToken = d.RefreshToken - op.MustSaveDriverStorage(d) - return nil -} +//func (d *PikPak) refreshToken() error { +// url := "https://user.mypikpak.com/v1/auth/token" +// var e ErrResp +// res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e). +// SetHeader("user-agent", "").SetBody(base.Json{ +// "client_id": ClientID, +// "client_secret": ClientSecret, +// "grant_type": "refresh_token", +// "refresh_token": d.RefreshToken, +// }).SetQueryParam("client_id", ClientID).Post(url) +// if err != nil { +// d.Status = err.Error() +// op.MustSaveDriverStorage(d) +// return err +// } +// if e.ErrorCode != 0 { +// if e.ErrorCode == 4126 { +// // refresh_token invalid, re-login +// return d.login() +// } +// d.Status = e.Error() +// op.MustSaveDriverStorage(d) +// return errors.New(e.Error()) +// } +// data := res.Body() +// d.Status = "work" +// d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() +// d.AccessToken = jsoniter.Get(data, "access_token").ToString() +// d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) +// d.Addition.RefreshToken = d.RefreshToken +// op.MustSaveDriverStorage(d) +// return nil +//} func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ - "Authorization": "Bearer " + d.AccessToken, + //"Authorization": "Bearer " + d.AccessToken, "User-Agent": d.GetUserAgent(), "X-Device-ID": d.GetDeviceID(), "X-Captcha-Token": d.GetCaptchaToken(), }) + if d.oauth2Token != nil { + // 使用oauth2 获取 access_token + token, err := d.oauth2Token.Token() + if err != nil { + return nil, err + } + req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) + } + if callback != nil { callback(req) } @@ -132,18 +166,31 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r switch e.ErrorCode { case 0: return res.Body(), nil - case 4122, 4121, 10, 16: - if err1 := d.refreshToken(); err1 != nil { - return nil, err1 + case 4122, 4121, 16: + // access_token 过期 + + //if err1 := d.refreshToken(); err1 != nil { + // return nil, err1 + //} + t, err := d.oauth2Token.Token() + if err != nil { + return nil, err } + d.AccessToken = t.AccessToken + d.RefreshToken = t.RefreshToken + d.Addition.RefreshToken = t.RefreshToken + op.MustSaveDriverStorage(d) + return d.request(url, method, callback, resp) case 9: // 验证码token过期 if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.Common.UserID); err != nil { return nil, err } return d.request(url, method, callback, resp) + case 10: // 操作频繁 + return nil, errors.New(e.ErrorDescription) default: - return nil, err + return nil, errors.New(e.Error()) } } @@ -185,8 +232,13 @@ type Common struct { CaptchaToken string UserID string // 必要值,签名相关 - DeviceID string - UserAgent string + ClientID string + ClientSecret string + ClientVersion string + PackageName string + Algorithms []string + DeviceID string + UserAgent string // 验证码token刷新成功回调 RefreshCTokenCk func(token string) } @@ -275,8 +327,8 @@ func (c *Common) GetDeviceID() string { // RefreshCaptchaTokenAtLogin 刷新验证码token(登录后) func (d *PikPak) RefreshCaptchaTokenAtLogin(action, userID string) error { metas := map[string]string{ - "client_version": ClientVersion, - "package_name": PackageName, + "client_version": d.ClientVersion, + "package_name": d.PackageName, "user_id": userID, } metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() @@ -299,8 +351,8 @@ func (d *PikPak) RefreshCaptchaTokenInLogin(action, username string) error { // GetCaptchaSign 获取验证码签名 func (c *Common) GetCaptchaSign() (timestamp, sign string) { timestamp = fmt.Sprint(time.Now().UnixMilli()) - str := fmt.Sprint(ClientID, ClientVersion, PackageName, c.DeviceID, timestamp) - for _, algorithm := range Algorithms { + str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) + for _, algorithm := range c.Algorithms { str = utils.GetMD5EncodeStr(str + algorithm) } sign = "1." + str @@ -311,16 +363,16 @@ func (c *Common) GetCaptchaSign() (timestamp, sign string) { func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) error { param := CaptchaTokenRequest{ Action: action, - CaptchaToken: d.Common.CaptchaToken, - ClientID: ClientID, - DeviceID: d.Common.DeviceID, + CaptchaToken: d.GetCaptchaToken(), + ClientID: d.ClientID, + DeviceID: d.GetDeviceID(), Meta: metas, RedirectUri: "xlaccsdk01://xbase.cloud/callback?state=harbor", } var e ErrResp var resp CaptchaTokenResponse _, err := d.request("https://user.mypikpak.com/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { - req.SetError(&e).SetBody(param).SetQueryParam("client_id", ClientID) + req.SetError(&e).SetBody(param).SetQueryParam("client_id", d.ClientID) }, &resp) if err != nil { @@ -328,15 +380,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err } if e.IsError() { - return &e - } - - if resp.CaptchaToken == "" { - return fmt.Errorf("empty captchaToken") - } else { - // 对 被风控的情况 进行处理 - d.Addition.CaptchaToken = resp.CaptchaToken - op.MustSaveDriverStorage(d) + return errors.New(e.Error()) } if resp.Url != "" { From e2fcd73720cb56f9d27eff1c087c87ed6a88feda Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 23:29:24 +0800 Subject: [PATCH 291/659] fix(deps): update module github.com/go-webauthn/webauthn to v0.11.1 (#6901) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ae7c36c89cf..c6a3c8da3d2 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 github.com/go-resty/resty/v2 v2.13.1 - github.com/go-webauthn/webauthn v0.10.2 + github.com/go-webauthn/webauthn v0.11.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 @@ -125,7 +125,7 @@ require ( github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/fxamacker/cbor/v2 v2.6.0 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -135,13 +135,13 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/go-webauthn/x v0.1.9 // indirect + github.com/go-webauthn/x v0.1.12 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-tpm v0.9.0 // indirect + github.com/google/go-tpm v0.9.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect diff --git a/go.sum b/go.sum index d0cfa9fc926..5855951fbfc 100644 --- a/go.sum +++ b/go.sum @@ -153,6 +153,8 @@ github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rE github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw= @@ -196,8 +198,12 @@ github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.10.2 h1:OG7B+DyuTytrEPFmTX503K77fqs3HDK/0Iv+z8UYbq4= github.com/go-webauthn/webauthn v0.10.2/go.mod h1:Gd1IDsGAybuvK1NkwUTLbGmeksxuRJjVN2PE/xsPxHs= +github.com/go-webauthn/webauthn v0.11.1 h1:5G/+dg91/VcaJHTtJUfwIlNJkLwbJCcnUc4W8VtkpzA= +github.com/go-webauthn/webauthn v0.11.1/go.mod h1:YXRm1WG0OtUyDFaVAgB5KG7kVqW+6dYCJ7FTQH4SxEE= github.com/go-webauthn/x v0.1.9 h1:v1oeLmoaa+gPOaZqUdDentu6Rl7HkSSsmOT6gxEQHhE= github.com/go-webauthn/x v0.1.9/go.mod h1:pJNMlIMP1SU7cN8HNlKJpLEnFHCygLCvaLZ8a1xeoQA= +github.com/go-webauthn/x v0.1.12 h1:RjQ5cvApzyU/xLCiP+rub0PE4HBZsLggbxGR5ZpUf/A= +github.com/go-webauthn/x v0.1.12/go.mod h1:XlRcGkNH8PT45TfeJYc6gqpOtiOendHhVmnOxh+5yHs= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -224,6 +230,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= +github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= +github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= From a54a09314fa206ea741702447826a0c2125d3a0a Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 23:29:45 +0800 Subject: [PATCH 292/659] fix(deps): update module github.com/aws/aws-sdk-go to v1.55.5 (#6813) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c6a3c8da3d2..a7ef83d7a60 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.54.19 + github.com/aws/aws-sdk-go v1.55.5 github.com/blevesearch/bleve/v2 v2.4.2 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.18.0 diff --git a/go.sum b/go.sum index 5855951fbfc..36e8b2769d5 100644 --- a/go.sum +++ b/go.sum @@ -34,6 +34,8 @@ github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevB github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI= github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= From 9af7aaab5980df100d71b9794056d55dba04ac7c Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 23:30:08 +0800 Subject: [PATCH 293/659] fix(deps): update github.com/city404/v6-public-rpc-proto/go digest to 90f8e24 (#7028) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index a7ef83d7a60..e5820134a2d 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.26.6 github.com/charmbracelet/lipgloss v0.12.1 - github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d + github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.6.0 github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 diff --git a/go.sum b/go.sum index 36e8b2769d5..bb1b378085f 100644 --- a/go.sum +++ b/go.sum @@ -114,6 +114,8 @@ github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyV github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d h1:p5T6ZPvh7nihJfjI9M/W2cbcX7n766u/OGorLmE4xoQ= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d/go.mod h1:akxZg8LuwOIeCPRjcDrUS1WWcIwmLNSR2lfe4y85PH4= +github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= +github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= From 4c48a816bf1169e1e4b810c62cf6a1fb7a6c14c4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 23:30:26 +0800 Subject: [PATCH 294/659] fix(deps): update module github.com/charmbracelet/bubbletea to v0.27.0 (#7025) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e5820134a2d..7e70148e9e2 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/blevesearch/bleve/v2 v2.4.2 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.18.0 - github.com/charmbracelet/bubbletea v0.26.6 + github.com/charmbracelet/bubbletea v0.27.0 github.com/charmbracelet/lipgloss v0.12.1 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e github.com/coreos/go-oidc v2.2.1+incompatible @@ -220,7 +220,7 @@ require ( go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.23.0 // indirect + golang.org/x/sys v0.24.0 // indirect golang.org/x/term v0.23.0 // indirect golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.24.0 // indirect diff --git a/go.sum b/go.sum index bb1b378085f..6b1041b6309 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/ github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= github.com/charmbracelet/bubbletea v0.26.6 h1:zTCWSuST+3yZYZnVSvbXwKOPRSNZceVeqpzOLN2zq1s= github.com/charmbracelet/bubbletea v0.26.6/go.mod h1:dz8CWPlfCCGLFbBlTY4N7bjLiyOGDJEnd2Muu7pOWhk= +github.com/charmbracelet/bubbletea v0.27.0 h1:Mznj+vvYuYagD9Pn2mY7fuelGvP0HAXtZYGgRBCbHvU= +github.com/charmbracelet/bubbletea v0.27.0/go.mod h1:5MdP9XH6MbQkgGhnlxUqCNmBXf9I74KRQ8HIidRxV1Y= github.com/charmbracelet/lipgloss v0.12.1 h1:/gmzszl+pedQpjCOH+wFkZr/N90Snz40J/NR7A0zQcs= github.com/charmbracelet/lipgloss v0.12.1/go.mod h1:V2CiwIuhx9S1S1ZlADfOj9HmxeMAORuz5izHb0zGbB8= github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= @@ -628,6 +630,8 @@ golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= From 18176c659c8351ddcd8aa7402455a2bd93e03ba4 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 20 Aug 2024 21:36:36 +0800 Subject: [PATCH 295/659] ci: add `beta` tag to newest docker image --- .github/workflows/build_docker.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index cfd045117e1..8f37688d07f 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -23,6 +23,12 @@ jobs: uses: docker/metadata-action@v5 with: images: xhofe/alist + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=beta,enable={{is_default_branch}} - name: Docker meta with ffmpeg id: meta-ffmpeg @@ -30,7 +36,13 @@ jobs: with: images: xhofe/alist flavor: | - suffix=-ffmpeg,onlatest=true + suffix=-ffmpeg + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=beta,enable={{is_default_branch}} - uses: actions/setup-go@v5 with: From 489b28bdf79e21c5f8ac866328810e16dfcc7d23 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Thu, 22 Aug 2024 00:35:14 +0800 Subject: [PATCH 296/659] fix(pikpak_share): add captcha_token generation function (#7045) --- drivers/pikpak_share/driver.go | 68 +++++---- drivers/pikpak_share/meta.go | 11 +- drivers/pikpak_share/types.go | 39 +++++- drivers/pikpak_share/util.go | 244 +++++++++++++++++++++++++++++++-- 4 files changed, 316 insertions(+), 46 deletions(-) diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index 1862db06bf4..448ad2dd94a 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -2,20 +2,20 @@ package pikpak_share import ( "context" + "github.com/alist-org/alist/v3/internal/op" "net/http" + "time" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - "golang.org/x/oauth2" ) type PikPakShare struct { model.Storage Addition - oauth2Token oauth2.TokenSource + *Common PassCodeToken string } @@ -28,28 +28,45 @@ func (d *PikPakShare) GetAddition() driver.Additional { } func (d *PikPakShare) Init(ctx context.Context) error { - if d.ClientID == "" || d.ClientSecret == "" { - d.ClientID = "YNxT9w7GMdWvEOKa" - d.ClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + if d.Common == nil { + d.Common = &Common{ + DeviceID: utils.GetMD5EncodeStr(d.Addition.ShareId + d.Addition.SharePwd + time.Now().String()), + UserAgent: "", + RefreshCTokenCk: func(token string) { + d.Common.CaptchaToken = token + op.MustSaveDriverStorage(d) + }, + } } - oauth2Config := &oauth2.Config{ - ClientID: d.ClientID, - ClientSecret: d.ClientSecret, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://user.mypikpak.com/v1/auth/signin", - TokenURL: "https://user.mypikpak.com/v1/auth/token", - AuthStyle: oauth2.AuthStyleInParams, - }, + if d.Addition.DeviceID != "" { + d.SetDeviceID(d.Addition.DeviceID) + } else { + d.Addition.DeviceID = d.Common.DeviceID + op.MustSaveDriverStorage(d) } - d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { - return oauth2Config.PasswordCredentialsToken( - context.WithValue(context.Background(), oauth2.HTTPClient, base.HttpClient), - d.Username, - d.Password, - ) - })) + if d.Platform == "android" { + d.ClientID = AndroidClientID + d.ClientSecret = AndroidClientSecret + d.ClientVersion = AndroidClientVersion + d.PackageName = AndroidPackageName + d.Algorithms = AndroidAlgorithms + d.UserAgent = BuildCustomUserAgent(d.GetDeviceID(), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, "") + } else if d.Platform == "web" { + d.ClientID = WebClientID + d.ClientSecret = WebClientSecret + d.ClientVersion = WebClientVersion + d.PackageName = WebPackageName + d.Algorithms = WebAlgorithms + d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" + } + + // 获取CaptchaToken + err := d.RefreshCaptchaToken(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/share:batch_file_info"), "") + if err != nil { + return err + } if d.SharePwd != "" { return d.getSharePassToken() @@ -87,9 +104,14 @@ func (d *PikPakShare) Link(ctx context.Context, file model.Obj, args model.LinkA downloadUrl := resp.FileInfo.WebContentLink if downloadUrl == "" && len(resp.FileInfo.Medias) > 0 { - downloadUrl = resp.FileInfo.Medias[0].Link.Url - } + // 使用转码后的链接 + if d.Addition.UseTransCodingAddress && len(resp.FileInfo.Medias) > 1 { + downloadUrl = resp.FileInfo.Medias[1].Link.Url + } else { + downloadUrl = resp.FileInfo.Medias[0].Link.Url + } + } link := model.Link{ URL: downloadUrl, } diff --git a/drivers/pikpak_share/meta.go b/drivers/pikpak_share/meta.go index 5d05badb590..e6f00cdad54 100644 --- a/drivers/pikpak_share/meta.go +++ b/drivers/pikpak_share/meta.go @@ -7,12 +7,11 @@ import ( type Addition struct { driver.RootID - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - ShareId string `json:"share_id" required:"true"` - SharePwd string `json:"share_pwd"` - ClientID string `json:"client_id" required:"true" default:"YNxT9w7GMdWvEOKa"` - ClientSecret string `json:"client_secret" required:"true" default:"dbw2OtmVEeuUvIptb1Coyg"` + ShareId string `json:"share_id" required:"true"` + SharePwd string `json:"share_pwd"` + Platform string `json:"platform" required:"true" type:"select" options:"android,web"` + DeviceID string `json:"device_id" required:"false" default:""` + UseTransCodingAddress bool `json:"use_transcoding_address" required:"true" default:"false"` } var config = driver.Config{ diff --git a/drivers/pikpak_share/types.go b/drivers/pikpak_share/types.go index 144a05a8744..78ea2ff8bbc 100644 --- a/drivers/pikpak_share/types.go +++ b/drivers/pikpak_share/types.go @@ -1,20 +1,16 @@ package pikpak_share import ( + "fmt" "strconv" "time" "github.com/alist-org/alist/v3/internal/model" ) -type RespErr struct { - ErrorCode int `json:"error_code"` - Error string `json:"error"` -} - type ShareResp struct { - ShareStatus string `json:"share_status"` - ShareStatusText string `json:"share_status_text"` + ShareStatus string `json:"share_status"` + ShareStatusText string `json:"share_status_text"` FileInfo File `json:"file_info"` Files []File `json:"files"` NextPageToken string `json:"next_page_token"` @@ -78,3 +74,32 @@ type Media struct { IsVisible bool `json:"is_visible"` Category string `json:"category"` } + +type CaptchaTokenRequest struct { + Action string `json:"action"` + CaptchaToken string `json:"captcha_token"` + ClientID string `json:"client_id"` + DeviceID string `json:"device_id"` + Meta map[string]string `json:"meta"` + RedirectUri string `json:"redirect_uri"` +} + +type CaptchaTokenResponse struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Url string `json:"url"` +} + +type ErrResp struct { + ErrorCode int64 `json:"error_code"` + ErrorMsg string `json:"error"` + ErrorDescription string `json:"error_description"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) +} diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index 41bb30d4aa6..a9c8fffe84e 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -1,21 +1,78 @@ package pikpak_share import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" "errors" + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" "net/http" + "regexp" + "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/go-resty/resty/v2" ) +var AndroidAlgorithms = []string{ + "Gez0T9ijiI9WCeTsKSg3SMlx", + "zQdbalsolyb1R/", + "ftOjr52zt51JD68C3s", + "yeOBMH0JkbQdEFNNwQ0RI9T3wU/v", + "BRJrQZiTQ65WtMvwO", + "je8fqxKPdQVJiy1DM6Bc9Nb1", + "niV", + "9hFCW2R1", + "sHKHpe2i96", + "p7c5E6AcXQ/IJUuAEC9W6", + "", + "aRv9hjc9P+Pbn+u3krN6", + "BzStcgE8qVdqjEH16l4", + "SqgeZvL5j9zoHP95xWHt", + "zVof5yaJkPe3VFpadPof", +} + +var WebAlgorithms = []string{ + "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", + "+r6CQVxjzJV6LCV", + "F", + "pFJRC", + "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", + "/750aCr4lm/Sly/c", + "RB+DT/gZCrbV", + "", + "CyLsf7hdkIRxRm215hl", + "7xHvLi2tOYP0Y92b", + "ZGTXXxu8E/MIWaEDB+Sm/", + "1UI3", + "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", + "ihtqpG6FMt65+Xk+tWUH2", + "NhXXU9rg4XXdzo7u5o", +} + +const ( + AndroidClientID = "YNxT9w7GMdWvEOKa" + AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + AndroidClientVersion = "1.47.1" + AndroidPackageName = "com.pikcloud.pikpak" + AndroidSdkVersion = "2.0.4.204000" + WebClientID = "YUMx5nI8ZU8Ap8pm" + WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + WebClientVersion = "2.0.0" + WebPackageName = "mypikpak.com" + WebSdkVersion = "8.0.3" +) + func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() - - token, err := d.oauth2Token.Token() - if err != nil { - return nil, err - } - req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) + req.SetHeaders(map[string]string{ + "User-Agent": d.GetUserAgent(), + "X-Client-ID": d.GetClientID(), + "X-Device-ID": d.GetDeviceID(), + "X-Captcha-Token": d.GetCaptchaToken(), + }) if callback != nil { callback(req) @@ -23,16 +80,25 @@ func (d *PikPakShare) request(url string, method string, callback base.ReqCallba if resp != nil { req.SetResult(resp) } - var e RespErr + var e ErrResp req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } - if e.ErrorCode != 0 { - return nil, errors.New(e.Error) + switch e.ErrorCode { + case 0: + return res.Body(), nil + case 9: // 验证码token过期 + if err = d.RefreshCaptchaToken(GetAction(method, url), ""); err != nil { + return nil, err + } + return d.request(url, method, callback, resp) + case 10: // 操作频繁 + return nil, errors.New(e.ErrorDescription) + default: + return nil, errors.New(e.Error()) } - return res.Body(), nil } func (d *PikPakShare) getSharePassToken() error { @@ -92,3 +158,161 @@ func (d *PikPakShare) getFiles(id string) ([]File, error) { } return res, nil } + +func GetAction(method string, url string) string { + urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] + return method + ":" + urlpath +} + +type Common struct { + client *resty.Client + CaptchaToken string + // 必要值,签名相关 + ClientID string + ClientSecret string + ClientVersion string + PackageName string + Algorithms []string + DeviceID string + UserAgent string + // 验证码token刷新成功回调 + RefreshCTokenCk func(token string) +} + +func (c *Common) SetUserAgent(userAgent string) { + c.UserAgent = userAgent +} + +func (c *Common) SetCaptchaToken(captchaToken string) { + c.CaptchaToken = captchaToken +} + +func (c *Common) SetDeviceID(deviceID string) { + c.DeviceID = deviceID +} + +func (c *Common) GetCaptchaToken() string { + return c.CaptchaToken +} + +func (c *Common) GetClientID() string { + return c.ClientID +} + +func (c *Common) GetUserAgent() string { + return c.UserAgent +} + +func (c *Common) GetDeviceID() string { + return c.DeviceID +} + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} + +func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { + deviceSign := generateDeviceSign(deviceID, packageName) + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) + sb.WriteString("protocolVersion/200 ") + sb.WriteString("accesstype/ ") + sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) + sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) + sb.WriteString("action_type/ ") + sb.WriteString("networktype/WIFI ") + sb.WriteString("sessionid/ ") + sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) + sb.WriteString("providername/NONE ") + sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) + sb.WriteString("refresh_token/ ") + sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) + sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) + sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) + sb.WriteString(fmt.Sprintf("appname/android-%s ", appName)) + sb.WriteString(fmt.Sprintf("session_origin/ ")) + sb.WriteString(fmt.Sprintf("grant_type/ ")) + sb.WriteString(fmt.Sprintf("appid/ ")) + sb.WriteString(fmt.Sprintf("clientip/ ")) + sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) + sb.WriteString(fmt.Sprintf("osversion/13 ")) + sb.WriteString(fmt.Sprintf("platformversion/10 ")) + sb.WriteString(fmt.Sprintf("accessmode/ ")) + sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) + + return sb.String() +} + +// RefreshCaptchaToken 刷新验证码token +func (d *PikPakShare) RefreshCaptchaToken(action, userID string) error { + metas := map[string]string{ + "client_version": d.ClientVersion, + "package_name": d.PackageName, + "user_id": userID, + } + metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() + return d.refreshCaptchaToken(action, metas) +} + +// GetCaptchaSign 获取验证码签名 +func (c *Common) GetCaptchaSign() (timestamp, sign string) { + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) + for _, algorithm := range c.Algorithms { + str = utils.GetMD5EncodeStr(str + algorithm) + } + sign = "1." + str + return +} + +// refreshCaptchaToken 刷新CaptchaToken +func (d *PikPakShare) refreshCaptchaToken(action string, metas map[string]string) error { + param := CaptchaTokenRequest{ + Action: action, + CaptchaToken: d.GetCaptchaToken(), + ClientID: d.ClientID, + DeviceID: d.GetDeviceID(), + Meta: metas, + } + var e ErrResp + var resp CaptchaTokenResponse + _, err := d.request("https://user.mypikpak.com/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + req.SetError(&e).SetBody(param) + }, &resp) + + if err != nil { + return err + } + + if e.IsError() { + return errors.New(e.Error()) + } + + //if resp.Url != "" { + // return fmt.Errorf(`need verify: Click Here`, resp.Url) + //} + + if d.Common.RefreshCTokenCk != nil { + d.Common.RefreshCTokenCk(resp.CaptchaToken) + } + d.Common.SetCaptchaToken(resp.CaptchaToken) + return nil +} From ef5e192c3bb1376a839bebd5eb60b44fc4e4a7e4 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Thu, 22 Aug 2024 00:35:52 +0800 Subject: [PATCH 297/659] fix(pikpak): webdav upload issue (#7050) --- drivers/pikpak/driver.go | 52 ++++---- drivers/pikpak/types.go | 24 ++-- drivers/pikpak/util.go | 249 ++++++++++++++++++++++++++++++++++++++- 3 files changed, 282 insertions(+), 43 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 13b9843015d..e2a2b82e7a7 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -4,24 +4,18 @@ import ( "context" "encoding/json" "fmt" - "github.com/alist-org/alist/v3/internal/op" - "golang.org/x/oauth2" - "io" - "net/http" - "strconv" - "strings" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" + "net/http" + "strconv" + "strings" ) type PikPak struct { @@ -123,10 +117,16 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.AccessToken = token.AccessToken // 获取CaptchaToken - err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Common.UserID) + err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Username) if err != nil { return err } + + // 获取用户ID + userID := token.Extra("sub").(string) + if userID != "" { + d.Common.SetUserID(userID) + } // 更新UserAgent if d.Platform == "android" { d.Common.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(d.Username+d.Password), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, d.Common.UserID) @@ -271,27 +271,17 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } params := resp.Resumable.Params - endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") - cfg := &aws.Config{ - Credentials: credentials.NewStaticCredentials(params.AccessKeyID, params.AccessKeySecret, params.SecurityToken), - Region: aws.String("pikpak"), - Endpoint: &endpoint, + //endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") + // web 端上传 返回的endpoint 为 `mypikpak.com` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.com`· + if d.Addition.Platform == "android" { + params.Endpoint = "mypikpak.com" } - ss, err := session.NewSession(cfg) - if err != nil { - return err - } - uploader := s3manager.NewUploader(ss) - if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) - } - input := &s3manager.UploadInput{ - Bucket: ¶ms.Bucket, - Key: ¶ms.Key, - Body: io.TeeReader(stream, driver.NewProgress(stream.GetSize(), up)), + + if stream.GetSize() <= 10*utils.MB { // 文件大小 小于10MB，改用普通模式上传 + return d.UploadByOSS(¶ms, stream, up) } - _, err = uploader.UploadWithContext(ctx, input) - return err + // 分片上传 + return d.UploadByMultipart(¶ms, stream.GetSize(), stream, up) } // 离线下载文件 diff --git a/drivers/pikpak/types.go b/drivers/pikpak/types.go index b27b905568a..2a959ebf05d 100644 --- a/drivers/pikpak/types.go +++ b/drivers/pikpak/types.go @@ -80,22 +80,24 @@ type UploadTaskData struct { UploadType string `json:"upload_type"` //UPLOAD_TYPE_RESUMABLE Resumable *struct { - Kind string `json:"kind"` - Params struct { - AccessKeyID string `json:"access_key_id"` - AccessKeySecret string `json:"access_key_secret"` - Bucket string `json:"bucket"` - Endpoint string `json:"endpoint"` - Expiration time.Time `json:"expiration"` - Key string `json:"key"` - SecurityToken string `json:"security_token"` - } `json:"params"` - Provider string `json:"provider"` + Kind string `json:"kind"` + Params S3Params `json:"params"` + Provider string `json:"provider"` } `json:"resumable"` File File `json:"file"` } +type S3Params struct { + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + Bucket string `json:"bucket"` + Endpoint string `json:"endpoint"` + Expiration time.Time `json:"expiration"` + Key string `json:"key"` + SecurityToken string `json:"security_token"` +} + // 添加离线下载响应 type OfflineDownloadResp struct { File *string `json:"file"` diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 902a9d32191..d3371a2558a 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -1,17 +1,24 @@ package pikpak import ( + "bytes" "crypto/md5" "crypto/sha1" "encoding/hex" - "errors" "fmt" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" + "io" "net/http" + "path/filepath" "regexp" "strings" + "sync" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -56,6 +63,12 @@ var WebAlgorithms = []string{ "NhXXU9rg4XXdzo7u5o", } +const ( + OSSUserAgent = "aliyun-sdk-android/2.9.13(Linux/Android 14/M2004j7ac;UKQ1.231108.001)" + OssSecurityTokenHeaderName = "X-OSS-Security-Token" + ThreadsNum = 10 +) + const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" @@ -393,3 +406,237 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err d.Common.SetCaptchaToken(resp.CaptchaToken) return nil } + +func (d *PikPak) UploadByOSS(params *S3Params, stream model.FileStreamer, up driver.UpdateProgress) error { + ossClient, err := oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret) + if err != nil { + return err + } + bucket, err := ossClient.Bucket(params.Bucket) + if err != nil { + return err + } + + err = bucket.PutObject(params.Key, stream, OssOption(params)...) + if err != nil { + return err + } + return nil +} + +func (d *PikPak) UploadByMultipart(params *S3Params, fileSize int64, stream model.FileStreamer, up driver.UpdateProgress) error { + var ( + chunks []oss.FileChunk + parts []oss.UploadPart + imur oss.InitiateMultipartUploadResult + ossClient *oss.Client + bucket *oss.Bucket + err error + ) + + tmpF, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + + if ossClient, err = oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret); err != nil { + return err + } + + if bucket, err = ossClient.Bucket(params.Bucket); err != nil { + return err + } + + ticker := time.NewTicker(time.Hour * 12) + defer ticker.Stop() + // 设置超时 + timeout := time.NewTimer(time.Hour * 24) + + if chunks, err = SplitFile(fileSize); err != nil { + return err + } + + if imur, err = bucket.InitiateMultipartUpload(params.Key, + oss.SetHeader(OssSecurityTokenHeaderName, params.SecurityToken), + oss.UserAgentHeader(OSSUserAgent), + ); err != nil { + return err + } + + wg := sync.WaitGroup{} + wg.Add(len(chunks)) + + chunksCh := make(chan oss.FileChunk) + errCh := make(chan error) + UploadedPartsCh := make(chan oss.UploadPart) + quit := make(chan struct{}) + + // producer + go chunksProducer(chunksCh, chunks) + go func() { + wg.Wait() + quit <- struct{}{} + }() + + // consumers + for i := 0; i < ThreadsNum; i++ { + go func(threadId int) { + defer func() { + if r := recover(); r != nil { + errCh <- fmt.Errorf("recovered in %v", r) + } + }() + for chunk := range chunksCh { + var part oss.UploadPart // 出现错误就继续尝试，共尝试3次 + for retry := 0; retry < 3; retry++ { + select { + case <-ticker.C: + errCh <- errors.Wrap(err, "ossToken 过期") + default: + } + + buf := make([]byte, chunk.Size) + if _, err = tmpF.ReadAt(buf, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { + continue + } + + b := bytes.NewBuffer(buf) + if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, OssOption(params)...); err == nil { + break + } + } + if err != nil { + errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误：%v", stream.GetName(), chunk.Number, err)) + } + UploadedPartsCh <- part + } + }(i) + } + + go func() { + for part := range UploadedPartsCh { + parts = append(parts, part) + wg.Done() + } + }() +LOOP: + for { + select { + case <-ticker.C: + // ossToken 过期 + return err + case <-quit: + break LOOP + case <-errCh: + return err + case <-timeout.C: + return fmt.Errorf("time out") + } + } + + // EOF错误是xml的Unmarshal导致的，响应其实是json格式，所以实际上上传是成功的 + if _, err = bucket.CompleteMultipartUpload(imur, parts, OssOption(params)...); err != nil && !errors.Is(err, io.EOF) { + // 当文件名含有 &< 这两个字符之一时响应的xml解析会出现错误，实际上上传是成功的 + if filename := filepath.Base(stream.GetName()); !strings.ContainsAny(filename, "&<") { + return err + } + } + return nil +} + +func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { + for _, chunk := range chunks { + ch <- chunk + } +} + +func SplitFile(fileSize int64) (chunks []oss.FileChunk, err error) { + for i := int64(1); i < 10; i++ { + if fileSize < i*utils.GB { // 文件大小小于iGB时分为i*100片 + if chunks, err = SplitFileByPartNum(fileSize, int(i*100)); err != nil { + return + } + break + } + } + if fileSize > 9*utils.GB { // 文件大小大于9GB时分为1000片 + if chunks, err = SplitFileByPartNum(fileSize, 1000); err != nil { + return + } + } + // 单个分片大小不能小于1MB + if chunks[0].Size < 1*utils.MB { + if chunks, err = SplitFileByPartSize(fileSize, 1*utils.MB); err != nil { + return + } + } + return +} + +// SplitFileByPartNum splits big file into parts by the num of parts. +// Split the file with specified parts count, returns the split result when error is nil. +func SplitFileByPartNum(fileSize int64, chunkNum int) ([]oss.FileChunk, error) { + if chunkNum <= 0 || chunkNum > 10000 { + return nil, errors.New("chunkNum invalid") + } + + if int64(chunkNum) > fileSize { + return nil, errors.New("oss: chunkNum invalid") + } + + var chunks []oss.FileChunk + chunk := oss.FileChunk{} + chunkN := (int64)(chunkNum) + for i := int64(0); i < chunkN; i++ { + chunk.Number = int(i + 1) + chunk.Offset = i * (fileSize / chunkN) + if i == chunkN-1 { + chunk.Size = fileSize/chunkN + fileSize%chunkN + } else { + chunk.Size = fileSize / chunkN + } + chunks = append(chunks, chunk) + } + + return chunks, nil +} + +// SplitFileByPartSize splits big file into parts by the size of parts. +// Splits the file by the part size. Returns the FileChunk when error is nil. +func SplitFileByPartSize(fileSize int64, chunkSize int64) ([]oss.FileChunk, error) { + if chunkSize <= 0 { + return nil, errors.New("chunkSize invalid") + } + + chunkN := fileSize / chunkSize + if chunkN >= 10000 { + return nil, errors.New("Too many parts, please increase part size") + } + + var chunks []oss.FileChunk + chunk := oss.FileChunk{} + for i := int64(0); i < chunkN; i++ { + chunk.Number = int(i + 1) + chunk.Offset = i * chunkSize + chunk.Size = chunkSize + chunks = append(chunks, chunk) + } + + if fileSize%chunkSize > 0 { + chunk.Number = len(chunks) + 1 + chunk.Offset = int64(len(chunks)) * chunkSize + chunk.Size = fileSize % chunkSize + chunks = append(chunks, chunk) + } + + return chunks, nil +} + +// OssOption get options +func OssOption(params *S3Params) []oss.Option { + options := []oss.Option{ + oss.SetHeader(OssSecurityTokenHeaderName, params.SecurityToken), + oss.UserAgentHeader(OSSUserAgent), + } + return options +} From 0715198c7f43ae990766ac00d307cb15469ded39 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Thu, 22 Aug 2024 00:42:19 +0800 Subject: [PATCH 298/659] chore: fix log format typo (#7056) Signed-off-by: cuishuang --- cmd/lang.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/lang.go b/cmd/lang.go index 8d816ca2b30..56ef037ba24 100644 --- a/cmd/lang.go +++ b/cmd/lang.go @@ -139,7 +139,7 @@ var LangCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { err := os.MkdirAll("lang", 0777) if err != nil { - utils.Log.Fatal("failed create folder: %s", err.Error()) + utils.Log.Fatalf("failed create folder: %s", err.Error()) } generateDriversJson() generateSettingsJson() From d9a1809313bbd20f41378597fe12c2880e488977 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 00:42:50 +0800 Subject: [PATCH 299/659] fix(deps): update module github.com/charmbracelet/lipgloss to v0.13.0 (#7049) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 7e70148e9e2..e011d882851 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.27.0 - github.com/charmbracelet/lipgloss v0.12.1 + github.com/charmbracelet/lipgloss v0.13.0 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e github.com/coreos/go-oidc v2.2.1+incompatible github.com/deckarep/golang-set/v2 v2.6.0 diff --git a/go.sum b/go.sum index 6b1041b6309..48151219fcd 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,8 @@ github.com/charmbracelet/bubbletea v0.27.0 h1:Mznj+vvYuYagD9Pn2mY7fuelGvP0HAXtZY github.com/charmbracelet/bubbletea v0.27.0/go.mod h1:5MdP9XH6MbQkgGhnlxUqCNmBXf9I74KRQ8HIidRxV1Y= github.com/charmbracelet/lipgloss v0.12.1 h1:/gmzszl+pedQpjCOH+wFkZr/N90Snz40J/NR7A0zQcs= github.com/charmbracelet/lipgloss v0.12.1/go.mod h1:V2CiwIuhx9S1S1ZlADfOj9HmxeMAORuz5izHb0zGbB8= +github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= +github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/input v0.1.0 h1:TEsGSfZYQyOtp+STIjyBq6tpRaorH0qpwZUj8DavAhQ= From db1494455dc0240c9d02a1c2167db7417516daed Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 00:43:11 +0800 Subject: [PATCH 300/659] fix(deps): update module github.com/charmbracelet/bubbles to v0.19.0 (#7048) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index e011d882851..24f2a0ac13f 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/aws/aws-sdk-go v1.55.5 github.com/blevesearch/bleve/v2 v2.4.2 github.com/caarlos0/env/v9 v9.0.0 - github.com/charmbracelet/bubbles v0.18.0 + github.com/charmbracelet/bubbles v0.19.0 github.com/charmbracelet/bubbletea v0.27.0 github.com/charmbracelet/lipgloss v0.13.0 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e @@ -169,7 +169,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/go.sum b/go.sum index 48151219fcd..65ca68e34e3 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= +github.com/charmbracelet/bubbles v0.19.0 h1:gKZkKXPP6GlDk6EcfujDK19PCQqRjaJZQ7QRERx1UF0= +github.com/charmbracelet/bubbles v0.19.0/go.mod h1:WILteEqZ+krG5c3ntGEMeG99nCupcuIk7V0/zOP0tOA= github.com/charmbracelet/bubbletea v0.26.6 h1:zTCWSuST+3yZYZnVSvbXwKOPRSNZceVeqpzOLN2zq1s= github.com/charmbracelet/bubbletea v0.26.6/go.mod h1:dz8CWPlfCCGLFbBlTY4N7bjLiyOGDJEnd2Muu7pOWhk= github.com/charmbracelet/bubbletea v0.27.0 h1:Mznj+vvYuYagD9Pn2mY7fuelGvP0HAXtZYGgRBCbHvU= @@ -350,6 +352,8 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.27.2 h1:3G21dJ5i208shnLPDsIEZ0L0Geg/5oeXABFV7nlK94k= From bcb24d61eaf0aabff851ad5d4e203f9553ee83bd Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 00:43:35 +0800 Subject: [PATCH 301/659] fix(deps): update module github.com/go-resty/resty/v2 to v2.14.0 (#6981) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 24f2a0ac13f..4dcf664c446 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/gaoyb7/115drive-webdav v0.1.8 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 - github.com/go-resty/resty/v2 v2.13.1 + github.com/go-resty/resty/v2 v2.14.0 github.com/go-webauthn/webauthn v0.11.1 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum index 65ca68e34e3..fdffd2ef124 100644 --- a/go.sum +++ b/go.sum @@ -204,6 +204,8 @@ github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaC github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= +github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= +github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-webauthn/webauthn v0.10.2 h1:OG7B+DyuTytrEPFmTX503K77fqs3HDK/0Iv+z8UYbq4= @@ -564,8 +566,10 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= @@ -579,6 +583,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -592,8 +599,10 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= @@ -603,6 +612,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -631,20 +643,25 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -656,8 +673,10 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -673,6 +692,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 74887922b4735d6aaf4cfbdfbbb0b8f1342620fb Mon Sep 17 00:00:00 2001 From: Wang Xiaoqing Date: Thu, 22 Aug 2024 00:44:23 +0800 Subject: [PATCH 302/659] fix(offline_download): os.create failure while the name of downloaded file is empty (#7041) --- internal/offline_download/http/client.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/internal/offline_download/http/client.go b/internal/offline_download/http/client.go index 0db05f35c15..6f22fcf7b98 100644 --- a/internal/offline_download/http/client.go +++ b/internal/offline_download/http/client.go @@ -2,14 +2,16 @@ package http import ( "fmt" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/offline_download/tool" - "github.com/alist-org/alist/v3/pkg/utils" "net/http" "net/url" "os" "path" "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/pkg/utils" ) type SimpleHttp struct { @@ -63,7 +65,12 @@ func (s SimpleHttp) Run(task *tool.DownloadTask) error { if resp.StatusCode >= 400 { return fmt.Errorf("http status code %d", resp.StatusCode) } - filename := path.Base(_u.Path) + // If Path is empty, use Hostname; otherwise, filePath euqals TempDir which causes os.Create to fail + urlPath := _u.Path + if urlPath == "" { + urlPath = strings.ReplaceAll(_u.Host, ".", "_") + } + filename := path.Base(urlPath) if n, err := parseFilenameFromContentDisposition(resp.Header.Get("Content-Disposition")); err == nil { filename = n } From 48f50a2cebfc3d9469338ec7a18aa8854d6b50f2 Mon Sep 17 00:00:00 2001 From: Rammiah Date: Thu, 22 Aug 2024 00:44:55 +0800 Subject: [PATCH 303/659] fix(search): BuildIndex concurrency error (#7035) --- internal/errs/search.go | 3 ++- internal/op/fs.go | 4 +-- internal/search/build.go | 52 ++++++++++++++++++++++++++++++--------- internal/search/search.go | 2 +- pkg/mq/mq.go | 2 ++ server/handles/index.go | 14 +++++++---- 6 files changed, 55 insertions(+), 22 deletions(-) diff --git a/internal/errs/search.go b/internal/errs/search.go index 9c864f4d241..3da92898e65 100644 --- a/internal/errs/search.go +++ b/internal/errs/search.go @@ -3,5 +3,6 @@ package errs import "fmt" var ( - SearchNotAvailable = fmt.Errorf("search not available") + SearchNotAvailable = fmt.Errorf("search not available") + BuildIndexIsRunning = fmt.Errorf("build index is running, please try later") ) diff --git a/internal/op/fs.go b/internal/op/fs.go index ed28039529f..e0153952638 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -136,9 +136,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li model.WrapObjsName(files) // call hooks go func(reqPath string, files []model.Obj) { - for _, hook := range objsUpdateHooks { - hook(reqPath, files) - } + HandleObjsUpdateHook(reqPath, files) }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) // sort objs diff --git a/internal/search/build.go b/internal/search/build.go index 1d3bfb7cd5d..9865b2988a1 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -5,10 +5,12 @@ import ( "path" "path/filepath" "strings" + "sync" "sync/atomic" "time" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" @@ -21,10 +23,13 @@ import ( ) var ( - Running = atomic.Bool{} - Quit chan struct{} + Quit = atomic.Pointer[chan struct{}]{} ) +func Running() bool { + return Quit.Load() != nil +} + func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error { var ( err error @@ -33,11 +38,27 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth ) log.Infof("build index for: %+v", indexPaths) log.Infof("ignore paths: %+v", ignorePaths) - Running.Store(true) - Quit = make(chan struct{}, 1) - indexMQ := mq.NewInMemoryMQ[ObjWithParent]() + quit := make(chan struct{}, 1) + if !Quit.CompareAndSwap(nil, &quit) { + // other goroutine is running + return errs.BuildIndexIsRunning + } + var ( + indexMQ = mq.NewInMemoryMQ[ObjWithParent]() + running = atomic.Bool{} // current goroutine running + wg = &sync.WaitGroup{} + ) + running.Store(true) + wg.Add(1) go func() { ticker := time.NewTicker(time.Second) + defer func() { + Quit.Store(nil) + wg.Done() + // notify walk to exit when StopIndex api called + running.Store(false) + ticker.Stop() + }() tickCount := 0 for { select { @@ -70,9 +91,8 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth } }) - case <-Quit: - Running.Store(false) - ticker.Stop() + case <-quit: + log.Debugf("build index for %+v received quit", indexPaths) eMsg := "" now := time.Now() originErr := err @@ -100,14 +120,22 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth }) } }) + log.Debugf("build index for %+v quit success", indexPaths) return } } }() defer func() { - if Running.Load() { - Quit <- struct{}{} + if !running.Load() || Quit.Load() != &quit { + log.Debugf("build index for %+v stopped by StopIndex", indexPaths) + return + } + select { + // avoid goroutine leak + case quit <- struct{}{}: + default: } + wg.Wait() }() admin, err := op.GetAdmin() if err != nil { @@ -121,7 +149,7 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth } for _, indexPath := range indexPaths { walkFn := func(indexPath string, info model.Obj) error { - if !Running.Load() { + if !running.Load() { return filepath.SkipDir } for _, avoidPath := range ignorePaths { @@ -167,7 +195,7 @@ func Config(ctx context.Context) searcher.Config { } func Update(parent string, objs []model.Obj) { - if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running.Load() { + if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() { return } if isIgnorePath(parent) { diff --git a/internal/search/search.go b/internal/search/search.go index c1a23b85ca8..d420eb16dd3 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -27,7 +27,7 @@ func Init(mode string) error { } instance = nil } - if Running.Load() { + if Running() { return fmt.Errorf("index is running") } if mode == "none" { diff --git a/pkg/mq/mq.go b/pkg/mq/mq.go index 35f5a159de4..cdae0425130 100644 --- a/pkg/mq/mq.go +++ b/pkg/mq/mq.go @@ -57,5 +57,7 @@ func (mq *inMemoryMQ[T]) Clear() { } func (mq *inMemoryMQ[T]) Len() int { + mq.Lock() + defer mq.Unlock() return mq.queue.Len() } diff --git a/server/handles/index.go b/server/handles/index.go index 0fa1fa0e9bf..5610688da1f 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -19,7 +19,7 @@ type UpdateIndexReq struct { } func BuildIndex(c *gin.Context) { - if search.Running.Load() { + if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } @@ -45,7 +45,7 @@ func UpdateIndex(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if search.Running.Load() { + if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } @@ -72,16 +72,20 @@ func UpdateIndex(c *gin.Context) { } func StopIndex(c *gin.Context) { - if !search.Running.Load() { + quit := search.Quit.Load() + if quit == nil { common.ErrorStrResp(c, "index is not running", 400) return } - search.Quit <- struct{}{} + select { + case *quit <- struct{}{}: + default: + } common.SuccessResp(c) } func ClearIndex(c *gin.Context) { - if search.Running.Load() { + if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } From 34b6785fabe831b23808baa49eec0a3bab6d6975 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 22 Aug 2024 00:45:31 +0800 Subject: [PATCH 304/659] fix(deps): update module github.com/meilisearch/meilisearch-go to v0.28.0 (#7061) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4dcf664c446..c2f3ec3ff3e 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/larksuite/oapi-sdk-go/v3 v3.3.1 github.com/maruel/natural v1.1.1 - github.com/meilisearch/meilisearch-go v0.27.2 + github.com/meilisearch/meilisearch-go v0.28.0 github.com/minio/sio v0.4.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.2 diff --git a/go.sum b/go.sum index fdffd2ef124..d3975001b1f 100644 --- a/go.sum +++ b/go.sum @@ -360,6 +360,8 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.27.2 h1:3G21dJ5i208shnLPDsIEZ0L0Geg/5oeXABFV7nlK94k= github.com/meilisearch/meilisearch-go v0.27.2/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= +github.com/meilisearch/meilisearch-go v0.28.0 h1:f3XJ66ZM+R8bANAOLqsjvoq/HhQNpVJPYoNt6QgNzME= +github.com/meilisearch/meilisearch-go v0.28.0/go.mod h1:Szcc9CaDiKIfjdgdt49jlmDKpEzjD+x+b6Y6heMdlQ0= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= From d2514d236ffa93e30d3cb22ec9df4d03e10e1000 Mon Sep 17 00:00:00 2001 From: ice yao Date: Thu, 22 Aug 2024 00:46:38 +0800 Subject: [PATCH 305/659] feat(drivers): add kodbox storage (#7059 close #7058) - kodbox: https://github.com/kalcaddle/kodbox --- drivers/all.go | 1 + drivers/kodbox/driver.go | 273 +++++++++++++++++++++++++++++++++++++++ drivers/kodbox/meta.go | 25 ++++ drivers/kodbox/types.go | 24 ++++ drivers/kodbox/util.go | 86 ++++++++++++ 5 files changed, 409 insertions(+) create mode 100644 drivers/kodbox/driver.go create mode 100644 drivers/kodbox/meta.go create mode 100644 drivers/kodbox/types.go create mode 100644 drivers/kodbox/util.go diff --git a/drivers/all.go b/drivers/all.go index 1f015ef7d61..40062a1aea1 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -28,6 +28,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/halalcloud" _ "github.com/alist-org/alist/v3/drivers/ilanzou" _ "github.com/alist-org/alist/v3/drivers/ipfs_api" + _ "github.com/alist-org/alist/v3/drivers/kodbox" _ "github.com/alist-org/alist/v3/drivers/lanzou" _ "github.com/alist-org/alist/v3/drivers/lenovonas_share" _ "github.com/alist-org/alist/v3/drivers/local" diff --git a/drivers/kodbox/driver.go b/drivers/kodbox/driver.go new file mode 100644 index 00000000000..eb5120a67c1 --- /dev/null +++ b/drivers/kodbox/driver.go @@ -0,0 +1,273 @@ +package kodbox + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) + +type KodBox struct { + model.Storage + Addition + authorization string +} + +func (d *KodBox) Config() driver.Config { + return config +} + +func (d *KodBox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *KodBox) Init(ctx context.Context) error { + d.Address = strings.TrimSuffix(d.Address, "/") + d.RootFolderPath = strings.TrimPrefix(utils.FixAndCleanPath(d.RootFolderPath), "/") + return d.getToken() +} + +func (d *KodBox) Drop(ctx context.Context) error { + return nil +} + +func (d *KodBox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var ( + resp *CommonResp + listPathData *ListPathData + ) + + _, err := d.request(http.MethodPost, "/?explorer/list/path", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "path": dir.GetPath(), + }) + }, true) + if err != nil { + return nil, err + } + + dataBytes, err := utils.Json.Marshal(resp.Data) + if err != nil { + return nil, err + } + + err = utils.Json.Unmarshal(dataBytes, &listPathData) + if err != nil { + return nil, err + } + FolderAndFiles := append(listPathData.FolderList, listPathData.FileList...) + + return utils.SliceConvert(FolderAndFiles, func(f FolderOrFile) (model.Obj, error) { + return &model.ObjThumb{ + Object: model.Object{ + Path: f.Path, + Name: f.Name, + Ctime: time.Unix(f.CreateTime, 0), + Modified: time.Unix(f.ModifyTime, 0), + Size: f.Size, + IsFolder: f.Type == "folder", + }, + //Thumbnail: model.Thumbnail{}, + }, nil + }) +} + +func (d *KodBox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + path := file.GetPath() + return &model.Link{ + URL: fmt.Sprintf("%s/?explorer/index/fileOut&path=%s&download=1&accessToken=%s", + d.Address, + path, + d.authorization)}, nil +} + +func (d *KodBox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var resp *CommonResp + newDirPath := filepath.Join(parentDir.GetPath(), dirName) + + _, err := d.request(http.MethodPost, "/?explorer/index/mkdir", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "path": newDirPath, + }) + }) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + + return &model.ObjThumb{ + Object: model.Object{ + Path: resp.Info.(string), + Name: dirName, + IsFolder: true, + Modified: time.Now(), + Ctime: time.Now(), + }, + }, nil +} + +func (d *KodBox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathCuteTo", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", + srcObj.GetPath(), + srcObj.GetName()), + "path": dstDir.GetPath(), + }) + }, true) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + + return &model.ObjThumb{ + Object: model.Object{ + Path: srcObj.GetPath(), + Name: srcObj.GetName(), + IsFolder: srcObj.IsDir(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + }, + }, nil +} + +func (d *KodBox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathRename", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "path": srcObj.GetPath(), + "newName": newName, + }) + }, true) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + return &model.ObjThumb{ + Object: model.Object{ + Path: srcObj.GetPath(), + Name: newName, + IsFolder: srcObj.IsDir(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + }, + }, nil +} + +func (d *KodBox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathCopyTo", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", + srcObj.GetPath(), + srcObj.GetName()), + "path": dstDir.GetPath(), + }) + }) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + + path := resp.Info.([]interface{})[0].(string) + objectName, err := d.getFileOrFolderName(ctx, path) + if err != nil { + return nil, err + } + return &model.ObjThumb{ + Object: model.Object{ + Path: path, + Name: *objectName, + IsFolder: srcObj.IsDir(), + Modified: time.Now(), + Ctime: time.Now(), + }, + }, nil +} + +func (d *KodBox) Remove(ctx context.Context, obj model.Obj) error { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathDelete", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", + obj.GetPath(), + obj.GetName()), + "shiftDelete": "1", + }) + }) + if err != nil { + return err + } + code := resp.Code.(bool) + if !code { + return fmt.Errorf("%s", resp.Data) + } + return nil +} + +func (d *KodBox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/upload/fileUpload", func(req *resty.Request) { + req.SetFileReader("file", stream.GetName(), stream). + SetResult(&resp). + SetFormData(map[string]string{ + "path": dstDir.GetPath(), + }) + }) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + return &model.ObjThumb{ + Object: model.Object{ + Path: resp.Info.(string), + Name: stream.GetName(), + Size: stream.GetSize(), + IsFolder: false, + Modified: time.Now(), + Ctime: time.Now(), + }, + }, nil +} + +func (d *KodBox) getFileOrFolderName(ctx context.Context, path string) (*string, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathInfo", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "dataArr": fmt.Sprintf("[{\"path\": \"%s\"}]", path)}) + }) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + folderOrFileName := resp.Data.(map[string]any)["name"].(string) + return &folderOrFileName, nil +} + +var _ driver.Driver = (*KodBox)(nil) diff --git a/drivers/kodbox/meta.go b/drivers/kodbox/meta.go new file mode 100644 index 00000000000..318fb9ec56f --- /dev/null +++ b/drivers/kodbox/meta.go @@ -0,0 +1,25 @@ +package kodbox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + + Address string `json:"address" required:"true"` + UserName string `json:"username" required:"false"` + Password string `json:"password" required:"false"` +} + +var config = driver.Config{ + Name: "KodBox", + DefaultRoot: "", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &KodBox{} + }) +} diff --git a/drivers/kodbox/types.go b/drivers/kodbox/types.go new file mode 100644 index 00000000000..9bd45d9b366 --- /dev/null +++ b/drivers/kodbox/types.go @@ -0,0 +1,24 @@ +package kodbox + +type CommonResp struct { + Code any `json:"code"` + TimeUse string `json:"timeUse"` + TimeNow string `json:"timeNow"` + Data any `json:"data"` + Info any `json:"info"` +} + +type ListPathData struct { + FolderList []FolderOrFile `json:"folderList"` + FileList []FolderOrFile `json:"fileList"` +} + +type FolderOrFile struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + Ext string `json:"ext,omitempty"` // 文件特有字段 + Size int64 `json:"size"` + CreateTime int64 `json:"createTime"` + ModifyTime int64 `json:"modifyTime"` +} diff --git a/drivers/kodbox/util.go b/drivers/kodbox/util.go new file mode 100644 index 00000000000..2c04cd73f29 --- /dev/null +++ b/drivers/kodbox/util.go @@ -0,0 +1,86 @@ +package kodbox + +import ( + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "strings" +) + +func (d *KodBox) getToken() error { + var authResp CommonResp + res, err := base.RestyClient.R(). + SetResult(&authResp). + SetQueryParams(map[string]string{ + "name": d.UserName, + "password": d.Password, + }). + Post(d.Address + "/?user/index/loginSubmit") + if err != nil { + return err + } + if res.StatusCode() >= 400 { + return fmt.Errorf("get token failed: %s", res.String()) + } + + if res.StatusCode() == 200 && authResp.Code.(bool) == false { + return fmt.Errorf("get token failed: %s", res.String()) + } + + d.authorization = fmt.Sprintf("%s", authResp.Info) + return nil +} + +func (d *KodBox) request(method string, pathname string, callback base.ReqCallback, noRedirect ...bool) ([]byte, error) { + full := pathname + if !strings.HasPrefix(pathname, "http") { + full = d.Address + pathname + } + req := base.RestyClient.R() + if len(noRedirect) > 0 && noRedirect[0] { + req = base.NoRedirectClient.R() + } + req.SetFormData(map[string]string{ + "accessToken": d.authorization, + }) + callback(req) + + var ( + res *resty.Response + commonResp *CommonResp + err error + skip bool + ) + for i := 0; i < 2; i++ { + if skip { + break + } + res, err = req.Execute(method, full) + if err != nil { + return nil, err + } + + err := utils.Json.Unmarshal(res.Body(), &commonResp) + if err != nil { + return nil, err + } + + switch commonResp.Code.(type) { + case bool: + skip = true + case string: + if commonResp.Code.(string) == "10001" { + err = d.getToken() + if err != nil { + return nil, err + } + req.SetFormData(map[string]string{"accessToken": d.authorization}) + } + } + } + if commonResp.Code.(bool) == false { + return nil, fmt.Errorf("request failed: %s", commonResp.Data) + } + return res.Body(), nil +} From e21edf98e2f1eb23819e06733fff5dc82db655b4 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 21 Aug 2024 17:08:03 +0000 Subject: [PATCH 306/659] revert: 34b6785fabe831b23808baa49eec0a3bab6d6975 --- go.mod | 2 +- go.sum | 33 ++++----------------------------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index c2f3ec3ff3e..4dcf664c446 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/larksuite/oapi-sdk-go/v3 v3.3.1 github.com/maruel/natural v1.1.1 - github.com/meilisearch/meilisearch-go v0.28.0 + github.com/meilisearch/meilisearch-go v0.27.2 github.com/minio/sio v0.4.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.2 diff --git a/go.sum b/go.sum index d3975001b1f..ac0ecee759f 100644 --- a/go.sum +++ b/go.sum @@ -32,14 +32,14 @@ github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHG github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI= -github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -96,20 +96,16 @@ github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/bubbles v0.18.0 h1:PYv1A036luoBGroX6VWjQIE9Syf2Wby2oOl/39KLfy0= -github.com/charmbracelet/bubbles v0.18.0/go.mod h1:08qhZhtIwzgrtBjAcJnij1t1H0ZRjwHyGsy6AL11PSw= github.com/charmbracelet/bubbles v0.19.0 h1:gKZkKXPP6GlDk6EcfujDK19PCQqRjaJZQ7QRERx1UF0= github.com/charmbracelet/bubbles v0.19.0/go.mod h1:WILteEqZ+krG5c3ntGEMeG99nCupcuIk7V0/zOP0tOA= -github.com/charmbracelet/bubbletea v0.26.6 h1:zTCWSuST+3yZYZnVSvbXwKOPRSNZceVeqpzOLN2zq1s= -github.com/charmbracelet/bubbletea v0.26.6/go.mod h1:dz8CWPlfCCGLFbBlTY4N7bjLiyOGDJEnd2Muu7pOWhk= github.com/charmbracelet/bubbletea v0.27.0 h1:Mznj+vvYuYagD9Pn2mY7fuelGvP0HAXtZYGgRBCbHvU= github.com/charmbracelet/bubbletea v0.27.0/go.mod h1:5MdP9XH6MbQkgGhnlxUqCNmBXf9I74KRQ8HIidRxV1Y= -github.com/charmbracelet/lipgloss v0.12.1 h1:/gmzszl+pedQpjCOH+wFkZr/N90Snz40J/NR7A0zQcs= -github.com/charmbracelet/lipgloss v0.12.1/go.mod h1:V2CiwIuhx9S1S1ZlADfOj9HmxeMAORuz5izHb0zGbB8= github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q= +github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/charmbracelet/x/input v0.1.0 h1:TEsGSfZYQyOtp+STIjyBq6tpRaorH0qpwZUj8DavAhQ= github.com/charmbracelet/x/input v0.1.0/go.mod h1:ZZwaBxPF7IG8gWWzPUVqHEtWhc1+HXJPNuerJGRGZ28= github.com/charmbracelet/x/term v0.1.1 h1:3cosVAiPOig+EV4X9U+3LDgtwwAoEzJjNdwbXDjF6yI= @@ -118,8 +114,6 @@ github.com/charmbracelet/x/windows v0.1.0 h1:gTaxdvzDM5oMa/I2ZNF7wN78X/atWemG9Wp github.com/charmbracelet/x/windows v0.1.0/go.mod h1:GLEO/l+lizvFDBPLIOk+49gdX49L9YWMB5t+DZd0jkQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= -github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d h1:p5T6ZPvh7nihJfjI9M/W2cbcX7n766u/OGorLmE4xoQ= -github.com/city404/v6-public-rpc-proto/go v0.0.0-20240708163039-9a9b82a0ce4d/go.mod h1:akxZg8LuwOIeCPRjcDrUS1WWcIwmLNSR2lfe4y85PH4= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= @@ -161,8 +155,6 @@ github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5 github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA= -github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= @@ -202,18 +194,12 @@ github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4 github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= -github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g= -github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0= github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-webauthn/webauthn v0.10.2 h1:OG7B+DyuTytrEPFmTX503K77fqs3HDK/0Iv+z8UYbq4= -github.com/go-webauthn/webauthn v0.10.2/go.mod h1:Gd1IDsGAybuvK1NkwUTLbGmeksxuRJjVN2PE/xsPxHs= github.com/go-webauthn/webauthn v0.11.1 h1:5G/+dg91/VcaJHTtJUfwIlNJkLwbJCcnUc4W8VtkpzA= github.com/go-webauthn/webauthn v0.11.1/go.mod h1:YXRm1WG0OtUyDFaVAgB5KG7kVqW+6dYCJ7FTQH4SxEE= -github.com/go-webauthn/x v0.1.9 h1:v1oeLmoaa+gPOaZqUdDentu6Rl7HkSSsmOT6gxEQHhE= -github.com/go-webauthn/x v0.1.9/go.mod h1:pJNMlIMP1SU7cN8HNlKJpLEnFHCygLCvaLZ8a1xeoQA= github.com/go-webauthn/x v0.1.12 h1:RjQ5cvApzyU/xLCiP+rub0PE4HBZsLggbxGR5ZpUf/A= github.com/go-webauthn/x v0.1.12/go.mod h1:XlRcGkNH8PT45TfeJYc6gqpOtiOendHhVmnOxh+5yHs= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= @@ -240,8 +226,6 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= -github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -352,16 +336,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.27.2 h1:3G21dJ5i208shnLPDsIEZ0L0Geg/5oeXABFV7nlK94k= github.com/meilisearch/meilisearch-go v0.27.2/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= -github.com/meilisearch/meilisearch-go v0.28.0 h1:f3XJ66ZM+R8bANAOLqsjvoq/HhQNpVJPYoNt6QgNzME= -github.com/meilisearch/meilisearch-go v0.28.0/go.mod h1:Szcc9CaDiKIfjdgdt49jlmDKpEzjD+x+b6Y6heMdlQ0= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= @@ -577,8 +557,6 @@ golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn5 golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.18.0 h1:jGzIakQa/ZXI1I0Fxvaa9W7yP25TqT6cHIHn+6CqvSQ= -golang.org/x/image v0.18.0/go.mod h1:4yyo5vMFQjVjUcVk4jEQcU9MGy/rulF5WvUILseCM2E= golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -650,8 +628,6 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -683,7 +659,6 @@ golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 868b0ec25cf3c2ddfeb9f6c74f91db3085256e80 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 23 Aug 2024 12:27:19 +0800 Subject: [PATCH 307/659] chore: replace link of zhaoziyuan [skip ci] --- README.md | 2 +- README_cn.md | 2 +- README_ja.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9f0b7ab8312..bed2eadf160 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ https://alist.nn.ci/guide/sponsor.html - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) -- [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 +- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## Contributors diff --git a/README_cn.md b/README_cn.md index ec45c6ef9bc..7e45d60f757 100644 --- a/README_cn.md +++ b/README_cn.md @@ -115,7 +115,7 @@ AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我 - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - 苹果生态下优雅的网盘视频播放器，iPhone，iPad，Mac，Apple TV全平台支持。 - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (国内API服务器赞助) -- [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 +- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## 贡献者 diff --git a/README_ja.md b/README_ja.md index ef1351dfd8b..453e7b9966b 100644 --- a/README_ja.md +++ b/README_ja.md @@ -117,7 +117,7 @@ https://alist.nn.ci/guide/sponsor.html - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. - [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) -- [找资源](https://zhaoziyuan.pw/) - 阿里云盘资源搜索引擎 +- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## コントリビューター From d92744e673a5e5ab00cfe5740073f3509af367c9 Mon Sep 17 00:00:00 2001 From: Mmx Date: Sat, 24 Aug 2024 22:20:20 +0800 Subject: [PATCH 308/659] chore(local): decrease mass ffmpeg logs (#7073) --- drivers/local/util.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/local/util.go b/drivers/local/util.go index 84f1822bf24..b994c2056b7 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -36,12 +36,12 @@ func isSymlinkDir(f fs.FileInfo, path string) bool { func GetSnapshot(videoPath string, frameNum int) (imgData *bytes.Buffer, err error) { srcBuf := bytes.NewBuffer(nil) - err = ffmpeg.Input(videoPath).Filter("select", ffmpeg.Args{fmt.Sprintf("gte(n,%d)", frameNum)}). + stream := ffmpeg.Input(videoPath). + Filter("select", ffmpeg.Args{fmt.Sprintf("gte(n,%d)", frameNum)}). Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg"}). - WithOutput(srcBuf, os.Stdout). - Run() - - if err != nil { + GlobalArgs("-loglevel", "error").Silent(true). + WithOutput(srcBuf, os.Stdout) + if err = stream.Run(); err != nil { return nil, err } return srcBuf, nil From b910b8917f4bc857c9767102a81c4208b1bc6908 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sat, 24 Aug 2024 22:24:18 +0800 Subject: [PATCH 309/659] fix(deps): update golang.org/x/exp digest to 9b4947d (#7065) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 4dcf664c446..da423f570d1 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.26.0 - golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa + golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 golang.org/x/image v0.19.0 golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 diff --git a/go.sum b/go.sum index ac0ecee759f..0cf0833d5d8 100644 --- a/go.sum +++ b/go.sum @@ -556,6 +556,8 @@ golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= +golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= From d4f9c4b6afb4c7149d51188661dd56d3d4c58aa7 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 29 Aug 2024 23:03:54 +0800 Subject: [PATCH 310/659] ci: trigger desktop beta version --- .github/workflows/beta_release.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 4252b1309e5..dc25928aa15 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -85,4 +85,16 @@ jobs: with: files: build/compress/* prerelease: true - tag_name: beta \ No newline at end of file + tag_name: beta + + desktop: + needs: + - release + name: Beta Release Desktop + runs-on: ubuntu-latest + steps: + - uses: dusansimic/trigger-workflow-action@v0 + with: + name: release_beta.yml + owner: alist-org + repo: desktop-release \ No newline at end of file From ba716ae325ae809acbcfe0ce98f13232050abee7 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sun, 1 Sep 2024 23:06:51 +0800 Subject: [PATCH 311/659] fix(pikpak): error when passing the user_id field (#7117 close #7118) --- drivers/pikpak/driver.go | 41 ++++++-------- drivers/pikpak/meta.go | 15 ++--- drivers/pikpak/util.go | 117 ++++++++++++++++++++++++--------------- 3 files changed, 97 insertions(+), 76 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index e2a2b82e7a7..bc15f403364 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -90,43 +90,36 @@ func (d *PikPak) Init(ctx context.Context) (err error) { // 如果已经有RefreshToken，直接获取AccessToken if d.Addition.RefreshToken != "" { - // 使用 oauth2 刷新令牌 - // 初始化 oauth2Token - d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { - return oauth2Config.TokenSource(ctx, &oauth2.Token{ - RefreshToken: d.Addition.RefreshToken, - }).Token() - })) + if d.RefreshTokenMethod == "oauth2" { + // 使用 oauth2 刷新令牌 + // 初始化 oauth2Token + d.initializeOAuth2Token(ctx, oauth2Config, d.Addition.RefreshToken) + if err := d.refreshTokenByOAuth2(); err != nil { + return err + } + } else { + if err := d.refreshToken(d.Addition.RefreshToken); err != nil { + return err + } + } + } else { // 如果没有填写RefreshToken，尝试登录 获取 refreshToken if err := d.login(); err != nil { return err } - d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { - return oauth2Config.TokenSource(ctx, &oauth2.Token{ - RefreshToken: d.RefreshToken, - }).Token() - })) - } + if d.RefreshTokenMethod == "oauth2" { + d.initializeOAuth2Token(ctx, oauth2Config, d.RefreshToken) + } - token, err := d.oauth2Token.Token() - if err != nil { - return err } - d.RefreshToken = token.RefreshToken - d.AccessToken = token.AccessToken // 获取CaptchaToken - err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Username) + err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Common.GetUserID()) if err != nil { return err } - // 获取用户ID - userID := token.Extra("sub").(string) - if userID != "" { - d.Common.SetUserID(userID) - } // 更新UserAgent if d.Platform == "android" { d.Common.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(d.Username+d.Password), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, d.Common.UserID) diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index d27cee32c74..9871c6ee77d 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -7,13 +7,14 @@ import ( type Addition struct { driver.RootID - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - Platform string `json:"platform" required:"true" type:"select" options:"android,web"` - RefreshToken string `json:"refresh_token" required:"true" default:""` - CaptchaToken string `json:"captcha_token" default:""` - DeviceID string `json:"device_id" required:"false" default:""` - DisableMediaLink bool `json:"disable_media_link" default:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + Platform string `json:"platform" required:"true" type:"select" options:"android,web"` + RefreshToken string `json:"refresh_token" required:"true" default:""` + RefreshTokenMethod string `json:"refresh_token_method" required:"true" type:"select" options:"oauth2,http"` + CaptchaToken string `json:"captcha_token" default:""` + DeviceID string `json:"device_id" required:"false" default:""` + DisableMediaLink bool `json:"disable_media_link" default:"true"` } var config = driver.Config{ diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index d3371a2558a..6ffcae56780 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -2,6 +2,7 @@ package pikpak import ( "bytes" + "context" "crypto/md5" "crypto/sha1" "encoding/hex" @@ -13,6 +14,7 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" + "golang.org/x/oauth2" "io" "net/http" "path/filepath" @@ -112,39 +114,63 @@ func (d *PikPak) login() error { return nil } -//func (d *PikPak) refreshToken() error { -// url := "https://user.mypikpak.com/v1/auth/token" -// var e ErrResp -// res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e). -// SetHeader("user-agent", "").SetBody(base.Json{ -// "client_id": ClientID, -// "client_secret": ClientSecret, -// "grant_type": "refresh_token", -// "refresh_token": d.RefreshToken, -// }).SetQueryParam("client_id", ClientID).Post(url) -// if err != nil { -// d.Status = err.Error() -// op.MustSaveDriverStorage(d) -// return err -// } -// if e.ErrorCode != 0 { -// if e.ErrorCode == 4126 { -// // refresh_token invalid, re-login -// return d.login() -// } -// d.Status = e.Error() -// op.MustSaveDriverStorage(d) -// return errors.New(e.Error()) -// } -// data := res.Body() -// d.Status = "work" -// d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() -// d.AccessToken = jsoniter.Get(data, "access_token").ToString() -// d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) -// d.Addition.RefreshToken = d.RefreshToken -// op.MustSaveDriverStorage(d) -// return nil -//} +func (d *PikPak) refreshToken(refreshToken string) error { + url := "https://user.mypikpak.com/v1/auth/token" + var e ErrResp + res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e). + SetHeader("user-agent", "").SetBody(base.Json{ + "client_id": d.ClientID, + "client_secret": d.ClientSecret, + "grant_type": "refresh_token", + "refresh_token": refreshToken, + }).SetQueryParam("client_id", d.ClientID).Post(url) + if err != nil { + d.Status = err.Error() + op.MustSaveDriverStorage(d) + return err + } + if e.ErrorCode != 0 { + if e.ErrorCode == 4126 { + // refresh_token invalid, re-login + return d.login() + } + d.Status = e.Error() + op.MustSaveDriverStorage(d) + return errors.New(e.Error()) + } + data := res.Body() + d.Status = "work" + d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() + d.AccessToken = jsoniter.Get(data, "access_token").ToString() + d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) + d.Addition.RefreshToken = d.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *PikPak) initializeOAuth2Token(ctx context.Context, oauth2Config *oauth2.Config, refreshToken string) { + d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { + return oauth2Config.TokenSource(ctx, &oauth2.Token{ + RefreshToken: refreshToken, + }).Token() + })) +} + +func (d *PikPak) refreshTokenByOAuth2() error { + token, err := d.oauth2Token.Token() + if err != nil { + return err + } + d.Status = "work" + d.RefreshToken = token.RefreshToken + d.AccessToken = token.AccessToken + // 获取用户ID + userID := token.Extra("sub").(string) + d.Common.SetUserID(userID) + d.Addition.RefreshToken = d.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() @@ -181,22 +207,19 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r return res.Body(), nil case 4122, 4121, 16: // access_token 过期 - - //if err1 := d.refreshToken(); err1 != nil { - // return nil, err1 - //} - t, err := d.oauth2Token.Token() - if err != nil { - return nil, err + if d.RefreshTokenMethod == "oauth2" { + if err1 := d.refreshTokenByOAuth2(); err1 != nil { + return nil, err1 + } + } else { + if err1 := d.refreshToken(d.RefreshToken); err1 != nil { + return nil, err1 + } } - d.AccessToken = t.AccessToken - d.RefreshToken = t.RefreshToken - d.Addition.RefreshToken = t.RefreshToken - op.MustSaveDriverStorage(d) return d.request(url, method, callback, resp) case 9: // 验证码token过期 - if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.Common.UserID); err != nil { + if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.GetUserID()); err != nil { return nil, err } return d.request(url, method, callback, resp) @@ -337,6 +360,10 @@ func (c *Common) GetDeviceID() string { return c.DeviceID } +func (c *Common) GetUserID() string { + return c.UserID +} + // RefreshCaptchaTokenAtLogin 刷新验证码token(登录后) func (d *PikPak) RefreshCaptchaTokenAtLogin(action, userID string) error { metas := map[string]string{ From 34ada815823b114f38fdf8f3f020f70d439b6d5e Mon Sep 17 00:00:00 2001 From: Mmx Date: Tue, 3 Sep 2024 20:02:13 +0800 Subject: [PATCH 312/659] fix(webdav): memory leak in HttpServer (#7123 close #7088) * chore(webdav): fix warnings in HttpServe * fix(webdav): HttpServe memory leak --- internal/net/serve.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/net/serve.go b/internal/net/serve.go index e58d7eb9f46..0eb8cbb8866 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -87,9 +87,9 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time sendSize := size var sendContent io.ReadCloser ranges, err := http_range.ParseRange(rangeReq, size) - switch err { - case nil: - case http_range.ErrNoOverlap: + switch { + case err == nil: + case errors.Is(err, http_range.ErrNoOverlap): if size == 0 { // Some clients add a Range header to all requests to // limit the size of the response. If the file is empty, @@ -105,7 +105,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time return } - if sumRangesSize(ranges) > size || size < 0 { + if sumRangesSize(ranges) > size { // The total number of bytes in all the ranges is larger than the size of the file // or unknown file size, ignore the range request. ranges = nil @@ -174,6 +174,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time pw.Close() }() } + defer sendContent.Close() w.Header().Set("Accept-Ranges", "bytes") if w.Header().Get("Content-Encoding") == "" { @@ -192,7 +193,6 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time http.Error(w, err.Error(), http.StatusInternalServerError) } } - //defer sendContent.Close() } func ProcessHeader(origin, override http.Header) http.Header { result := http.Header{} From 4874c9e43ba384558de0dacc32c326f9b83d440b Mon Sep 17 00:00:00 2001 From: Mmx Date: Tue, 3 Sep 2024 20:03:30 +0800 Subject: [PATCH 313/659] fix(local): thumbnails oom (#7124 close #7082) * add my_build.sh * Fix OOM of thumbnail generation of LoaclDrive by using a task queue to control thread count * remove my_build.sh * chore(local): allow ThumbConcurrency set to zero * revert(local): changes to thumbnail generating functions * feat(local): implement static token bucket * feat(local): use static token bucket to limit thumbnails generating concurrent --------- Co-authored-by: KKJas <75424880+Muione@users.noreply.github.com> --- drivers/local/driver.go | 25 ++++++++++++-- drivers/local/meta.go | 1 + drivers/local/token_bucket.go | 61 +++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 drivers/local/token_bucket.go diff --git a/drivers/local/driver.go b/drivers/local/driver.go index abe5c6b5744..bf993e5d5f8 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -30,6 +30,10 @@ type Local struct { model.Storage Addition mkdirPerm int32 + + // zero means no limit + thumbConcurrency int + thumbTokenBucket TokenBucket } func (d *Local) Config() driver.Config { @@ -62,6 +66,18 @@ func (d *Local) Init(ctx context.Context) error { return err } } + if d.ThumbConcurrency != "" { + v, err := strconv.ParseUint(d.ThumbConcurrency, 10, 32) + if err != nil { + return err + } + d.thumbConcurrency = int(v) + } + if d.thumbConcurrency == 0 { + d.thumbTokenBucket = NewNopTokenBucket() + } else { + d.thumbTokenBucket = NewStaticTokenBucket(d.thumbConcurrency) + } return nil } @@ -126,7 +142,6 @@ func (d *Local) FileInfoToObj(f fs.FileInfo, reqPath string, fullPath string) mo }, } return &file - } func (d *Local) GetMeta(ctx context.Context, path string) (model.Obj, error) { f, err := os.Stat(path) @@ -178,7 +193,13 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( fullPath := file.GetPath() var link model.Link if args.Type == "thumb" && utils.Ext(file.GetName()) != "svg" { - buf, thumbPath, err := d.getThumb(file) + var buf *bytes.Buffer + var thumbPath *string + err := d.thumbTokenBucket.Do(ctx, func() error { + var err error + buf, thumbPath, err = d.getThumb(file) + return err + }) if err != nil { return nil, err } diff --git a/drivers/local/meta.go b/drivers/local/meta.go index 51b49e64ef4..5ffac920234 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -9,6 +9,7 @@ type Addition struct { driver.RootPath Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` ThumbCacheFolder string `json:"thumb_cache_folder"` + ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` MkdirPerm string `json:"mkdir_perm" default:"777"` RecycleBinPath string `json:"recycle_bin_path" default:"delete permanently" help:"path to recycle bin, delete permanently if empty or keep 'delete permanently'"` diff --git a/drivers/local/token_bucket.go b/drivers/local/token_bucket.go new file mode 100644 index 00000000000..38fbe73fc9b --- /dev/null +++ b/drivers/local/token_bucket.go @@ -0,0 +1,61 @@ +package local + +import "context" + +type TokenBucket interface { + Take() <-chan struct{} + Put() + Do(context.Context, func() error) error +} + +// StaticTokenBucket is a bucket with a fixed number of tokens, +// where the retrieval and return of tokens are manually controlled. +// In the initial state, the bucket is full. +type StaticTokenBucket struct { + bucket chan struct{} +} + +func NewStaticTokenBucket(size int) StaticTokenBucket { + bucket := make(chan struct{}, size) + for range size { + bucket <- struct{}{} + } + return StaticTokenBucket{bucket: bucket} +} + +func (b StaticTokenBucket) Take() <-chan struct{} { + return b.bucket +} + +func (b StaticTokenBucket) Put() { + b.bucket <- struct{}{} +} + +func (b StaticTokenBucket) Do(ctx context.Context, f func() error) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-b.bucket: + defer b.Put() + } + return f() +} + +// NopTokenBucket all function calls to this bucket will success immediately +type NopTokenBucket struct { + nop chan struct{} +} + +func NewNopTokenBucket() NopTokenBucket { + nop := make(chan struct{}) + close(nop) + return NopTokenBucket{nop} +} + +func (b NopTokenBucket) Take() <-chan struct{} { + return b.nop +} + +func (b NopTokenBucket) Put() {} + +func (b NopTokenBucket) Do(_ context.Context, f func() error) error { return f() } From c9fa3d7cd60796bc30df45ef8a6b52edab1b1313 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sun, 8 Sep 2024 10:44:34 +0800 Subject: [PATCH 314/659] fix: broken file with local proxy (#7132 close #7112) * fix: local proxy download file damage * fix: temp dir remove --- internal/bootstrap/config.go | 10 ++++++++-- internal/op/fs.go | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index ff36509cf5f..27174c23633 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -102,7 +102,13 @@ func initURL() { } func CleanTempDir() { - if err := os.RemoveAll(conf.Conf.TempDir); err != nil { - log.Errorln("failed delete temp file: ", err) + files, err := os.ReadDir(conf.Conf.TempDir) + if err != nil { + log.Errorln("failed list temp file: ", err) + } + for _, file := range files { + if err := os.RemoveAll(filepath.Join(conf.Conf.TempDir, file.Name())); err != nil { + log.Errorln("failed delete temp file: ", err) + } } } diff --git a/internal/op/fs.go b/internal/op/fs.go index e0153952638..e49c941a62f 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -267,6 +267,12 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li } return link, nil } + + if storage.Config().OnlyLocal { + link, err := fn() + return link, file, err + } + link, err, _ := linkG.Do(key, fn) return link, file, err } From 716d33fddd32fcf9b3ab916c789a525f828ee72a Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sun, 8 Sep 2024 10:45:43 +0800 Subject: [PATCH 315/659] feat(pikpak&pikpak_share): add download address delay detection (#7136) * feat(pikpak): add download address delay detection * feat(pikpak_share): add download address delay detection --- drivers/pikpak/driver.go | 37 ++++++++-- drivers/pikpak/meta.go | 18 +++-- drivers/pikpak/util.go | 131 ++++++++++++++++++++++++++++----- drivers/pikpak_share/driver.go | 30 +++++++- drivers/pikpak_share/meta.go | 12 +-- drivers/pikpak_share/util.go | 128 +++++++++++++++++++++++++++----- 6 files changed, 300 insertions(+), 56 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index bc15f403364..4208bb8765a 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -14,6 +14,7 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/oauth2" "net/http" + "regexp" "strconv" "strings" ) @@ -48,6 +49,7 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.Common.CaptchaToken = token op.MustSaveDriverStorage(d) }, + LowLatencyAddr: "", } } @@ -65,6 +67,13 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.PackageName = WebPackageName d.Algorithms = WebAlgorithms d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" + } else if d.Platform == "pc" { + d.ClientID = PCClientID + d.ClientSecret = PCClientSecret + d.ClientVersion = PCClientVersion + d.PackageName = PCPackageName + d.Algorithms = PCAlgorithms + d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.5.6.4831 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" } if d.Addition.CaptchaToken != "" && d.Addition.RefreshToken == "" { @@ -128,6 +137,15 @@ func (d *PikPak) Init(ctx context.Context) (err error) { // 保存 有效的 RefreshToken d.Addition.RefreshToken = d.RefreshToken op.MustSaveDriverStorage(d) + + if d.UseLowLatencyAddress && d.Addition.CustomLowLatencyAddress != "" { + d.Common.LowLatencyAddr = d.Addition.CustomLowLatencyAddress + } else if d.UseLowLatencyAddress { + d.Common.LowLatencyAddr = findLowestLatencyAddress(DlAddr) + d.Addition.CustomLowLatencyAddress = d.Common.LowLatencyAddr + op.MustSaveDriverStorage(d) + } + return nil } @@ -147,6 +165,7 @@ func (d *PikPak) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp File + var url string queryParams := map[string]string{ "_magic": "2021", "usage": "FETCH", @@ -162,14 +181,22 @@ func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err != nil { return nil, err } - link := model.Link{ - URL: resp.WebContentLink, - } + url = resp.WebContentLink + if !d.DisableMediaLink && len(resp.Medias) > 0 && resp.Medias[0].Link.Url != "" { log.Debugln("use media link") - link.URL = resp.Medias[0].Link.Url + url = resp.Medias[0].Link.Url + } + + if d.UseLowLatencyAddress && d.Common.LowLatencyAddr != "" { + // 替换为加速链接 + re := regexp.MustCompile(`https://[^/]+/download/`) + url = re.ReplaceAllString(url, "https://"+d.Common.LowLatencyAddr+"/download/") } - return &link, nil + + return &model.Link{ + URL: url, + }, nil } func (d *PikPak) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index 9871c6ee77d..4d25ecc605e 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -7,14 +7,16 @@ import ( type Addition struct { driver.RootID - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - Platform string `json:"platform" required:"true" type:"select" options:"android,web"` - RefreshToken string `json:"refresh_token" required:"true" default:""` - RefreshTokenMethod string `json:"refresh_token_method" required:"true" type:"select" options:"oauth2,http"` - CaptchaToken string `json:"captcha_token" default:""` - DeviceID string `json:"device_id" required:"false" default:""` - DisableMediaLink bool `json:"disable_media_link" default:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + Platform string `json:"platform" required:"true" type:"select" options:"android,web,pc"` + RefreshToken string `json:"refresh_token" required:"true" default:""` + RefreshTokenMethod string `json:"refresh_token_method" required:"true" type:"select" options:"oauth2,http"` + CaptchaToken string `json:"captcha_token" default:""` + DeviceID string `json:"device_id" required:"false" default:""` + DisableMediaLink bool `json:"disable_media_link" default:"true"` + UseLowLatencyAddress bool `json:"use_low_latency_address" default:"false"` + CustomLowLatencyAddress string `json:"custom_low_latency_address" default:""` } var config = driver.Config{ diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 6ffcae56780..cc745d520cb 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -30,21 +30,14 @@ import ( // do others that not defined in Driver interface var AndroidAlgorithms = []string{ - "Gez0T9ijiI9WCeTsKSg3SMlx", - "zQdbalsolyb1R/", - "ftOjr52zt51JD68C3s", - "yeOBMH0JkbQdEFNNwQ0RI9T3wU/v", - "BRJrQZiTQ65WtMvwO", - "je8fqxKPdQVJiy1DM6Bc9Nb1", - "niV", - "9hFCW2R1", - "sHKHpe2i96", - "p7c5E6AcXQ/IJUuAEC9W6", - "", - "aRv9hjc9P+Pbn+u3krN6", - "BzStcgE8qVdqjEH16l4", - "SqgeZvL5j9zoHP95xWHt", - "zVof5yaJkPe3VFpadPof", + "aDhgaSE3MsjROCmpmsWqP1sJdFJ", + "+oaVkqdd8MJuKT+uMr2AYKcd9tdWge3XPEPR2hcePUknd", + "u/sd2GgT2fTytRcKzGicHodhvIltMntA3xKw2SRv7S48OdnaQIS5mn", + "2WZiae2QuqTOxBKaaqCNHCW3olu2UImelkDzBn", + "/vJ3upic39lgmrkX855Qx", + "yNc9ruCVMV7pGV7XvFeuLMOcy1", + "4FPq8mT3JQ1jzcVxMVfwFftLQm33M7i", + "xozoy5e3Ea", } var WebAlgorithms = []string{ @@ -65,6 +58,19 @@ var WebAlgorithms = []string{ "NhXXU9rg4XXdzo7u5o", } +var PCAlgorithms = []string{ + "KHBJ07an7ROXDoK7Db", + "G6n399rSWkl7WcQmw5rpQInurc1DkLmLJqE", + "JZD1A3M4x+jBFN62hkr7VDhkkZxb9g3rWqRZqFAAb", + "fQnw/AmSlbbI91Ik15gpddGgyU7U", + "/Dv9JdPYSj3sHiWjouR95NTQff", + "yGx2zuTjbWENZqecNI+edrQgqmZKP", + "ljrbSzdHLwbqcRn", + "lSHAsqCkGDGxQqqwrVu", + "TsWXI81fD1", + "vk7hBjawK/rOSrSWajtbMk95nfgf3", +} + const ( OSSUserAgent = "aliyun-sdk-android/2.9.13(Linux/Android 14/M2004j7ac;UKQ1.231108.001)" OssSecurityTokenHeaderName = "X-OSS-Security-Token" @@ -74,16 +80,59 @@ const ( const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - AndroidClientVersion = "1.47.1" + AndroidClientVersion = "1.48.3" AndroidPackageName = "com.pikcloud.pikpak" - AndroidSdkVersion = "2.0.4.204000" + AndroidSdkVersion = "2.0.4.204101" WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" WebPackageName = "mypikpak.com" WebSdkVersion = "8.0.3" + PCClientID = "YvtoWO6GNHiuCl7x" + PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" + PCClientVersion = "undefined" // 2.5.6.4831 + PCPackageName = "mypikpak.com" + PCSdkVersion = "8.0.3" ) +var DlAddr = []string{ + "dl-a10b-0621.mypikpak.com", + "dl-a10b-0622.mypikpak.com", + "dl-a10b-0623.mypikpak.com", + "dl-a10b-0624.mypikpak.com", + "dl-a10b-0625.mypikpak.com", + "dl-a10b-0858.mypikpak.com", + "dl-a10b-0859.mypikpak.com", + "dl-a10b-0860.mypikpak.com", + "dl-a10b-0861.mypikpak.com", + "dl-a10b-0862.mypikpak.com", + "dl-a10b-0863.mypikpak.com", + "dl-a10b-0864.mypikpak.com", + "dl-a10b-0865.mypikpak.com", + "dl-a10b-0866.mypikpak.com", + "dl-a10b-0867.mypikpak.com", + "dl-a10b-0868.mypikpak.com", + "dl-a10b-0869.mypikpak.com", + "dl-a10b-0870.mypikpak.com", + "dl-a10b-0871.mypikpak.com", + "dl-a10b-0872.mypikpak.com", + "dl-a10b-0873.mypikpak.com", + "dl-a10b-0874.mypikpak.com", + "dl-a10b-0875.mypikpak.com", + "dl-a10b-0876.mypikpak.com", + "dl-a10b-0877.mypikpak.com", + "dl-a10b-0878.mypikpak.com", + "dl-a10b-0879.mypikpak.com", + "dl-a10b-0880.mypikpak.com", + "dl-a10b-0881.mypikpak.com", + "dl-a10b-0882.mypikpak.com", + "dl-a10b-0883.mypikpak.com", + "dl-a10b-0884.mypikpak.com", + "dl-a10b-0885.mypikpak.com", + "dl-a10b-0886.mypikpak.com", + "dl-a10b-0887.mypikpak.com", +} + func (d *PikPak) login() error { url := "https://user.mypikpak.com/v1/auth/signin" // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) @@ -180,13 +229,15 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r "X-Device-ID": d.GetDeviceID(), "X-Captcha-Token": d.GetCaptchaToken(), }) - if d.oauth2Token != nil { + if d.RefreshTokenMethod == "oauth2" { // 使用oauth2 获取 access_token token, err := d.oauth2Token.Token() if err != nil { return nil, err } req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) + } else { + req.SetHeader("Authorization", "Bearer "+d.AccessToken) } if callback != nil { @@ -277,6 +328,7 @@ type Common struct { UserAgent string // 验证码token刷新成功回调 RefreshCTokenCk func(token string) + LowLatencyAddr string } func generateDeviceSign(deviceID, packageName string) string { @@ -667,3 +719,46 @@ func OssOption(params *S3Params) []oss.Option { } return options } + +type AddressLatency struct { + Address string + Latency time.Duration +} + +func checkLatency(address string, wg *sync.WaitGroup, ch chan<- AddressLatency) { + defer wg.Done() + start := time.Now() + resp, err := http.Get("https://" + address + "/generate_204") + if err != nil { + ch <- AddressLatency{Address: address, Latency: time.Hour} // Set high latency on error + return + } + defer resp.Body.Close() + latency := time.Since(start) + ch <- AddressLatency{Address: address, Latency: latency} +} + +func findLowestLatencyAddress(addresses []string) string { + var wg sync.WaitGroup + ch := make(chan AddressLatency, len(addresses)) + + for _, address := range addresses { + wg.Add(1) + go checkLatency(address, &wg, ch) + } + + wg.Wait() + close(ch) + + var lowestLatencyAddress string + lowestLatency := time.Hour + + for result := range ch { + if result.Latency < lowestLatency { + lowestLatency = result.Latency + lowestLatencyAddress = result.Address + } + } + + return lowestLatencyAddress +} diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index 448ad2dd94a..91cb45ca1cf 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -4,6 +4,7 @@ import ( "context" "github.com/alist-org/alist/v3/internal/op" "net/http" + "regexp" "time" "github.com/alist-org/alist/v3/internal/driver" @@ -36,6 +37,7 @@ func (d *PikPakShare) Init(ctx context.Context) error { d.Common.CaptchaToken = token op.MustSaveDriverStorage(d) }, + LowLatencyAddr: "", } } @@ -60,6 +62,21 @@ func (d *PikPakShare) Init(ctx context.Context) error { d.PackageName = WebPackageName d.Algorithms = WebAlgorithms d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" + } else if d.Platform == "pc" { + d.ClientID = PCClientID + d.ClientSecret = PCClientSecret + d.ClientVersion = PCClientVersion + d.PackageName = PCPackageName + d.Algorithms = PCAlgorithms + d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.5.6.4831 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" + } + + if d.UseLowLatencyAddress && d.Addition.CustomLowLatencyAddress != "" { + d.Common.LowLatencyAddr = d.Addition.CustomLowLatencyAddress + } else if d.UseLowLatencyAddress { + d.Common.LowLatencyAddr = findLowestLatencyAddress(DlAddr) + d.Addition.CustomLowLatencyAddress = d.Common.LowLatencyAddr + op.MustSaveDriverStorage(d) } // 获取CaptchaToken @@ -71,6 +88,7 @@ func (d *PikPakShare) Init(ctx context.Context) error { if d.SharePwd != "" { return d.getSharePassToken() } + return nil } @@ -112,10 +130,16 @@ func (d *PikPakShare) Link(ctx context.Context, file model.Obj, args model.LinkA } } - link := model.Link{ - URL: downloadUrl, + + if d.UseLowLatencyAddress && d.Common.LowLatencyAddr != "" { + // 替换为加速链接 + re := regexp.MustCompile(`https://[^/]+/download/`) + downloadUrl = re.ReplaceAllString(downloadUrl, "https://"+d.Common.LowLatencyAddr+"/download/") } - return &link, nil + + return &model.Link{ + URL: downloadUrl, + }, nil } var _ driver.Driver = (*PikPakShare)(nil) diff --git a/drivers/pikpak_share/meta.go b/drivers/pikpak_share/meta.go index e6f00cdad54..dc551e028a0 100644 --- a/drivers/pikpak_share/meta.go +++ b/drivers/pikpak_share/meta.go @@ -7,11 +7,13 @@ import ( type Addition struct { driver.RootID - ShareId string `json:"share_id" required:"true"` - SharePwd string `json:"share_pwd"` - Platform string `json:"platform" required:"true" type:"select" options:"android,web"` - DeviceID string `json:"device_id" required:"false" default:""` - UseTransCodingAddress bool `json:"use_transcoding_address" required:"true" default:"false"` + ShareId string `json:"share_id" required:"true"` + SharePwd string `json:"share_pwd"` + Platform string `json:"platform" required:"true" type:"select" options:"android,web,pc"` + DeviceID string `json:"device_id" required:"false" default:""` + UseTransCodingAddress bool `json:"use_transcoding_address" required:"true" default:"false"` + UseLowLatencyAddress bool `json:"use_low_latency_address" default:"false"` + CustomLowLatencyAddress string `json:"custom_low_latency_address" default:""` } var config = driver.Config{ diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index a9c8fffe84e..f333ca5f706 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -10,6 +10,7 @@ import ( "net/http" "regexp" "strings" + "sync" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -17,21 +18,14 @@ import ( ) var AndroidAlgorithms = []string{ - "Gez0T9ijiI9WCeTsKSg3SMlx", - "zQdbalsolyb1R/", - "ftOjr52zt51JD68C3s", - "yeOBMH0JkbQdEFNNwQ0RI9T3wU/v", - "BRJrQZiTQ65WtMvwO", - "je8fqxKPdQVJiy1DM6Bc9Nb1", - "niV", - "9hFCW2R1", - "sHKHpe2i96", - "p7c5E6AcXQ/IJUuAEC9W6", - "", - "aRv9hjc9P+Pbn+u3krN6", - "BzStcgE8qVdqjEH16l4", - "SqgeZvL5j9zoHP95xWHt", - "zVof5yaJkPe3VFpadPof", + "aDhgaSE3MsjROCmpmsWqP1sJdFJ", + "+oaVkqdd8MJuKT+uMr2AYKcd9tdWge3XPEPR2hcePUknd", + "u/sd2GgT2fTytRcKzGicHodhvIltMntA3xKw2SRv7S48OdnaQIS5mn", + "2WZiae2QuqTOxBKaaqCNHCW3olu2UImelkDzBn", + "/vJ3upic39lgmrkX855Qx", + "yNc9ruCVMV7pGV7XvFeuLMOcy1", + "4FPq8mT3JQ1jzcVxMVfwFftLQm33M7i", + "xozoy5e3Ea", } var WebAlgorithms = []string{ @@ -52,19 +46,75 @@ var WebAlgorithms = []string{ "NhXXU9rg4XXdzo7u5o", } +var PCAlgorithms = []string{ + "KHBJ07an7ROXDoK7Db", + "G6n399rSWkl7WcQmw5rpQInurc1DkLmLJqE", + "JZD1A3M4x+jBFN62hkr7VDhkkZxb9g3rWqRZqFAAb", + "fQnw/AmSlbbI91Ik15gpddGgyU7U", + "/Dv9JdPYSj3sHiWjouR95NTQff", + "yGx2zuTjbWENZqecNI+edrQgqmZKP", + "ljrbSzdHLwbqcRn", + "lSHAsqCkGDGxQqqwrVu", + "TsWXI81fD1", + "vk7hBjawK/rOSrSWajtbMk95nfgf3", +} + const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - AndroidClientVersion = "1.47.1" + AndroidClientVersion = "1.48.3" AndroidPackageName = "com.pikcloud.pikpak" - AndroidSdkVersion = "2.0.4.204000" + AndroidSdkVersion = "2.0.4.204101" WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" WebPackageName = "mypikpak.com" WebSdkVersion = "8.0.3" + PCClientID = "YvtoWO6GNHiuCl7x" + PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" + PCClientVersion = "undefined" // 2.5.6.4831 + PCPackageName = "mypikpak.com" + PCSdkVersion = "8.0.3" ) +var DlAddr = []string{ + "dl-a10b-0621.mypikpak.com", + "dl-a10b-0622.mypikpak.com", + "dl-a10b-0623.mypikpak.com", + "dl-a10b-0624.mypikpak.com", + "dl-a10b-0625.mypikpak.com", + "dl-a10b-0858.mypikpak.com", + "dl-a10b-0859.mypikpak.com", + "dl-a10b-0860.mypikpak.com", + "dl-a10b-0861.mypikpak.com", + "dl-a10b-0862.mypikpak.com", + "dl-a10b-0863.mypikpak.com", + "dl-a10b-0864.mypikpak.com", + "dl-a10b-0865.mypikpak.com", + "dl-a10b-0866.mypikpak.com", + "dl-a10b-0867.mypikpak.com", + "dl-a10b-0868.mypikpak.com", + "dl-a10b-0869.mypikpak.com", + "dl-a10b-0870.mypikpak.com", + "dl-a10b-0871.mypikpak.com", + "dl-a10b-0872.mypikpak.com", + "dl-a10b-0873.mypikpak.com", + "dl-a10b-0874.mypikpak.com", + "dl-a10b-0875.mypikpak.com", + "dl-a10b-0876.mypikpak.com", + "dl-a10b-0877.mypikpak.com", + "dl-a10b-0878.mypikpak.com", + "dl-a10b-0879.mypikpak.com", + "dl-a10b-0880.mypikpak.com", + "dl-a10b-0881.mypikpak.com", + "dl-a10b-0882.mypikpak.com", + "dl-a10b-0883.mypikpak.com", + "dl-a10b-0884.mypikpak.com", + "dl-a10b-0885.mypikpak.com", + "dl-a10b-0886.mypikpak.com", + "dl-a10b-0887.mypikpak.com", +} + func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ @@ -177,6 +227,7 @@ type Common struct { UserAgent string // 验证码token刷新成功回调 RefreshCTokenCk func(token string) + LowLatencyAddr string } func (c *Common) SetUserAgent(userAgent string) { @@ -316,3 +367,46 @@ func (d *PikPakShare) refreshCaptchaToken(action string, metas map[string]string d.Common.SetCaptchaToken(resp.CaptchaToken) return nil } + +type AddressLatency struct { + Address string + Latency time.Duration +} + +func checkLatency(address string, wg *sync.WaitGroup, ch chan<- AddressLatency) { + defer wg.Done() + start := time.Now() + resp, err := http.Get("https://" + address + "/generate_204") + if err != nil { + ch <- AddressLatency{Address: address, Latency: time.Hour} // Set high latency on error + return + } + defer resp.Body.Close() + latency := time.Since(start) + ch <- AddressLatency{Address: address, Latency: latency} +} + +func findLowestLatencyAddress(addresses []string) string { + var wg sync.WaitGroup + ch := make(chan AddressLatency, len(addresses)) + + for _, address := range addresses { + wg.Add(1) + go checkLatency(address, &wg, ch) + } + + wg.Wait() + close(ch) + + var lowestLatencyAddress string + lowestLatency := time.Hour + + for result := range ch { + if result.Latency < lowestLatency { + lowestLatency = result.Latency + lowestLatencyAddress = result.Address + } + } + + return lowestLatencyAddress +} From 92713ef5c42ced316803d05d6070d39b0e9cede4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 10:57:04 +0800 Subject: [PATCH 316/659] fix(deps): update module github.com/charmbracelet/bubbletea to v1 (#7103) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index da423f570d1..67edcddf1dc 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/blevesearch/bleve/v2 v2.4.2 github.com/caarlos0/env/v9 v9.0.0 github.com/charmbracelet/bubbles v0.19.0 - github.com/charmbracelet/bubbletea v0.27.0 + github.com/charmbracelet/bubbletea v1.1.0 github.com/charmbracelet/lipgloss v0.13.0 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e github.com/coreos/go-oidc v2.2.1+incompatible @@ -77,9 +77,9 @@ require ( github.com/blevesearch/go-faiss v1.0.20 // indirect github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect - github.com/charmbracelet/x/ansi v0.1.4 // indirect + github.com/charmbracelet/x/ansi v0.2.3 // indirect github.com/charmbracelet/x/input v0.1.0 // indirect - github.com/charmbracelet/x/term v0.1.1 // indirect + github.com/charmbracelet/x/term v0.2.0 // indirect github.com/charmbracelet/x/windows v0.1.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect diff --git a/go.sum b/go.sum index 0cf0833d5d8..32867a89e03 100644 --- a/go.sum +++ b/go.sum @@ -100,16 +100,22 @@ github.com/charmbracelet/bubbles v0.19.0 h1:gKZkKXPP6GlDk6EcfujDK19PCQqRjaJZQ7QR github.com/charmbracelet/bubbles v0.19.0/go.mod h1:WILteEqZ+krG5c3ntGEMeG99nCupcuIk7V0/zOP0tOA= github.com/charmbracelet/bubbletea v0.27.0 h1:Mznj+vvYuYagD9Pn2mY7fuelGvP0HAXtZYGgRBCbHvU= github.com/charmbracelet/bubbletea v0.27.0/go.mod h1:5MdP9XH6MbQkgGhnlxUqCNmBXf9I74KRQ8HIidRxV1Y= +github.com/charmbracelet/bubbletea v1.1.0 h1:FjAl9eAL3HBCHenhz/ZPjkKdScmaS5SK69JAK2YJK9c= +github.com/charmbracelet/bubbletea v1.1.0/go.mod h1:9Ogk0HrdbHolIKHdjfFpyXJmiCzGwy+FesYkZr7hYU4= github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/ansi v0.2.3 h1:VfFN0NUpcjBRd4DnKfRaIRo53KRgey/nhOoEqosGDEY= +github.com/charmbracelet/x/ansi v0.2.3/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q= github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= github.com/charmbracelet/x/input v0.1.0 h1:TEsGSfZYQyOtp+STIjyBq6tpRaorH0qpwZUj8DavAhQ= github.com/charmbracelet/x/input v0.1.0/go.mod h1:ZZwaBxPF7IG8gWWzPUVqHEtWhc1+HXJPNuerJGRGZ28= github.com/charmbracelet/x/term v0.1.1 h1:3cosVAiPOig+EV4X9U+3LDgtwwAoEzJjNdwbXDjF6yI= github.com/charmbracelet/x/term v0.1.1/go.mod h1:wB1fHt5ECsu3mXYusyzcngVWWlu1KKUmmLhfgr/Flxw= +github.com/charmbracelet/x/term v0.2.0 h1:cNB9Ot9q8I711MyZ7myUR5HFWL/lc3OpU8jZ4hwm0x0= +github.com/charmbracelet/x/term v0.2.0/go.mod h1:GVxgxAbjUrmpvIINHIQnJJKpMlHiZ4cktEQCN6GWyF0= github.com/charmbracelet/x/windows v0.1.0 h1:gTaxdvzDM5oMa/I2ZNF7wN78X/atWemG9Wph7Ika2k4= github.com/charmbracelet/x/windows v0.1.0/go.mod h1:GLEO/l+lizvFDBPLIOk+49gdX49L9YWMB5t+DZd0jkQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= From 40a68bcee6b4e683e9cf3780f2da525730ba45c1 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 10:58:04 +0800 Subject: [PATCH 317/659] fix(deps): update module github.com/ncw/swift/v2 to v2.0.3 (#7107) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 67edcddf1dc..9ee1afa4731 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/meilisearch/meilisearch-go v0.27.2 github.com/minio/sio v0.4.0 github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/ncw/swift/v2 v2.0.2 + github.com/ncw/swift/v2 v2.0.3 github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 diff --git a/go.sum b/go.sum index 32867a89e03..1e9ab22e7f0 100644 --- a/go.sum +++ b/go.sum @@ -393,6 +393,8 @@ github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= +github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= +github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu4h5aYIOzUtLjN08L4Qt4WGaJONMgcaD0ayBJQ= github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= From 0242f36e1c050ea76df9c21bd542fc8dbcf6dc69 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 10:58:52 +0800 Subject: [PATCH 318/659] fix(deps): update module google.golang.org/grpc to v1.66.0 (#7098) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9ee1afa4731..d932f648aa0 100644 --- a/go.mod +++ b/go.mod @@ -225,8 +225,8 @@ require ( golang.org/x/text v0.17.0 // indirect golang.org/x/tools v0.24.0 // indirect google.golang.org/api v0.169.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect - google.golang.org/grpc v1.65.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/grpc v1.66.0 google.golang.org/protobuf v1.34.2 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect diff --git a/go.sum b/go.sum index 1e9ab22e7f0..044431083fc 100644 --- a/go.sum +++ b/go.sum @@ -693,8 +693,12 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= From c8317250c14cbca3d9e27e08355bd857224d4694 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 8 Sep 2024 11:10:07 +0800 Subject: [PATCH 319/659] fix(deps): update golang.org/x/exp digest to e7e105d (#7139) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d932f648aa0..8b43b91a981 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.26.0 - golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 + golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 diff --git a/go.sum b/go.sum index 044431083fc..24ca007f1c0 100644 --- a/go.sum +++ b/go.sum @@ -566,6 +566,8 @@ golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDT golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= From b36d38f63f5efe0e8625a354456008d3742228dd Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 8 Sep 2024 11:12:53 +0800 Subject: [PATCH 320/659] chore: go mod tidy --- go.mod | 3 --- go.sum | 22 ---------------------- 2 files changed, 25 deletions(-) diff --git a/go.mod b/go.mod index 8b43b91a981..084a640a46e 100644 --- a/go.mod +++ b/go.mod @@ -78,15 +78,12 @@ require ( github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/charmbracelet/x/ansi v0.2.3 // indirect - github.com/charmbracelet/x/input v0.1.0 // indirect github.com/charmbracelet/x/term v0.2.0 // indirect - github.com/charmbracelet/x/windows v0.1.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/ipfs/boxo v0.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect - github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect ) require ( diff --git a/go.sum b/go.sum index 24ca007f1c0..5a7f2eb31b3 100644 --- a/go.sum +++ b/go.sum @@ -98,26 +98,16 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.19.0 h1:gKZkKXPP6GlDk6EcfujDK19PCQqRjaJZQ7QRERx1UF0= github.com/charmbracelet/bubbles v0.19.0/go.mod h1:WILteEqZ+krG5c3ntGEMeG99nCupcuIk7V0/zOP0tOA= -github.com/charmbracelet/bubbletea v0.27.0 h1:Mznj+vvYuYagD9Pn2mY7fuelGvP0HAXtZYGgRBCbHvU= -github.com/charmbracelet/bubbletea v0.27.0/go.mod h1:5MdP9XH6MbQkgGhnlxUqCNmBXf9I74KRQ8HIidRxV1Y= github.com/charmbracelet/bubbletea v1.1.0 h1:FjAl9eAL3HBCHenhz/ZPjkKdScmaS5SK69JAK2YJK9c= github.com/charmbracelet/bubbletea v1.1.0/go.mod h1:9Ogk0HrdbHolIKHdjfFpyXJmiCzGwy+FesYkZr7hYU4= github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= -github.com/charmbracelet/x/ansi v0.1.4 h1:IEU3D6+dWwPSgZ6HBH+v6oUuZ/nVawMiWj5831KfiLM= -github.com/charmbracelet/x/ansi v0.1.4/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/ansi v0.2.3 h1:VfFN0NUpcjBRd4DnKfRaIRo53KRgey/nhOoEqosGDEY= github.com/charmbracelet/x/ansi v0.2.3/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q= github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= -github.com/charmbracelet/x/input v0.1.0 h1:TEsGSfZYQyOtp+STIjyBq6tpRaorH0qpwZUj8DavAhQ= -github.com/charmbracelet/x/input v0.1.0/go.mod h1:ZZwaBxPF7IG8gWWzPUVqHEtWhc1+HXJPNuerJGRGZ28= -github.com/charmbracelet/x/term v0.1.1 h1:3cosVAiPOig+EV4X9U+3LDgtwwAoEzJjNdwbXDjF6yI= -github.com/charmbracelet/x/term v0.1.1/go.mod h1:wB1fHt5ECsu3mXYusyzcngVWWlu1KKUmmLhfgr/Flxw= github.com/charmbracelet/x/term v0.2.0 h1:cNB9Ot9q8I711MyZ7myUR5HFWL/lc3OpU8jZ4hwm0x0= github.com/charmbracelet/x/term v0.2.0/go.mod h1:GVxgxAbjUrmpvIINHIQnJJKpMlHiZ4cktEQCN6GWyF0= -github.com/charmbracelet/x/windows v0.1.0 h1:gTaxdvzDM5oMa/I2ZNF7wN78X/atWemG9Wph7Ika2k4= -github.com/charmbracelet/x/windows v0.1.0/go.mod h1:GLEO/l+lizvFDBPLIOk+49gdX49L9YWMB5t+DZd0jkQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= @@ -391,8 +381,6 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= -github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu4h5aYIOzUtLjN08L4Qt4WGaJONMgcaD0ayBJQ= @@ -520,8 +508,6 @@ github.com/xhofe/tache v0.1.2 h1:pHrXlrWcbTb4G7hVUDW7Rc+YTUnLJvnLBrdktVE1Fqg= github.com/xhofe/tache v0.1.2/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= -github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -562,10 +548,6 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= -golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= -golang.org/x/exp v0.0.0-20240823005443-9b4947da3948 h1:kx6Ds3MlpiUHKj7syVnbp57++8WpuKPcR5yjLBjvLEA= -golang.org/x/exp v0.0.0-20240823005443-9b4947da3948/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -693,12 +675,8 @@ google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= From 9667832b325c665cfe4e81d1e9331fa742408787 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:45:42 +0800 Subject: [PATCH 321/659] fix(pikpak): fix nil pointer error (#7150) --- drivers/pikpak/util.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index cc745d520cb..1fd26020a60 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -134,6 +134,11 @@ var DlAddr = []string{ } func (d *PikPak) login() error { + // 检查用户名和密码是否为空 + if d.Addition.Username == "" || d.Addition.Password == "" { + return errors.New("username or password is empty") + } + url := "https://user.mypikpak.com/v1/auth/signin" // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) if d.GetCaptchaToken() == "" { @@ -180,8 +185,13 @@ func (d *PikPak) refreshToken(refreshToken string) error { } if e.ErrorCode != 0 { if e.ErrorCode == 4126 { - // refresh_token invalid, re-login - return d.login() + // 1. 未填写 username 或 password + if d.Addition.Username == "" || d.Addition.Password == "" { + return errors.New("refresh_token invalid, please re-provide refresh_token") + } else { + // refresh_token invalid, re-login + return d.login() + } } d.Status = e.Error() op.MustSaveDriverStorage(d) @@ -229,14 +239,14 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r "X-Device-ID": d.GetDeviceID(), "X-Captcha-Token": d.GetCaptchaToken(), }) - if d.RefreshTokenMethod == "oauth2" { + if d.RefreshTokenMethod == "oauth2" && d.oauth2Token != nil { // 使用oauth2 获取 access_token token, err := d.oauth2Token.Token() if err != nil { return nil, err } req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) - } else { + } else if d.AccessToken != "" { req.SetHeader("Authorization", "Bearer "+d.AccessToken) } From cdbfda8921f8d98ded3e178080be65af98482227 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sun, 8 Sep 2024 19:46:23 +0800 Subject: [PATCH 322/659] fix(baidu_photo): change download api (#7144 close #7133) --- drivers/baidu_photo/driver.go | 18 ++++++---- drivers/baidu_photo/utils.go | 67 +++++++++++++++++++++++------------ 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index 7477a8eb527..94716983e52 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -137,13 +137,19 @@ func (d *BaiduPhoto) Link(ctx context.Context, file model.Obj, args model.LinkAr case *File: return d.linkFile(ctx, file, args) case *AlbumFile: - f, err := d.CopyAlbumFile(ctx, file) - if err != nil { - return nil, err + // 处理共享相册 + if d.Uk != file.Uk { + // 有概率无法获取到链接 + return d.linkAlbum(ctx, file, args) + + // 接口被限制，只能使用cookie + // f, err := d.CopyAlbumFile(ctx, file) + // if err != nil { + // return nil, err + // } + // return d.linkFile(ctx, f, args) } - return d.linkFile(ctx, f, args) - // 有概率无法获取到链接 - //return d.linkAlbum(ctx, file, args) + return d.linkFile(ctx, &file.File, args) } return nil, errs.NotFile } diff --git a/drivers/baidu_photo/utils.go b/drivers/baidu_photo/utils.go index c93f6f1265a..be0ed1336e6 100644 --- a/drivers/baidu_photo/utils.go +++ b/drivers/baidu_photo/utils.go @@ -21,8 +21,8 @@ const ( FILE_API_URL_V2 = API_URL + "/file/v2" ) -func (d *BaiduPhoto) Request(furl string, method string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { - req := base.RestyClient.R(). +func (d *BaiduPhoto) Request(client *resty.Client, furl string, method string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { + req := client.R(). SetQueryParam("access_token", d.AccessToken) if callback != nil { callback(req) @@ -88,11 +88,11 @@ func (d *BaiduPhoto) refreshToken() error { } func (d *BaiduPhoto) Get(furl string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { - return d.Request(furl, http.MethodGet, callback, resp) + return d.Request(base.RestyClient, furl, http.MethodGet, callback, resp) } func (d *BaiduPhoto) Post(furl string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { - return d.Request(furl, http.MethodPost, callback, resp) + return d.Request(base.RestyClient, furl, http.MethodPost, callback, resp) } // 获取所有文件 @@ -338,24 +338,33 @@ func (d *BaiduPhoto) linkAlbum(ctx context.Context, file *AlbumFile, args model. headers["X-Forwarded-For"] = args.IP } - res, err := base.NoRedirectClient.R(). - SetContext(ctx). - SetHeaders(headers). - SetQueryParams(map[string]string{ - "access_token": d.AccessToken, - "fsid": fmt.Sprint(file.Fsid), - "album_id": file.AlbumID, - "tid": fmt.Sprint(file.Tid), - "uk": fmt.Sprint(file.Uk), - }). - Head(ALBUM_API_URL + "/download") + resp, err := d.Request(base.NoRedirectClient, ALBUM_API_URL+"/download", http.MethodHead, func(r *resty.Request) { + r.SetContext(ctx) + r.SetHeaders(headers) + r.SetQueryParams(map[string]string{ + "fsid": fmt.Sprint(file.Fsid), + "album_id": file.AlbumID, + "tid": fmt.Sprint(file.Tid), + "uk": fmt.Sprint(file.Uk), + }) + }, nil) + + if err != nil { + return nil, err + } + + if resp.StatusCode() != 302 { + return nil, fmt.Errorf("not found 302 redirect") + } + + location := resp.Header().Get("Location") if err != nil { return nil, err } link := &model.Link{ - URL: res.Header().Get("location"), + URL: location, Header: http.Header{ "User-Agent": []string{headers["User-Agent"]}, "Referer": []string{"https://photo.baidu.com/"}, @@ -375,22 +384,36 @@ func (d *BaiduPhoto) linkFile(ctx context.Context, file *File, args model.LinkAr headers["X-Forwarded-For"] = args.IP } - var downloadUrl struct { - Dlink string `json:"dlink"` - } - _, err := d.Get(FILE_API_URL_V2+"/download", func(r *resty.Request) { + // var downloadUrl struct { + // Dlink string `json:"dlink"` + // } + // _, err := d.Get(FILE_API_URL_V1+"/download", func(r *resty.Request) { + // r.SetContext(ctx) + // r.SetHeaders(headers) + // r.SetQueryParams(map[string]string{ + // "fsid": fmt.Sprint(file.Fsid), + // }) + // }, &downloadUrl) + + resp, err := d.Request(base.NoRedirectClient, FILE_API_URL_V1+"/download", http.MethodHead, func(r *resty.Request) { r.SetContext(ctx) r.SetHeaders(headers) r.SetQueryParams(map[string]string{ "fsid": fmt.Sprint(file.Fsid), }) - }, &downloadUrl) + }, nil) + if err != nil { return nil, err } + if resp.StatusCode() != 302 { + return nil, fmt.Errorf("not found 302 redirect") + } + + location := resp.Header().Get("Location") link := &model.Link{ - URL: downloadUrl.Dlink, + URL: location, Header: http.Header{ "User-Agent": []string{headers["User-Agent"]}, "Referer": []string{"https://photo.baidu.com/"}, From 8316f81e4116425b491aa4eeee6afeb7d50b39f5 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 8 Sep 2024 23:03:58 +0800 Subject: [PATCH 323/659] ci: update beta tag to newest commit --- .github/workflows/beta_release.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index dc25928aa15..16e03eb2020 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -22,14 +22,19 @@ jobs: with: fetch-depth: 0 + - name: Create or update ref + id: create-or-update-ref + uses: ovsds/create-or-update-ref-action@v1 + with: + ref: tags/beta + sha: ${{ github.sha }} + - name: changelog # or changelogithub@0.12 if ensure the stable result id: changelog run: | git tag -l npx changelogithub --output CHANGELOG.md # npx changelogen@latest --output CHANGELOG.md - - - name: Upload assets uses: softprops/action-gh-release@v2 with: @@ -97,4 +102,5 @@ jobs: with: name: release_beta.yml owner: alist-org - repo: desktop-release \ No newline at end of file + repo: desktop-release + github-token: ${{ secrets.MY_TOKEN }} \ No newline at end of file From 73f0b135b6ee288fa0b5aabe7afdf7856e265f03 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 8 Sep 2024 23:58:53 +0800 Subject: [PATCH 324/659] ci: split arm and non-arm target on beta release workflow --- .github/workflows/beta_release.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 16e03eb2020..6dcef971faf 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -51,8 +51,10 @@ jobs: include: - target: '!(*musl*|*windows-arm64*|*android*)' # xgo hash: "md5" - - target: 'linux-*-musl*' #musl + - target: 'linux-!(arm*)-musl*' #musl-not-arm hash: "md5-linux-musl" + - target: 'linux-arm*-musl*' #musl-arm + hash: "md5-linux-musl-arm" - target: 'windows-arm64' #win-arm64 hash: "md5-windows-arm64" - target: 'android-*' #android @@ -98,9 +100,14 @@ jobs: name: Beta Release Desktop runs-on: ubuntu-latest steps: - - uses: dusansimic/trigger-workflow-action@v0 + - uses: peter-evans/create-or-update-comment@v4 with: - name: release_beta.yml - owner: alist-org - repo: desktop-release - github-token: ${{ secrets.MY_TOKEN }} \ No newline at end of file + issue-number: 69 + body: | + /release-beta + - triggered by ${{ github.actor }} + - commit sha: ${{ github.sha }} + - view files: https://github.com/alist-org/alist/tree/${{ github.sha }} + reactions: 'rocket' + token: ${{ secrets.MY_TOKEN }} + repository: alist-org/desktop-release \ No newline at end of file From 0310b70d9082da10bb99384c3756c3226b6e3037 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 00:01:51 +0800 Subject: [PATCH 325/659] fix(deps): update module golang.org/x/crypto to v0.27.0 (#7147) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 084a640a46e..b9fdab898ab 100644 --- a/go.mod +++ b/go.mod @@ -58,7 +58,7 @@ require ( github.com/xhofe/tache v0.1.2 github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.26.0 + golang.org/x/crypto v0.27.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 golang.org/x/net v0.28.0 @@ -217,9 +217,9 @@ require ( go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/term v0.23.0 // indirect - golang.org/x/text v0.17.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/term v0.24.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/tools v0.24.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect diff --git a/go.sum b/go.sum index 5a7f2eb31b3..e2613782933 100644 --- a/go.sum +++ b/go.sum @@ -548,6 +548,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -624,6 +626,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -636,6 +640,8 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= +golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -651,6 +657,8 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= From ffce61d22758a96acd0cc67178d2e2930dfdee7d Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 10 Sep 2024 00:02:24 +0800 Subject: [PATCH 326/659] ci: add @ to trigger by comment --- .github/workflows/beta_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 6dcef971faf..482195f9c3c 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -105,7 +105,7 @@ jobs: issue-number: 69 body: | /release-beta - - triggered by ${{ github.actor }} + - triggered by @${{ github.actor }} - commit sha: ${{ github.sha }} - view files: https://github.com/alist-org/alist/tree/${{ github.sha }} reactions: 'rocket' From bb58b94a108bd04fdb6771bc1bd385a7ea038fe3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 10 Sep 2024 00:02:47 +0800 Subject: [PATCH 327/659] fix(deps): update module github.com/charmbracelet/bubbles to v0.20.0 (#7142) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b9fdab898ab..8ec1c302760 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/aws/aws-sdk-go v1.55.5 github.com/blevesearch/bleve/v2 v2.4.2 github.com/caarlos0/env/v9 v9.0.0 - github.com/charmbracelet/bubbles v0.19.0 + github.com/charmbracelet/bubbles v0.20.0 github.com/charmbracelet/bubbletea v1.1.0 github.com/charmbracelet/lipgloss v0.13.0 github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e diff --git a/go.sum b/go.sum index e2613782933..6ba075f3b27 100644 --- a/go.sum +++ b/go.sum @@ -98,6 +98,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.19.0 h1:gKZkKXPP6GlDk6EcfujDK19PCQqRjaJZQ7QRERx1UF0= github.com/charmbracelet/bubbles v0.19.0/go.mod h1:WILteEqZ+krG5c3ntGEMeG99nCupcuIk7V0/zOP0tOA= +github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= +github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= github.com/charmbracelet/bubbletea v1.1.0 h1:FjAl9eAL3HBCHenhz/ZPjkKdScmaS5SK69JAK2YJK9c= github.com/charmbracelet/bubbletea v1.1.0/go.mod h1:9Ogk0HrdbHolIKHdjfFpyXJmiCzGwy+FesYkZr7hYU4= github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= From 1b42b9627cd811c621619bc9dcf0838fcda23b18 Mon Sep 17 00:00:00 2001 From: jindongh Date: Thu, 12 Sep 2024 04:08:13 -0700 Subject: [PATCH 328/659] fix(google_photo): fix issue copy videos from google photo (#7160 close #7158) #7158 During copy from google photo to aliyun, it failed consistently with 404 when copying mp4 file with =m37. Change =m37 to =dv will fix the issue --- drivers/google_photo/driver.go | 30 +++--------------------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/drivers/google_photo/driver.go b/drivers/google_photo/driver.go index 85a0520b4a5..b54132ef9ed 100644 --- a/drivers/google_photo/driver.go +++ b/drivers/google_photo/driver.go @@ -58,33 +58,9 @@ func (d *GooglePhoto) Link(ctx context.Context, file model.Obj, args model.LinkA URL: f.BaseURL + "=d", }, nil } else if strings.Contains(f.MimeType, "video/") { - var width, height int - - fmt.Sscanf(f.MediaMetadata.Width, "%d", &width) - fmt.Sscanf(f.MediaMetadata.Height, "%d", &height) - - switch { - // 1080P - case width == 1920 && height == 1080: - return &model.Link{ - URL: f.BaseURL + "=m37", - }, nil - // 720P - case width == 1280 && height == 720: - return &model.Link{ - URL: f.BaseURL + "=m22", - }, nil - // 360P - case width == 640 && height == 360: - return &model.Link{ - URL: f.BaseURL + "=m18", - }, nil - default: - return &model.Link{ - URL: f.BaseURL + "=dv", - }, nil - } - + return &model.Link{ + URL: f.BaseURL + "=dv", + }, nil } return &model.Link{}, nil } From 5d9167d6762b79efd83959bdb853508180bf95d1 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 13 Sep 2024 23:50:51 +0800 Subject: [PATCH 329/659] fix: recover panic on storage init --- internal/op/storage.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/op/storage.go b/internal/op/storage.go index 2f0831c452e..6790a8dffa6 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -2,6 +2,8 @@ package op import ( "context" + "fmt" + "runtime" "sort" "strings" "time" @@ -83,11 +85,25 @@ func LoadStorage(ctx context.Context, storage model.Storage) error { return err } +func getCurrentGoroutineStack() string { + buf := make([]byte, 1<<16) + n := runtime.Stack(buf, false) + return string(buf[:n]) +} + // initStorage initialize the driver and store to storagesMap func initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) { storageDriver.SetStorage(storage) driverStorage := storageDriver.GetStorage() - + defer func() { + if err := recover(); err != nil { + errInfo := fmt.Sprintf("[panic] err: %v\nstack: %s\n", err, getCurrentGoroutineStack()) + log.Errorf("panic init storage: %s", errInfo) + driverStorage.SetStatus(errInfo) + MustSaveDriverStorage(storageDriver) + storagesMap.Delete(driverStorage.MountPath) + } + }() // Unmarshal Addition err = utils.Json.UnmarshalFromString(driverStorage.Addition, storageDriver.GetAddition()) if err == nil { From b7ae56b109172759a965f030dd28bb658270a2df Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 14 Sep 2024 00:50:24 +0800 Subject: [PATCH 330/659] ci: delete beta tag before generating changelog --- .github/workflows/beta_release.yml | 5 +++++ .github/workflows/changelog.yml | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 482195f9c3c..32073eb9663 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -29,12 +29,17 @@ jobs: ref: tags/beta sha: ${{ github.sha }} + - name: Delete beta tag + run: git tag -d beta + continue-on-error: true + - name: changelog # or changelogithub@0.12 if ensure the stable result id: changelog run: | git tag -l npx changelogithub --output CHANGELOG.md # npx changelogen@latest --output CHANGELOG.md + - name: Upload assets uses: softprops/action-gh-release@v2 with: diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index 55efd9a8984..056883d6d96 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -15,6 +15,10 @@ jobs: with: fetch-depth: 0 + - name: Delete beta tag + run: git tag -d beta + continue-on-error: true + - run: npx changelogithub # or changelogithub@0.12 if ensure the stable result env: GITHUB_TOKEN: ${{secrets.MY_TOKEN}} From f06d2c03488b30d2184c9e383d759bb223898d44 Mon Sep 17 00:00:00 2001 From: lm379 <69002314+lm379@users.noreply.github.com> Date: Tue, 17 Sep 2024 01:34:47 +0800 Subject: [PATCH 331/659] fix(115): change ua (#7196 close #7191) --- drivers/115/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/115/util.go b/drivers/115/util.go index d88a9ce6381..992502c42c1 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -26,7 +26,7 @@ import ( "github.com/pkg/errors" ) -var UserAgent = driver115.UA115Desktop +var UserAgent = driver115.UA115Browser func (d *Pan115) login() error { var err error From b6451451b19c4584a41cac5d890fd369cf76bab8 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 17 Sep 2024 01:37:14 +0800 Subject: [PATCH 332/659] fix: release version (close #7182) --- build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sh b/build.sh index 5a7d0d666ef..18a30e633c2 100644 --- a/build.sh +++ b/build.sh @@ -8,6 +8,7 @@ if [ "$1" = "dev" ]; then version="dev" webVersion="dev" else + git tag -d beta version=$(git describe --abbrev=0 --tags) webVersion=$(wget -qO- -t1 -T2 "https://api.github.com/repos/alist-org/alist-web/releases/latest" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi From 6106a2d4cc275df59abb992b8ad8ad32d45f69ad Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Wed, 18 Sep 2024 23:30:28 +0800 Subject: [PATCH 333/659] fix: dynamic update app version (close #7198 in #7220) --- drivers/115/util.go | 23 ++++++++++++++++++++--- go.mod | 2 +- go.sum | 14 ++------------ 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/drivers/115/util.go b/drivers/115/util.go index 992502c42c1..ddddf6e9b15 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -74,9 +74,23 @@ func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { } const ( - appVer = "2.0.3.6" + appVer = "27.0.3.7" ) +func (c *Pan115) getAppVer() string { + // todo add some cache？ + vers, err := c.client.GetAppVersion() + if err != nil { + return appVer + } + for _, ver := range vers { + if ver.AppName == "win" { + return ver.Version + } + } + return appVer +} + func (c *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { key := crypto.GenerateKey() result := driver115.DownloadResp{} @@ -151,7 +165,7 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri userID := strconv.FormatInt(d.client.UserID, 10) form := url.Values{} form.Set("appid", "0") - form.Set("appversion", appVer) + form.Set("appversion", d.getAppVer()) form.Set("userid", userID) form.Set("filename", fileName) form.Set("filesize", fileSizeStr) @@ -161,7 +175,7 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri signKey, signVal := "", "" for retry := true; retry; { - t := driver115.Now() + t := driver115.NowMilli() if encodedToken, err = ecdhCipher.EncodeToken(t.ToInt64()); err != nil { return nil, err @@ -225,6 +239,9 @@ func UploadDigestRange(stream model.FileStreamer, rangeSpec string) (result stri length := end - start + 1 reader, err := stream.RangeRead(http_range.Range{Start: start, Length: length}) + if err != nil { + return "", err + } hashStr, err := utils.HashReader(utils.SHA1, reader) if err != nil { return "", err diff --git a/go.mod b/go.mod index 8ec1c302760..94e10ca1c42 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/alist-org/alist/v3 go 1.22.4 require ( - github.com/SheltonZhu/115driver v1.0.27 + github.com/SheltonZhu/115driver v1.0.29 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 diff --git a/go.sum b/go.sum index 6ba075f3b27..346a2d45125 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/SheltonZhu/115driver v1.0.27 h1:Ya1HYHYXFmi7JnqQ/+Vy6xZvq3leto+E+PxTm6UChj8= -github.com/SheltonZhu/115driver v1.0.27/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= +github.com/SheltonZhu/115driver v1.0.29 h1:yFBqFDYJyADo3eG2RjJgSovnFd1OrpGHmsHBi6j0+r4= +github.com/SheltonZhu/115driver v1.0.29/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= @@ -96,8 +96,6 @@ github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/bubbles v0.19.0 h1:gKZkKXPP6GlDk6EcfujDK19PCQqRjaJZQ7QRERx1UF0= -github.com/charmbracelet/bubbles v0.19.0/go.mod h1:WILteEqZ+krG5c3ntGEMeG99nCupcuIk7V0/zOP0tOA= github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= github.com/charmbracelet/bubbletea v1.1.0 h1:FjAl9eAL3HBCHenhz/ZPjkKdScmaS5SK69JAK2YJK9c= @@ -548,8 +546,6 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= @@ -626,8 +622,6 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -640,8 +634,6 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= -golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -657,8 +649,6 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From bdf4b52885299d8480dca554885811964fb0a94d Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Sat, 28 Sep 2024 23:15:58 +0800 Subject: [PATCH 334/659] feat(offline_download): add transmission (close #4102 in #7232) --- go.mod | 3 + go.sum | 6 + internal/conf/const.go | 12 +- internal/offline_download/all.go | 1 + internal/offline_download/tool/download.go | 13 ++ .../offline_download/transmission/client.go | 176 ++++++++++++++++++ server/handles/offline_download.go | 35 ++++ server/router.go | 10 +- 8 files changed, 248 insertions(+), 8 deletions(-) create mode 100644 internal/offline_download/transmission/client.go diff --git a/go.mod b/go.mod index 94e10ca1c42..9b9d859d3fe 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.0 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/hekmon/transmissionrpc/v3 v3.0.0 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 @@ -82,6 +83,8 @@ require ( github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect ) diff --git a/go.sum b/go.sum index 346a2d45125..f4699bc20b9 100644 --- a/go.sum +++ b/go.sum @@ -240,11 +240,17 @@ github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI0= +github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= +github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= +github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= diff --git a/internal/conf/const.go b/internal/conf/const.go index 2d53702e91a..13787b5e2ac 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -54,11 +54,15 @@ const ( Aria2Uri = "aria2_uri" Aria2Secret = "aria2_secret" + // transmission + TransmissionUri = "transmission_uri" + TransmissionSeedtime = "transmission_seedtime" + // single Token = "token" IndexProgress = "index_progress" - //SSO + // SSO SSOClientId = "sso_client_id" SSOClientSecret = "sso_client_secret" SSOLoginEnabled = "sso_login_enabled" @@ -73,7 +77,7 @@ const ( SSODefaultPermission = "sso_default_permission" SSOCompatibilityMode = "sso_compatibility_mode" - //ldap + // ldap LdapLoginEnabled = "ldap_login_enabled" LdapServer = "ldap_server" LdapManagerDN = "ldap_manager_dn" @@ -84,7 +88,7 @@ const ( LdapDefaultDir = "ldap_default_dir" LdapLoginTips = "ldap_login_tips" - //s3 + // s3 S3Buckets = "s3_buckets" S3AccessKeyId = "s3_access_key_id" S3SecretAccessKey = "s3_secret_access_key" @@ -97,7 +101,7 @@ const ( const ( UNKNOWN = iota FOLDER - //OFFICE + // OFFICE VIDEO AUDIO TEXT diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index ee80b5a0b8f..6682155dec8 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -6,4 +6,5 @@ import ( _ "github.com/alist-org/alist/v3/internal/offline_download/http" _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" + _ "github.com/alist-org/alist/v3/internal/offline_download/transmission" ) diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 4cc86a26124..ef9ceabfc8a 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -101,6 +101,19 @@ outer: } } } + + if t.tool.Name() == "transmission" { + // hack for transmission + seedTime := setting.GetInt(conf.TransmissionSeedtime, 0) + if seedTime >= 0 { + t.Status = "offline download completed, waiting for seeding" + <-time.After(time.Minute * time.Duration(seedTime)) + err := t.tool.Remove(t) + if err != nil { + log.Errorln(err.Error()) + } + } + } return nil } diff --git a/internal/offline_download/transmission/client.go b/internal/offline_download/transmission/client.go new file mode 100644 index 00000000000..a6075414814 --- /dev/null +++ b/internal/offline_download/transmission/client.go @@ -0,0 +1,176 @@ +package transmission + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/hekmon/transmissionrpc/v3" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type Transmission struct { + client *transmissionrpc.Client +} + +func (t *Transmission) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (t *Transmission) Name() string { + return "transmission" +} + +func (t *Transmission) Items() []model.SettingItem { + // transmission settings + return []model.SettingItem{ + {Key: conf.TransmissionUri, Value: "http://localhost:9091/transmission/rpc", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.TransmissionSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (t *Transmission) Init() (string, error) { + t.client = nil + uri := setting.GetStr(conf.TransmissionUri) + endpoint, err := url.Parse(uri) + if err != nil { + return "", errors.Wrap(err, "failed to init transmission client") + } + c, err := transmissionrpc.New(endpoint, nil) + if err != nil { + return "", errors.Wrap(err, "failed to init transmission client") + } + + ok, serverVersion, serverMinimumVersion, err := c.RPCVersion(context.Background()) + if err != nil { + return "", errors.Wrapf(err, "failed get transmission version") + } + + if !ok { + return "", fmt.Errorf("remote transmission RPC version (v%d) is incompatible with the transmission library (v%d): remote needs at least v%d", + serverVersion, transmissionrpc.RPCVersion, serverMinimumVersion) + } + + t.client = c + log.Infof("remote transmission RPC version (v%d) is compatible with our transmissionrpc library (v%d)\n", + serverVersion, transmissionrpc.RPCVersion) + log.Infof("using transmission version: %d", serverVersion) + return fmt.Sprintf("transmission version: %d", serverVersion), nil +} + +func (t *Transmission) IsReady() bool { + return t.client != nil +} + +func (t *Transmission) AddURL(args *tool.AddUrlArgs) (string, error) { + endpoint, err := url.Parse(args.Url) + if err != nil { + return "", errors.Wrap(err, "failed to parse transmission uri") + } + + rpcPayload := transmissionrpc.TorrentAddPayload{ + DownloadDir: &args.TempDir, + } + // http url for .torrent file + if endpoint.Scheme == "http" || endpoint.Scheme == "https" { + resp, err := http.Get(args.Url) + if err != nil { + return "", errors.Wrap(err, "failed to get .torrent file") + } + defer resp.Body.Close() + buffer := new(bytes.Buffer) + encoder := base64.NewEncoder(base64.StdEncoding, buffer) + // Stream file to the encoder + if _, err = io.Copy(encoder, resp.Body); err != nil { + return "", errors.Wrap(err, "can't copy file content into the base64 encoder") + } + // Flush last bytes + if err = encoder.Close(); err != nil { + return "", errors.Wrap(err, "can't flush last bytes of the base64 encoder") + } + // Get the string form + b64 := buffer.String() + rpcPayload.MetaInfo = &b64 + } else { // magnet uri + rpcPayload.Filename = &args.Url + } + + torrent, err := t.client.TorrentAdd(context.TODO(), rpcPayload) + if err != nil { + return "", err + } + + if torrent.ID == nil { + return "", fmt.Errorf("failed get torrent ID") + } + gid := strconv.FormatInt(*torrent.ID, 10) + return gid, nil +} + +func (t *Transmission) Remove(task *tool.DownloadTask) error { + gid, err := strconv.ParseInt(task.GID, 10, 64) + if err != nil { + return err + } + err = t.client.TorrentRemove(context.TODO(), transmissionrpc.TorrentRemovePayload{ + IDs: []int64{gid}, + DeleteLocalData: false, + }) + return err +} + +func (t *Transmission) Status(task *tool.DownloadTask) (*tool.Status, error) { + gid, err := strconv.ParseInt(task.GID, 10, 64) + if err != nil { + return nil, err + } + infos, err := t.client.TorrentGetAllFor(context.TODO(), []int64{gid}) + if err != nil { + return nil, err + } + + if len(infos) < 1 { + return nil, fmt.Errorf("failed get status, wrong gid: %s", task.GID) + } + info := infos[0] + + s := &tool.Status{ + Completed: *info.IsFinished, + Err: err, + } + s.Progress = *info.PercentDone * 100 + + switch *info.Status { + case transmissionrpc.TorrentStatusCheckWait, + transmissionrpc.TorrentStatusDownloadWait, + transmissionrpc.TorrentStatusCheck, + transmissionrpc.TorrentStatusDownload, + transmissionrpc.TorrentStatusIsolated: + s.Status = "[transmission] " + info.Status.String() + case transmissionrpc.TorrentStatusSeedWait, + transmissionrpc.TorrentStatusSeed: + s.Completed = true + case transmissionrpc.TorrentStatusStopped: + s.Err = errors.Errorf("[transmission] failed to download %s, status: %s, error: %s", task.GID, info.Status.String(), *info.ErrorString) + default: + s.Err = errors.Errorf("[transmission] unknown status occurred downloading %s, err: %s", task.GID, *info.ErrorString) + } + return s, nil +} + +var _ tool.Tool = (*Transmission)(nil) + +func init() { + tool.Tools.Add(&Transmission{}) +} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 0b019e9e48c..1c5f95557ff 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -30,6 +30,10 @@ func SetAria2(c *gin.Context) { return } _tool, err := tool.Tools.Get("aria2") + if err != nil { + common.ErrorResp(c, err, 500) + return + } version, err := _tool.Init() if err != nil { common.ErrorResp(c, err, 500) @@ -74,6 +78,37 @@ func OfflineDownloadTools(c *gin.Context) { common.SuccessResp(c, tools) } +type SetTransmissionReq struct { + Uri string `json:"uri" form:"uri"` + Seedtime string `json:"seedtime" form:"seedtime"` +} + +func SetTransmission(c *gin.Context) { + var req SetTransmissionReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + items := []model.SettingItem{ + {Key: conf.TransmissionUri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.TransmissionSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("transmission") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + type AddOfflineDownloadReq struct { Urls []string `json:"urls"` Path string `json:"path"` diff --git a/server/router.go b/server/router.go index 5be593f7497..07423f923cd 100644 --- a/server/router.go +++ b/server/router.go @@ -62,7 +62,7 @@ func Init(e *gin.Engine) { api.GET("/auth/get_sso_id", handles.SSOLoginCallback) api.GET("/auth/sso_get_token", handles.SSOLoginCallback) - //webauthn + // webauthn webauthn.GET("/webauthn_begin_registration", handles.BeginAuthnRegistration) webauthn.POST("/webauthn_finish_registration", handles.FinishAuthnRegistration) webauthn.GET("/webauthn_begin_login", handles.BeginAuthnLogin) @@ -125,6 +125,7 @@ func admin(g *gin.RouterGroup) { setting.POST("/reset_token", handles.ResetToken) setting.POST("/set_aria2", handles.SetAria2) setting.POST("/set_qbit", handles.SetQbittorrent) + setting.POST("/set_transmission", handles.SetTransmission) task := g.Group("/task") handles.SetupTaskRoute(task) @@ -159,14 +160,15 @@ func _fs(g *gin.RouterGroup) { g.PUT("/put", middlewares.FsUp, handles.FsStream) g.PUT("/form", middlewares.FsUp, handles.FsForm) g.POST("/link", middlewares.AuthAdmin, handles.Link) - //g.POST("/add_aria2", handles.AddOfflineDownload) - //g.POST("/add_qbit", handles.AddQbittorrent) + // g.POST("/add_aria2", handles.AddOfflineDownload) + // g.POST("/add_qbit", handles.AddQbittorrent) + // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) } func Cors(r *gin.Engine) { config := cors.DefaultConfig() - //config.AllowAllOrigins = true + // config.AllowAllOrigins = true config.AllowOrigins = conf.Conf.Cors.AllowOrigins config.AllowHeaders = conf.Conf.Cors.AllowHeaders config.AllowMethods = conf.Conf.Cors.AllowMethods From 5f19d73fcc57d85c6d40753f823867c534b45b36 Mon Sep 17 00:00:00 2001 From: URenko <18209292+URenko@users.noreply.github.com> Date: Fri, 4 Oct 2024 07:46:10 +0000 Subject: [PATCH 335/659] fix: Terabox ( close #6961 close #6983 in #7279) --- drivers/terabox/driver.go | 66 +++++++++++++++++++++++++++++++++------ drivers/terabox/types.go | 4 +++ drivers/terabox/util.go | 41 ++++++++++++++---------- 3 files changed, 84 insertions(+), 27 deletions(-) diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index c9662fce03a..11db351b75c 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -11,6 +11,7 @@ import ( stdpath "path" "strconv" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" @@ -23,7 +24,9 @@ import ( type Terabox struct { model.Storage Addition - JsToken string + JsToken string + url_domain_prefix string + base_url string } func (d *Terabox) Config() driver.Config { @@ -36,6 +39,8 @@ func (d *Terabox) GetAddition() driver.Additional { func (d *Terabox) Init(ctx context.Context) error { var resp CheckLoginResp + d.base_url = "https://www.terabox.com" + d.url_domain_prefix = "jp" _, err := d.get("/api/check/login", nil, &resp) if err != nil { return err @@ -71,7 +76,16 @@ func (d *Terabox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Terabox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.create(stdpath.Join(parentDir.GetPath(), dirName), 0, 1, "", "") + params := map[string]string{ + "a": "commit", + } + data := map[string]string{ + "path": stdpath.Join(parentDir.GetPath(), dirName), + "isdir": "1", + "block_list": "[]", + } + res, err := d.post_form("/api/create", params, data, nil) + log.Debugln(string(res)) return err } @@ -117,6 +131,20 @@ func (d *Terabox) Remove(ctx context.Context, obj model.Obj) error { } func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + resp, err := base.RestyClient.R(). + SetContext(ctx). + Get("https://" + d.url_domain_prefix + "-data.terabox.com/rest/2.0/pcs/file?method=locateupload") + if err != nil { + return err + } + var locateupload_resp LocateUploadResp + err = utils.Json.Unmarshal(resp.Body(), &locateupload_resp) + if err != nil { + log.Debugln(resp) + return err + } + log.Debugln(locateupload_resp) + tempFile, err := stream.CacheFullInTempFile() if err != nil { return err @@ -157,23 +185,28 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) path := encodeURIComponent(rawPath) block_list_str := fmt.Sprintf("[%s]", strings.Join(block_list, ",")) - data := fmt.Sprintf("path=%s&size=%d&isdir=0&autoinit=1&block_list=%s", - path, stream.GetSize(), - block_list_str) - params := map[string]string{} + data := map[string]string{ + "path": rawPath, + "autoinit": "1", + "target_path": dstDir.GetPath(), + "block_list": block_list_str, + "local_mtime": strconv.FormatInt(time.Now().Unix(), 10), + } var precreateResp PrecreateResp - _, err = d.post("/api/precreate", params, data, &precreateResp) + log.Debugln(data) + res, err := d.post_form("/api/precreate", nil, data, &precreateResp) if err != nil { return err } log.Debugf("%+v", precreateResp) if precreateResp.Errno != 0 { + log.Debugln(string(res)) return fmt.Errorf("[terabox] failed to precreate file, errno: %d", precreateResp.Errno) } if precreateResp.ReturnType == 2 { return nil } - params = map[string]string{ + params := map[string]string{ "method": "upload", "path": path, "uploadid": precreateResp.Uploadid, @@ -200,7 +233,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt if err != nil { return err } - u := "https://c-jp.terabox.com/rest/2.0/pcs/superfile2" + u := "https://" + locateupload_resp.Host + "/rest/2.0/pcs/superfile2" params["partseq"] = strconv.Itoa(partseq) res, err := base.RestyClient.R(). SetContext(ctx). @@ -216,7 +249,20 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt up(float64(i) * 100 / float64(len(precreateResp.BlockList))) } } - _, err = d.create(rawPath, stream.GetSize(), 0, precreateResp.Uploadid, block_list_str) + params = map[string]string{ + "isdir": "0", + "rtype": "1", + } + data = map[string]string{ + "path": rawPath, + "size": strconv.FormatInt(stream.GetSize(), 10), + "uploadid": precreateResp.Uploadid, + "target_path": dstDir.GetPath(), + "block_list": block_list_str, + "local_mtime": strconv.FormatInt(time.Now().Unix(), 10), + } + res, err = d.post_form("/api/create", params, data, nil) + log.Debugln(string(res)) return err } diff --git a/drivers/terabox/types.go b/drivers/terabox/types.go index 890d53056ea..8bdbc6fce1b 100644 --- a/drivers/terabox/types.go +++ b/drivers/terabox/types.go @@ -95,3 +95,7 @@ type PrecreateResp struct { type CheckLoginResp struct { Errno int `json:"errno"` } + +type LocateUploadResp struct { + Host string `json:"host"` +} diff --git a/drivers/terabox/util.go b/drivers/terabox/util.go index 0a4e7879379..e0f3d74e8f5 100644 --- a/drivers/terabox/util.go +++ b/drivers/terabox/util.go @@ -14,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) func getStrBetween(raw, start, end string) string { @@ -28,11 +29,11 @@ func getStrBetween(raw, start, end string) string { } func (d *Terabox) resetJsToken() error { - u := "https://www.terabox.com/main" + u := d.base_url res, err := base.RestyClient.R().SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", - "Referer": "https://www.terabox.com/", + "Referer": d.base_url, "User-Agent": base.UserAgent, "X-Requested-With": "XMLHttpRequest", }).Get(u) @@ -48,12 +49,12 @@ func (d *Terabox) resetJsToken() error { return nil } -func (d *Terabox) request(furl string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { +func (d *Terabox) request(rurl string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", - "Referer": "https://www.terabox.com/", + "Referer": d.base_url, "User-Agent": base.UserAgent, "X-Requested-With": "XMLHttpRequest", }) @@ -70,7 +71,7 @@ func (d *Terabox) request(furl string, method string, callback base.ReqCallback, if resp != nil { req.SetResult(resp) } - res, err := req.Execute(method, furl) + res, err := req.Execute(method, d.base_url+rurl) if err != nil { return nil, err } @@ -82,14 +83,20 @@ func (d *Terabox) request(furl string, method string, callback base.ReqCallback, return nil, err } if !utils.IsBool(noRetry...) { - return d.request(furl, method, callback, resp, true) + return d.request(rurl, method, callback, resp, true) } + } else if errno == -6 { + log.Debugln(res.Header()) + d.url_domain_prefix = res.Header()["Url-Domain-Prefix"][0] + d.base_url = "https://" + d.url_domain_prefix + ".terabox.com" + log.Debugln("Redirect base_url to", d.base_url) + return d.request(rurl, method, callback, resp, noRetry...) } return res.Body(), nil } func (d *Terabox) get(pathname string, params map[string]string, resp interface{}) ([]byte, error) { - return d.request("https://www.terabox.com"+pathname, http.MethodGet, func(req *resty.Request) { + return d.request(pathname, http.MethodGet, func(req *resty.Request) { if params != nil { req.SetQueryParams(params) } @@ -97,7 +104,7 @@ func (d *Terabox) get(pathname string, params map[string]string, resp interface{ } func (d *Terabox) post(pathname string, params map[string]string, data interface{}, resp interface{}) ([]byte, error) { - return d.request("https://www.terabox.com"+pathname, http.MethodPost, func(req *resty.Request) { + return d.request(pathname, http.MethodPost, func(req *resty.Request) { if params != nil { req.SetQueryParams(params) } @@ -105,6 +112,15 @@ func (d *Terabox) post(pathname string, params map[string]string, data interface }, resp) } +func (d *Terabox) post_form(pathname string, params map[string]string, data map[string]string, resp interface{}) ([]byte, error) { + return d.request(pathname, http.MethodPost, func(req *resty.Request) { + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(data) + }, resp) +} + func (d *Terabox) getFiles(dir string) ([]File, error) { page := 1 num := 100 @@ -237,15 +253,6 @@ func (d *Terabox) manage(opera string, filelist interface{}) ([]byte, error) { return d.post("/api/filemanager", params, data, nil) } -func (d *Terabox) create(path string, size int64, isdir int, uploadid, block_list string) ([]byte, error) { - params := map[string]string{} - data := fmt.Sprintf("path=%s&size=%d&isdir=%d", encodeURIComponent(path), size, isdir) - if uploadid != "" { - data += fmt.Sprintf("&uploadid=%s&block_list=%s", uploadid, block_list) - } - return d.post("/api/create", params, data, nil) -} - func encodeURIComponent(str string) string { r := url.QueryEscape(str) r = strings.ReplaceAll(r, "+", "%20") From c3e43ff60588f52f7b8e41c41c99980093b289fb Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 12 Oct 2024 00:48:54 +0800 Subject: [PATCH 336/659] fix(115): use latest appVer for upload (close #7315) --- drivers/115/appver.go | 43 +++++++++++++++++++++++++++++++++++++++++++ drivers/115/driver.go | 7 +++++-- drivers/115/util.go | 41 ++++++++++++++++++----------------------- 3 files changed, 66 insertions(+), 25 deletions(-) create mode 100644 drivers/115/appver.go diff --git a/drivers/115/appver.go b/drivers/115/appver.go new file mode 100644 index 00000000000..78e11a5443f --- /dev/null +++ b/drivers/115/appver.go @@ -0,0 +1,43 @@ +package _115 + +import ( + driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" + log "github.com/sirupsen/logrus" +) + +var ( + md5Salt = "Qclm8MGWUv59TnrR0XPg" + appVer = "27.0.5.7" +) + +func (d *Pan115) getAppVersion() ([]driver115.AppVersion, error) { + result := driver115.VersionResp{} + resp, err := base.RestyClient.R().Get(driver115.ApiGetVersion) + + err = driver115.CheckErr(err, &result, resp) + if err != nil { + return nil, err + } + + return result.Data.GetAppVersions(), nil +} + +func (d *Pan115) getAppVer() string { + // todo add some cache？ + vers, err := d.getAppVersion() + if err != nil { + log.Warnf("[115] get app version failed: %v", err) + return appVer + } + for _, ver := range vers { + if ver.AppName == "win" { + return ver.Version + } + } + return appVer +} + +func (d *Pan115) initAppVer() { + appVer = d.getAppVer() +} diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 2a1c8deef47..f6fb6b05618 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -3,6 +3,7 @@ package _115 import ( "context" "strings" + "sync" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/driver" @@ -16,8 +17,9 @@ import ( type Pan115 struct { model.Storage Addition - client *driver115.Pan115Client - limiter *rate.Limiter + client *driver115.Pan115Client + limiter *rate.Limiter + appVerOnce sync.Once } func (d *Pan115) Config() driver.Config { @@ -29,6 +31,7 @@ func (d *Pan115) GetAddition() driver.Additional { } func (d *Pan115) Init(ctx context.Context) error { + d.appVerOnce.Do(d.initAppVer) if d.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) } diff --git a/drivers/115/util.go b/drivers/115/util.go index ddddf6e9b15..7d5889af9f9 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -2,7 +2,9 @@ package _115 import ( "bytes" + "crypto/md5" "crypto/tls" + "encoding/hex" "encoding/json" "fmt" "io" @@ -26,12 +28,12 @@ import ( "github.com/pkg/errors" ) -var UserAgent = driver115.UA115Browser +//var UserAgent = driver115.UA115Browser func (d *Pan115) login() error { var err error opts := []driver115.Option{ - driver115.UA(UserAgent), + driver115.UA(d.getUA()), func(c *driver115.Pan115Client) { c.Client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) }, @@ -73,25 +75,11 @@ func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { return res, nil } -const ( - appVer = "27.0.3.7" -) - -func (c *Pan115) getAppVer() string { - // todo add some cache？ - vers, err := c.client.GetAppVersion() - if err != nil { - return appVer - } - for _, ver := range vers { - if ver.AppName == "win" { - return ver.Version - } - } - return appVer +func (d *Pan115) getUA() string { + return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } -func (c *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { +func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { key := crypto.GenerateKey() result := driver115.DownloadResp{} params, err := utils.Json.Marshal(map[string]string{"pickcode": pickCode}) @@ -105,10 +93,10 @@ func (c *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, e reqUrl := fmt.Sprintf("%s?t=%s", driver115.ApiDownloadGetUrl, driver115.Now().String()) req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Cookie", c.Cookie) + req.Header.Set("Cookie", d.Cookie) req.Header.Set("User-Agent", ua) - resp, err := c.client.Client.GetClient().Do(req) + resp, err := d.client.Client.GetClient().Do(req) if err != nil { return nil, err } @@ -146,6 +134,13 @@ func (c *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, e return nil, driver115.ErrUnexpected } +func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { + userID := strconv.FormatInt(c.client.UserID, 10) + userIDMd5 := md5.Sum([]byte(userID)) + tokenMd5 := md5.Sum([]byte(md5Salt + fileID + fileSize + signKey + signVal + userID + timeStamp + hex.EncodeToString(userIDMd5[:]) + appVer)) + return hex.EncodeToString(tokenMd5[:]) +} + func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID string, stream model.FileStreamer) (*driver115.UploadInitResp, error) { var ( ecdhCipher *cipher.EcdhCipher @@ -165,7 +160,7 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri userID := strconv.FormatInt(d.client.UserID, 10) form := url.Values{} form.Set("appid", "0") - form.Set("appversion", d.getAppVer()) + form.Set("appversion", appVer) form.Set("userid", userID) form.Set("filename", fileName) form.Set("filesize", fileSizeStr) @@ -186,7 +181,7 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri } form.Set("t", t.String()) - form.Set("token", d.client.GenerateToken(fileID, preID, t.String(), fileSizeStr, signKey, signVal)) + form.Set("token", d.GenerateToken(fileID, preID, t.String(), fileSizeStr, signKey, signVal)) if signKey != "" && signVal != "" { form.Set("sign_key", signKey) form.Set("sign_val", signVal) From e8538bd215a3c25a0b3c234a0aa5a470d972f436 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Mon, 14 Oct 2024 22:44:20 +0800 Subject: [PATCH 337/659] feat: add `febbox` driver (#7304 close #7293) --- drivers/all.go | 1 + drivers/febbox/driver.go | 132 +++++++++++++++++++++++ drivers/febbox/meta.go | 36 +++++++ drivers/febbox/oauth2.go | 88 +++++++++++++++ drivers/febbox/types.go | 123 +++++++++++++++++++++ drivers/febbox/util.go | 224 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 604 insertions(+) create mode 100644 drivers/febbox/driver.go create mode 100644 drivers/febbox/meta.go create mode 100644 drivers/febbox/oauth2.go create mode 100644 drivers/febbox/types.go create mode 100644 drivers/febbox/util.go diff --git a/drivers/all.go b/drivers/all.go index 40062a1aea1..4c4ef5c147b 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -22,6 +22,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/crypt" _ "github.com/alist-org/alist/v3/drivers/dropbox" + _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" diff --git a/drivers/febbox/driver.go b/drivers/febbox/driver.go new file mode 100644 index 00000000000..55c3aa211fe --- /dev/null +++ b/drivers/febbox/driver.go @@ -0,0 +1,132 @@ +package febbox + +import ( + "context" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type FebBox struct { + model.Storage + Addition + accessToken string + oauth2Token oauth2.TokenSource +} + +func (d *FebBox) Config() driver.Config { + return config +} + +func (d *FebBox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *FebBox) Init(ctx context.Context) error { + // 初始化 oauth2Config + oauth2Config := &clientcredentials.Config{ + ClientID: d.ClientID, + ClientSecret: d.ClientSecret, + AuthStyle: oauth2.AuthStyleInParams, + TokenURL: "https://api.febbox.com/oauth/token", + } + + d.initializeOAuth2Token(ctx, oauth2Config, d.Addition.RefreshToken) + + token, err := d.oauth2Token.Token() + if err != nil { + return err + } + d.accessToken = token.AccessToken + d.Addition.RefreshToken = token.RefreshToken + op.MustSaveDriverStorage(d) + + return nil +} + +func (d *FebBox) Drop(ctx context.Context) error { + return nil +} + +func (d *FebBox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFilesList(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *FebBox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var ip string + if d.Addition.UserIP != "" { + ip = d.Addition.UserIP + } else { + ip = args.IP + } + + url, err := d.getDownloadLink(file.GetID(), ip) + if err != nil { + return nil, err + } + return &model.Link{ + URL: url, + }, nil +} + +func (d *FebBox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + err := d.makeDir(parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (d *FebBox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + err := d.move(srcObj.GetID(), dstDir.GetID()) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (d *FebBox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + err := d.rename(srcObj.GetID(), newName) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (d *FebBox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + err := d.copy(srcObj.GetID(), dstDir.GetID()) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (d *FebBox) Remove(ctx context.Context, obj model.Obj) error { + err := d.remove(obj.GetID()) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*FebBox)(nil) diff --git a/drivers/febbox/meta.go b/drivers/febbox/meta.go new file mode 100644 index 00000000000..1daeeea8e52 --- /dev/null +++ b/drivers/febbox/meta.go @@ -0,0 +1,36 @@ +package febbox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + ClientID string `json:"client_id" required:"true" default:""` + ClientSecret string `json:"client_secret" required:"true" default:""` + RefreshToken string + SortRule string `json:"sort_rule" required:"true" type:"select" options:"size_asc,size_desc,name_asc,name_desc,update_asc,update_desc,ext_asc,ext_desc" default:"name_asc"` + PageSize int64 `json:"page_size" required:"true" type:"number" default:"100" help:"list api per page size of FebBox driver"` + UserIP string `json:"user_ip" default:"" help:"user ip address for download link which can speed up the download"` +} + +var config = driver.Config{ + Name: "FebBox", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: true, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &FebBox{} + }) +} diff --git a/drivers/febbox/oauth2.go b/drivers/febbox/oauth2.go new file mode 100644 index 00000000000..6345d1a711e --- /dev/null +++ b/drivers/febbox/oauth2.go @@ -0,0 +1,88 @@ +package febbox + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +type customTokenSource struct { + config *clientcredentials.Config + ctx context.Context + refreshToken string +} + +func (c *customTokenSource) Token() (*oauth2.Token, error) { + v := url.Values{} + if c.refreshToken != "" { + v.Set("grant_type", "refresh_token") + v.Set("refresh_token", c.refreshToken) + } else { + v.Set("grant_type", "client_credentials") + } + + v.Set("client_id", c.config.ClientID) + v.Set("client_secret", c.config.ClientSecret) + + req, err := http.NewRequest("POST", c.config.TokenURL, strings.NewReader(v.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := http.DefaultClient.Do(req.WithContext(c.ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.New("oauth2: cannot fetch token") + } + + var tokenResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + AccessToken string `json:"access_token"` + ExpiresIn int64 `json:"expires_in"` + TokenType string `json:"token_type"` + Scope string `json:"scope"` + RefreshToken string `json:"refresh_token"` + } `json:"data"` + } + + if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + return nil, err + } + + if tokenResp.Code != 1 { + return nil, errors.New("oauth2: server response error") + } + + c.refreshToken = tokenResp.Data.RefreshToken + + token := &oauth2.Token{ + AccessToken: tokenResp.Data.AccessToken, + TokenType: tokenResp.Data.TokenType, + RefreshToken: tokenResp.Data.RefreshToken, + Expiry: time.Now().Add(time.Duration(tokenResp.Data.ExpiresIn) * time.Second), + } + + return token, nil +} + +func (d *FebBox) initializeOAuth2Token(ctx context.Context, oauth2Config *clientcredentials.Config, refreshToken string) { + d.oauth2Token = oauth2.ReuseTokenSource(nil, &customTokenSource{ + config: oauth2Config, + ctx: ctx, + refreshToken: refreshToken, + }) +} diff --git a/drivers/febbox/types.go b/drivers/febbox/types.go new file mode 100644 index 00000000000..2ac6d6b76cc --- /dev/null +++ b/drivers/febbox/types.go @@ -0,0 +1,123 @@ +package febbox + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" + "strconv" + "time" +) + +type ErrResp struct { + ErrorCode int64 `json:"code"` + ErrorMsg string `json:"msg"` + ServerRunTime float64 `json:"server_runtime"` + ServerName string `json:"server_name"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ServerRunTime != 0 || e.ServerName != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ServerRunTime: %f ,ServerName: %s", e.ErrorCode, e.ErrorMsg, e.ServerRunTime, e.ServerName) +} + +type FileListResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileList []File `json:"file_list"` + ShowType string `json:"show_type"` + } `json:"data"` +} + +type Rules struct { + AllowCopy int64 `json:"allow_copy"` + AllowDelete int64 `json:"allow_delete"` + AllowDownload int64 `json:"allow_download"` + AllowComment int64 `json:"allow_comment"` + HideLocation int64 `json:"hide_location"` +} + +type File struct { + Fid int64 `json:"fid"` + UID int64 `json:"uid"` + FileSize int64 `json:"file_size"` + Path string `json:"path"` + FileName string `json:"file_name"` + Ext string `json:"ext"` + AddTime int64 `json:"add_time"` + FileCreateTime int64 `json:"file_create_time"` + FileUpdateTime int64 `json:"file_update_time"` + ParentID int64 `json:"parent_id"` + UpdateTime int64 `json:"update_time"` + LastOpenTime int64 `json:"last_open_time"` + IsDir int64 `json:"is_dir"` + Epub int64 `json:"epub"` + IsMusicList int64 `json:"is_music_list"` + OssFid int64 `json:"oss_fid"` + Faststart int64 `json:"faststart"` + HasVideoQuality int64 `json:"has_video_quality"` + TotalDownload int64 `json:"total_download"` + Status int64 `json:"status"` + Remark string `json:"remark"` + OldHash string `json:"old_hash"` + Hash string `json:"hash"` + HashType string `json:"hash_type"` + FromUID int64 `json:"from_uid"` + FidOrg int64 `json:"fid_org"` + ShareID int64 `json:"share_id"` + InvitePermission int64 `json:"invite_permission"` + ThumbSmall string `json:"thumb_small"` + ThumbSmallWidth int64 `json:"thumb_small_width"` + ThumbSmallHeight int64 `json:"thumb_small_height"` + Thumb string `json:"thumb"` + ThumbWidth int64 `json:"thumb_width"` + ThumbHeight int64 `json:"thumb_height"` + ThumbBig string `json:"thumb_big"` + ThumbBigWidth int64 `json:"thumb_big_width"` + ThumbBigHeight int64 `json:"thumb_big_height"` + IsCustomThumb int64 `json:"is_custom_thumb"` + Photos int64 `json:"photos"` + IsAlbum int64 `json:"is_album"` + ReadOnly int64 `json:"read_only"` + Rules Rules `json:"rules"` + IsShared int64 `json:"is_shared"` +} + +func fileToObj(f File) *model.ObjThumb { + return &model.ObjThumb{ + Object: model.Object{ + ID: strconv.FormatInt(f.Fid, 10), + Name: f.FileName, + Size: f.FileSize, + Ctime: time.Unix(f.FileCreateTime, 0), + Modified: time.Unix(f.FileUpdateTime, 0), + IsFolder: f.IsDir == 1, + HashInfo: utils.NewHashInfo(hash_extend.GCID, f.Hash), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: f.Thumb, + }, + } +} + +type FileDownloadResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data []struct { + Error int `json:"error"` + DownloadURL string `json:"download_url"` + Hash string `json:"hash"` + HashType string `json:"hash_type"` + Fid int `json:"fid"` + FileName string `json:"file_name"` + ParentID int `json:"parent_id"` + FileSize int `json:"file_size"` + Ext string `json:"ext"` + Thumb string `json:"thumb"` + VipLink int `json:"vip_link"` + } `json:"data"` +} diff --git a/drivers/febbox/util.go b/drivers/febbox/util.go new file mode 100644 index 00000000000..ac072edbde8 --- /dev/null +++ b/drivers/febbox/util.go @@ -0,0 +1,224 @@ +package febbox + +import ( + "encoding/json" + "errors" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/op" + "github.com/go-resty/resty/v2" + "net/http" + "strconv" +) + +func (d *FebBox) refreshTokenByOAuth2() error { + token, err := d.oauth2Token.Token() + if err != nil { + return err + } + d.Status = "work" + d.accessToken = token.AccessToken + d.Addition.RefreshToken = token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *FebBox) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + // 使用oauth2 获取 access_token + token, err := d.oauth2Token.Token() + if err != nil { + return nil, err + } + req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) + + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e ErrResp + req.SetError(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + + switch e.ErrorCode { + case 0: + return res.Body(), nil + case 1: + return res.Body(), nil + case -10001: + if e.ServerName != "" { + // access_token 过期 + if err = d.refreshTokenByOAuth2(); err != nil { + return nil, err + } + return d.request(url, method, callback, resp) + } else { + return nil, errors.New(e.Error()) + } + default: + return nil, errors.New(e.Error()) + } +} + +func (d *FebBox) getFilesList(id string) ([]File, error) { + if d.PageSize <= 0 { + d.PageSize = 100 + } + res, err := d.listWithLimit(id, d.PageSize) + if err != nil { + return nil, err + } + return *res, nil +} + +func (d *FebBox) listWithLimit(dirID string, pageLimit int64) (*[]File, error) { + var files []File + page := int64(1) + for { + result, err := d.getFiles(dirID, page, pageLimit) + if err != nil { + return nil, err + } + files = append(files, *result...) + if int64(len(*result)) < pageLimit { + break + } else { + page++ + } + } + return &files, nil +} + +func (d *FebBox) getFiles(dirID string, page, pageLimit int64) (*[]File, error) { + var fileList FileListResp + queryParams := map[string]string{ + "module": "file_list", + "parent_id": dirID, + "page": strconv.FormatInt(page, 10), + "pagelimit": strconv.FormatInt(pageLimit, 10), + "order": d.Addition.SortRule, + } + + res, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, &fileList) + if err != nil { + return nil, err + } + + if err = json.Unmarshal(res, &fileList); err != nil { + return nil, err + } + + return &fileList.Data.FileList, nil +} + +func (d *FebBox) getDownloadLink(id string, ip string) (string, error) { + var fileDownloadResp FileDownloadResp + queryParams := map[string]string{ + "module": "file_get_download_url", + "fids[]": id, + "ip": ip, + } + + res, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, &fileDownloadResp) + if err != nil { + return "", err + } + + if err = json.Unmarshal(res, &fileDownloadResp); err != nil { + return "", err + } + + return fileDownloadResp.Data[0].DownloadURL, nil +} + +func (d *FebBox) makeDir(id string, name string) error { + queryParams := map[string]string{ + "module": "create_dir", + "parent_id": id, + "name": name, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) move(id string, id2 string) error { + queryParams := map[string]string{ + "module": "file_move", + "fids[]": id, + "to": id2, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) rename(id string, name string) error { + queryParams := map[string]string{ + "module": "file_rename", + "fid": id, + "name": name, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) copy(id string, id2 string) error { + queryParams := map[string]string{ + "module": "file_copy", + "fids[]": id, + "to": id2, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) remove(id string) error { + queryParams := map[string]string{ + "module": "file_delete", + "fids[]": id, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} From 2830575490e72a7afc8fd8e6b790a163b92a13b5 Mon Sep 17 00:00:00 2001 From: hanbao233xD <39661586+hanbao233xD@users.noreply.github.com> Date: Tue, 15 Oct 2024 19:45:30 +0800 Subject: [PATCH 338/659] perf(123pan): change domain of login (#7325) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update driver.go * 1 * Update util.go * 123新登录接口 * Revert "Update util.go" This reverts commit a13a58f8a86c7c36d4fd7d91137229a7667f1fb5. * Update driver.go * Update util.go * Update util.go --- drivers/123/driver.go | 1 + drivers/123/util.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index aeda7fcf742..3620431d9b3 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -82,6 +82,7 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) "type": f.Type, } resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) { + req.SetBody(data).SetHeaders(headers) }, nil) if err != nil { diff --git a/drivers/123/util.go b/drivers/123/util.go index 73c73b3b3b3..6365b1c9a1e 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -26,8 +26,9 @@ const ( Api = "https://www.123pan.com/api" AApi = "https://www.123pan.com/a/api" BApi = "https://www.123pan.com/b/api" + LoginApi = "https://login.123pan.com/api" MainApi = BApi - SignIn = MainApi + "/user/sign_in" + SignIn = LoginApi + "/user/sign_in" Logout = MainApi + "/user/logout" UserInfo = MainApi + "/user/info" FileList = MainApi + "/file/list/new" From 48ac23c8de98e1bf6b6acf51e795c75b451493a7 Mon Sep 17 00:00:00 2001 From: Jason-Fly <869914918@qq.com> Date: Sun, 20 Oct 2024 23:53:40 +0800 Subject: [PATCH 339/659] fix(ilanzou): fix infinite loop when getting file list (#7366 close #7357) --- drivers/ilanzou/driver.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index ab5ebe7ee5d..24fcc436a0c 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -66,12 +66,13 @@ func (d *ILanZou) Drop(ctx context.Context) error { } func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + offset := 1 var res []ListItem for { var resp ListResp _, err := d.proved("/record/file/list", http.MethodGet, func(req *resty.Request) { params := []string{ - "offset=1", + "offset=" + strconv.Itoa(offset), "limit=60", "folderId=" + dir.GetID(), "type=0", @@ -83,7 +84,9 @@ func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } res = append(res, resp.List...) - if resp.TotalPage <= resp.Offset { + if resp.Offset < resp.TotalPage { + offset++ + } else { break } } From a2dc45a80bd15a3e7373a257bd2e81a02632d1b5 Mon Sep 17 00:00:00 2001 From: Jason-Fly <869914918@qq.com> Date: Sun, 20 Oct 2024 23:53:56 +0800 Subject: [PATCH 340/659] fix(ilanzou): fix upload failure for small files (#7368 close #7250) --- drivers/ilanzou/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 24fcc436a0c..90ef7c1a910 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -289,7 +289,7 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt req.SetBody(base.Json{ "fileId": "", "fileName": stream.GetName(), - "fileSize": stream.GetSize() / 1024, + "fileSize": stream.GetSize()/1024 + 1, "folderId": dstDir.GetID(), "md5": etag, "type": 1, From a701432b8bf5b43a78382d75e9090ed66c03a570 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 21 Oct 2024 00:05:56 +0800 Subject: [PATCH 341/659] ci: add freebsd to beta release --- .github/workflows/beta_release.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 32073eb9663..90c2836fd2d 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -8,6 +8,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +permissions: + contents: write + jobs: changelog: strategy: @@ -54,7 +57,7 @@ jobs: strategy: matrix: include: - - target: '!(*musl*|*windows-arm64*|*android*)' # xgo + - target: '!(*musl*|*windows-arm64*|*android*|*freebsd*)' # xgo hash: "md5" - target: 'linux-!(arm*)-musl*' #musl-not-arm hash: "md5-linux-musl" @@ -64,6 +67,9 @@ jobs: hash: "md5-windows-arm64" - target: 'android-*' #android hash: "md5-android" + - target: 'freebsd-*' #freebsd + hash: "md5-freebsd" + name: Beta Release runs-on: ubuntu-latest steps: From 216e3909f3946eb9c1b786c0d82c00f278f0ea25 Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Fri, 1 Nov 2024 20:52:19 +0800 Subject: [PATCH 342/659] fix(115): enforce 20GB file size limit on uploadev (#7447 close #7413) - Introduce a file size restriction to handle uploads more securely. - Provide an informative error for uploads that exceed the new limit. --- drivers/115/driver.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index f6fb6b05618..4857c1ec05e 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -2,6 +2,7 @@ package _115 import ( "context" + "fmt" "strings" "sync" @@ -121,7 +122,10 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr if err := d.WaitLimit(ctx); err != nil { return err } - + if stream.GetSize() > utils.GB*20 { // TODO 由于官方分片上传接口失效，所以使用普通上传小于20GB的文件 + return fmt.Errorf("unsupported file size: 20GB limit exceeded") + } + // 分片上传 var ( fastInfo *driver115.UploadInitResp dirID = dstDir.GetID() @@ -177,11 +181,13 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } // 闪传失败，上传 - if stream.GetSize() <= utils.KB { // 文件大小小于1KB，改用普通模式上传 + // if stream.GetSize() <= utils.KB{ // 文件大小小于1KB，改用普通模式上传 + if stream.GetSize() <= utils.GB*20 { // TODO 由于官方分片上传接口失效，所以使用普通上传小于20GB的文件 return d.client.UploadByOSS(&fastInfo.UploadOSSParams, stream, dirID) } + return driver115.ErrUnexpected // 分片上传 - return d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID) + // return d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID) } func (d *Pan115) OfflineList(ctx context.Context) ([]*driver115.OfflineTask, error) { From 4955d8cec8a839b95e66c620e3ca69c1348e65eb Mon Sep 17 00:00:00 2001 From: Mmx Date: Fri, 1 Nov 2024 20:53:53 +0800 Subject: [PATCH 343/659] ci(docker): support riscv64 and ppc64le (#7426) * ci(docker): bump cache key of musl library * build(docker): add new arches to build script * ci(docker): add new arches to buildx platforms --- .github/workflows/build_docker.yml | 6 +++--- .github/workflows/release_docker.yml | 6 +++--- build.sh | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml index 8f37688d07f..6384c374bf6 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/build_docker.yml @@ -53,7 +53,7 @@ jobs: uses: actions/cache@v4 with: path: build/musl-libs - key: docker-musl-libs + key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' @@ -84,7 +84,7 @@ jobs: push: ${{ github.event_name == 'push' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64 - name: Build and push with ffmpeg id: docker_build_ffmpeg @@ -96,7 +96,7 @@ jobs: tags: ${{ steps.meta-ffmpeg.outputs.tags }} labels: ${{ steps.meta-ffmpeg.outputs.labels }} build-args: INSTALL_FFMPEG=true - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64 build_docker_with_aria2: needs: build_docker diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 95a686b2fce..a2dd2dd72d8 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -22,7 +22,7 @@ jobs: uses: actions/cache@v4 with: path: build/musl-libs - key: docker-musl-libs + key: docker-musl-libs-v2 - name: Download Musl Library if: steps.cache-musl.outputs.cache-hit != 'true' @@ -58,7 +58,7 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64 - name: Docker meta with ffmpeg id: meta-ffmpeg @@ -79,7 +79,7 @@ jobs: tags: ${{ steps.meta-ffmpeg.outputs.tags }} labels: ${{ steps.meta-ffmpeg.outputs.labels }} build-args: INSTALL_FFMPEG=true - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x + platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64 release_docker_with_aria2: needs: release_docker diff --git a/build.sh b/build.sh index 18a30e633c2..6b28847c3b3 100644 --- a/build.sh +++ b/build.sh @@ -93,7 +93,7 @@ BuildDocker() { PrepareBuildDockerMusl() { mkdir -p build/musl-libs BASE="https://musl.cc/" - FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross) + FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross riscv64-linux-musl-cross powerpc64le-linux-musl-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" lib_tgz="build/${i}.tgz" @@ -112,8 +112,8 @@ BuildDockerMultiplatform() { docker_lflags="--extldflags '-static -fpic' $ldflags" export CGO_ENABLED=1 - OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-s390x) - CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc s390x-linux-musl-gcc) + OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-s390x linux-riscv64 linux-ppc64le) + CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc s390x-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc) for i in "${!OS_ARCHES[@]}"; do os_arch=${OS_ARCHES[$i]} cgo_cc=${CGO_ARGS[$i]} From 34a148c83de62258322228b43e63b59f0b2f1801 Mon Sep 17 00:00:00 2001 From: Mmx Date: Fri, 1 Nov 2024 20:58:53 +0800 Subject: [PATCH 344/659] feat(local): thumbnail token bucket smooth migration (#7425) * feat(local): allow to migrate static token buckets * improve(local): token bucket migration boundary handling --- drivers/local/driver.go | 2 +- drivers/local/token_bucket.go | 38 +++++++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index bf993e5d5f8..86980943ef5 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -76,7 +76,7 @@ func (d *Local) Init(ctx context.Context) error { if d.thumbConcurrency == 0 { d.thumbTokenBucket = NewNopTokenBucket() } else { - d.thumbTokenBucket = NewStaticTokenBucket(d.thumbConcurrency) + d.thumbTokenBucket = NewStaticTokenBucketWithMigration(d.thumbTokenBucket, d.thumbConcurrency) } return nil } diff --git a/drivers/local/token_bucket.go b/drivers/local/token_bucket.go index 38fbe73fc9b..23c6ebd63b7 100644 --- a/drivers/local/token_bucket.go +++ b/drivers/local/token_bucket.go @@ -23,6 +23,38 @@ func NewStaticTokenBucket(size int) StaticTokenBucket { return StaticTokenBucket{bucket: bucket} } +func NewStaticTokenBucketWithMigration(oldBucket TokenBucket, size int) StaticTokenBucket { + if oldBucket != nil { + oldStaticBucket, ok := oldBucket.(StaticTokenBucket) + if ok { + oldSize := cap(oldStaticBucket.bucket) + migrateSize := oldSize + if size < migrateSize { + migrateSize = size + } + + bucket := make(chan struct{}, size) + for range size - migrateSize { + bucket <- struct{}{} + } + + if migrateSize != 0 { + go func() { + for range migrateSize { + <-oldStaticBucket.bucket + bucket <- struct{}{} + } + close(oldStaticBucket.bucket) + }() + } + return StaticTokenBucket{bucket: bucket} + } + } + return NewStaticTokenBucket(size) +} + +// Take channel maybe closed when local driver is modified. +// don't call Put method after the channel is closed. func (b StaticTokenBucket) Take() <-chan struct{} { return b.bucket } @@ -35,8 +67,10 @@ func (b StaticTokenBucket) Do(ctx context.Context, f func() error) error { select { case <-ctx.Done(): return ctx.Err() - case <-b.bucket: - defer b.Put() + case _, ok := <-b.Take(): + if ok { + defer b.Put() + } } return f() } From ce0b99a510c227a27c34f2b442c5e0794d2488f3 Mon Sep 17 00:00:00 2001 From: Maxwell Davis <138968347+Unic96@users.noreply.github.com> Date: Fri, 1 Nov 2024 21:12:29 +0800 Subject: [PATCH 345/659] fix(cloudreve): path not exist when moving/copying files (#7432) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 马建军 <1432318228@qq.com> --- drivers/cloudreve/driver.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index dc6d1b13213..ec0f6ef2b29 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -4,6 +4,7 @@ import ( "context" "io" "net/http" + "path" "strconv" "strings" @@ -90,7 +91,7 @@ func (d *Cloudreve) MakeDir(ctx context.Context, parentDir model.Obj, dirName st func (d *Cloudreve) Move(ctx context.Context, srcObj, dstDir model.Obj) error { body := base.Json{ "action": "move", - "src_dir": srcObj.GetPath(), + "src_dir": path.Dir(srcObj.GetPath()), "dst": dstDir.GetPath(), "src": convertSrc(srcObj), } @@ -112,7 +113,7 @@ func (d *Cloudreve) Rename(ctx context.Context, srcObj model.Obj, newName string func (d *Cloudreve) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { body := base.Json{ - "src_dir": srcObj.GetPath(), + "src_dir": path.Dir(srcObj.GetPath()), "dst": dstDir.GetPath(), "src": convertSrc(srcObj), } From d0cda62703f48d45abebd25506fa11e0eea54a24 Mon Sep 17 00:00:00 2001 From: UUBulb <35923940+uubulb@users.noreply.github.com> Date: Fri, 1 Nov 2024 21:37:53 +0800 Subject: [PATCH 346/659] ci: add freebsd release build (#7344) --- .github/workflows/release_freebsd.yml | 35 +++++++++++++++++++++++++++ build.sh | 31 ++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 .github/workflows/release_freebsd.yml diff --git a/.github/workflows/release_freebsd.yml b/.github/workflows/release_freebsd.yml new file mode 100644 index 00000000000..46afb326bf3 --- /dev/null +++ b/.github/workflows/release_freebsd.yml @@ -0,0 +1,35 @@ +name: release_freebsd + +on: + release: + types: [ published ] + +jobs: + release_freebsd: + strategy: + matrix: + platform: [ ubuntu-latest ] + go-version: [ '1.21' ] + name: Release + runs-on: ${{ matrix.platform }} + steps: + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build + run: | + bash build.sh release freebsd + + - name: Upload assets + uses: softprops/action-gh-release@v2 + with: + tag_name: dev + files: build/compress/* diff --git a/build.sh b/build.sh index 6b28847c3b3..a87eabf4f81 100644 --- a/build.sh +++ b/build.sh @@ -233,6 +233,29 @@ BuildReleaseAndroid() { done } +BuildReleaseFreeBSD() { + rm -rf .git/ + mkdir -p "build/freebsd" + OS_ARCHES=(amd64 arm64 i386) + GO_ARCHES=(amd64 arm64 386) + CGO_ARGS=(x86_64-unknown-freebsd14.1 aarch64-unknown-freebsd14.1 i386-unknown-freebsd14.1) + for i in "${!OS_ARCHES[@]}"; do + os_arch=${OS_ARCHES[$i]} + cgo_cc="clang --target=${CGO_ARGS[$i]} --sysroot=/opt/freebsd/${os_arch}" + echo building for freebsd-${os_arch} + sudo mkdir -p "/opt/freebsd/${os_arch}" + wget -q https://download.freebsd.org/releases/${os_arch}/14.1-RELEASE/base.txz + sudo tar -xf ./base.txz -C /opt/freebsd/${os_arch} + rm base.txz + export GOOS=freebsd + export GOARCH=${GO_ARCHES[$i]} + export CC=${cgo_cc} + export CGO_ENABLED=1 + export CGO_LDFLAGS="-fuse-ld=lld" + go build -o ./build/$appName-freebsd-$os_arch -ldflags="$ldflags" -tags=jsoniter . + done +} + MakeRelease() { cd build mkdir compress @@ -251,6 +274,11 @@ MakeRelease() { tar -czvf compress/"$i".tar.gz alist rm -f alist done + for i in $(find . -type f -name "$appName-freebsd-*"); do + cp "$i" alist + tar -czvf compress/"$i".tar.gz alist + rm -f alist + done for i in $(find . -type f -name "$appName-windows-*"); do cp "$i" alist.exe zip compress/$(echo $i | sed 's/\.[^.]*$//').zip alist.exe @@ -288,6 +316,9 @@ elif [ "$1" = "release" ]; then elif [ "$2" = "android" ]; then BuildReleaseAndroid MakeRelease "md5-android.txt" + elif [ "$2" = "freebsd" ]; then + BuildReleaseFreeBSD + MakeRelease "md5-freebsd.txt" elif [ "$2" = "web" ]; then echo "web only" else From 10c7ebb1c0c917e848c53453f4644ef1f7b18922 Mon Sep 17 00:00:00 2001 From: Rirmach Date: Fri, 1 Nov 2024 23:31:33 +0800 Subject: [PATCH 347/659] fix(local): cross-device file move (#7430) --- drivers/local/driver.go | 36 ++++++++++++++++++++++-------------- go.mod | 1 + go.sum | 2 ++ 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 86980943ef5..c39cec10c6b 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -22,6 +22,7 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/times" + cp "github.com/otiai10/copy" log "github.com/sirupsen/logrus" _ "golang.org/x/image/webp" ) @@ -241,11 +242,22 @@ func (d *Local) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } - err := os.Rename(srcPath, dstPath) - if err != nil { + if err := os.Rename(srcPath, dstPath); err != nil && strings.Contains(err.Error(), "invalid cross-device link") { + // Handle cross-device file move in local driver + if err = d.Copy(ctx, srcObj, dstDir); err != nil { + return err + } else { + // Directly remove file without check recycle bin if successfully copied + if srcObj.IsDir() { + err = os.RemoveAll(srcObj.GetPath()) + } else { + err = os.Remove(srcObj.GetPath()) + } + return err + } + } else { return err } - return nil } func (d *Local) Rename(ctx context.Context, srcObj model.Obj, newName string) error { @@ -258,22 +270,18 @@ func (d *Local) Rename(ctx context.Context, srcObj model.Obj, newName string) er return nil } -func (d *Local) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *Local) Copy(_ context.Context, srcObj, dstDir model.Obj) error { srcPath := srcObj.GetPath() dstPath := filepath.Join(dstDir.GetPath(), srcObj.GetName()) if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } - var err error - if srcObj.IsDir() { - err = utils.CopyDir(srcPath, dstPath) - } else { - err = utils.CopyFile(srcPath, dstPath) - } - if err != nil { - return err - } - return nil + // Copy using otiai10/copy to perform more secure & efficient copy + return cp.Copy(srcPath, dstPath, cp.Options{ + Sync: true, // Sync file to disk after copy, may have performance penalty in filesystem such as ZFS + PreserveTimes: true, + NumOfWorkers: 0, // Serialized copy without using goroutine + }) } func (d *Local) Remove(ctx context.Context, obj model.Obj) error { diff --git a/go.mod b/go.mod index 9b9d859d3fe..45e2c643aa0 100644 --- a/go.mod +++ b/go.mod @@ -189,6 +189,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect + github.com/otiai10/copy v1.14.0 github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect diff --git a/go.sum b/go.sum index f4699bc20b9..420a259f370 100644 --- a/go.sum +++ b/go.sum @@ -391,6 +391,8 @@ github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu4h5aYIOzUtLjN08L4Qt4WGaJONMgcaD0ayBJQ= github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= From 64ceb5afb6ea94b0a71367c4d9cfa4a6a68dddc3 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Fri, 1 Nov 2024 23:32:26 +0800 Subject: [PATCH 348/659] feat: support general users view and cancel own tasks (#7416 close #7398) * feat: support general users view and cancel own tasks Add a creator attribute to the upload, copy and offline download tasks, so that a GENERAL task creator can view and cancel them. BREAKING CHANGE: 1. A new internal package `task` including the struct `TaskWithCreator` which embeds `tache.Base` is created, and the past dependence on `tache.Task` will all be transferred to dependence on this package. 2. The API `/admin/task` can now also be accessed via `/task`, and the old endpoint is retained to ensure compatibility with legacy automation scripts. Closes #7398 * fix(deps): update github.com/xhofe/tache to v0.1.3 --- go.mod | 2 +- go.sum | 2 + internal/fs/copy.go | 12 +- internal/fs/fs.go | 8 +- internal/fs/put.go | 9 +- internal/offline_download/tool/add.go | 11 +- internal/offline_download/tool/download.go | 6 +- internal/offline_download/tool/transfer.go | 3 +- internal/task/base.go | 26 ++++ server/handles/fsmanage.go | 4 +- server/handles/fsup.go | 13 +- server/handles/offline_download.go | 4 +- server/handles/task.go | 162 ++++++++++++++++----- server/middlewares/auth.go | 10 ++ server/router.go | 9 +- 15 files changed, 215 insertions(+), 66 deletions(-) create mode 100644 internal/task/base.go diff --git a/go.mod b/go.mod index 45e2c643aa0..19bc7c2e627 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 - github.com/xhofe/tache v0.1.2 + github.com/xhofe/tache v0.1.3 github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.27.0 diff --git a/go.sum b/go.sum index 420a259f370..78ac273a5bf 100644 --- a/go.sum +++ b/go.sum @@ -514,6 +514,8 @@ github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3K github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.2 h1:pHrXlrWcbTb4G7hVUDW7Rc+YTUnLJvnLBrdktVE1Fqg= github.com/xhofe/tache v0.1.2/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/xhofe/tache v0.1.3 h1:MipxzlljYX29E1YI/SLC7hVomVF+51iP1OUzlsuq1wE= +github.com/xhofe/tache v0.1.3/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/internal/fs/copy.go b/internal/fs/copy.go index 38407c9a863..d4ad452b169 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -11,13 +11,14 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/task" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" "github.com/xhofe/tache" ) type CopyTask struct { - tache.Base + task.TaskWithCreator Status string `json:"-"` //don't save status to save space SrcObjPath string `json:"src_path"` DstDirPath string `json:"dst_path"` @@ -53,7 +54,7 @@ var CopyTaskManager *tache.Manager[*CopyTask] // Copy if in the same storage, call move method // if not, add copy task -func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (tache.TaskWithInfo, error) { +func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskInfoWithCreator, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { return nil, errors.WithMessage(err, "failed get src storage") @@ -92,7 +93,11 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool } } // not in the same storage + taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed t := &CopyTask{ + TaskWithCreator: task.TaskWithCreator{ + Creator: taskCreator, + }, srcStorage: srcStorage, dstStorage: dstStorage, SrcObjPath: srcObjActualPath, @@ -123,6 +128,9 @@ func copyBetween2Storages(t *CopyTask, srcStorage, dstStorage driver.Driver, src srcObjPath := stdpath.Join(srcObjPath, obj.GetName()) dstObjPath := stdpath.Join(dstDirPath, srcObj.GetName()) CopyTaskManager.Add(&CopyTask{ + TaskWithCreator: task.TaskWithCreator{ + Creator: t.Creator, + }, srcStorage: srcStorage, dstStorage: dstStorage, SrcObjPath: srcObjPath, diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 23e8a87a6fd..65e5a2c264a 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -5,8 +5,8 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" log "github.com/sirupsen/logrus" - "github.com/xhofe/tache" ) // the param named path of functions in this package is a mount path @@ -69,7 +69,7 @@ func Move(ctx context.Context, srcPath, dstDirPath string, lazyCache ...bool) er return err } -func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (tache.TaskWithInfo, error) { +func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskInfoWithCreator, error) { res, err := _copy(ctx, srcObjPath, dstDirPath, lazyCache...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) @@ -101,8 +101,8 @@ func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer return err } -func PutAsTask(dstDirPath string, file model.FileStreamer) (tache.TaskWithInfo, error) { - t, err := putAsTask(dstDirPath, file) +func PutAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskInfoWithCreator, error) { + t, err := putAsTask(ctx, dstDirPath, file) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) } diff --git a/internal/fs/put.go b/internal/fs/put.go index 807b15e07d6..23197f5ba54 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -7,12 +7,13 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" "github.com/pkg/errors" "github.com/xhofe/tache" ) type UploadTask struct { - tache.Base + task.TaskWithCreator storage driver.Driver dstDirActualPath string file model.FileStreamer @@ -33,7 +34,7 @@ func (t *UploadTask) Run() error { var UploadTaskManager *tache.Manager[*UploadTask] // putAsTask add as a put task and return immediately -func putAsTask(dstDirPath string, file model.FileStreamer) (tache.TaskWithInfo, error) { +func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskInfoWithCreator, error) { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get storage") @@ -49,7 +50,11 @@ func putAsTask(dstDirPath string, file model.FileStreamer) (tache.TaskWithInfo, //file.SetReader(tempFile) //file.SetTmpFile(tempFile) } + taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed t := &UploadTask{ + TaskWithCreator: task.TaskWithCreator{ + Creator: taskCreator, + }, storage: storage, dstDirActualPath: dstDirActualPath, file: file, diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index c7c5c781f71..1c9da1467b5 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,6 +2,8 @@ package tool import ( "context" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/task" "path/filepath" "github.com/alist-org/alist/v3/internal/conf" @@ -9,7 +11,6 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/google/uuid" "github.com/pkg/errors" - "github.com/xhofe/tache" ) type DeletePolicy string @@ -28,7 +29,7 @@ type AddURLArgs struct { DeletePolicy DeletePolicy } -func AddURL(ctx context.Context, args *AddURLArgs) (tache.TaskWithInfo, error) { +func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskInfoWithCreator, error) { // get tool tool, err := Tools.Get(args.Tool) if err != nil { @@ -77,8 +78,12 @@ func AddURL(ctx context.Context, args *AddURLArgs) (tache.TaskWithInfo, error) { // 防止将下载好的文件删除 deletePolicy = DeleteNever } - + + taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed t := &DownloadTask{ + TaskWithCreator: task.TaskWithCreator{ + Creator: taskCreator, + }, Url: args.URL, DstDirPath: args.DstDirPath, TempDir: tempDir, diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index ef9ceabfc8a..038baf9690b 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -7,13 +7,14 @@ import ( "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/task" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" ) type DownloadTask struct { - tache.Base + task.TaskWithCreator Url string `json:"url"` DstDirPath string `json:"dst_dir_path"` TempDir string `json:"temp_dir"` @@ -171,6 +172,9 @@ func (t *DownloadTask) Complete() error { for i := range files { file := files[i] TransferTaskManager.Add(&TransferTask{ + TaskWithCreator: task.TaskWithCreator{ + Creator: t.Creator, + }, file: file, DstDirPath: t.DstDirPath, TempDir: t.TempDir, diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 3744c7b500f..085b4a66afa 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -8,6 +8,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/task" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -15,7 +16,7 @@ import ( ) type TransferTask struct { - tache.Base + task.TaskWithCreator FileDir string `json:"file_dir"` DstDirPath string `json:"dst_dir_path"` TempDir string `json:"temp_dir"` diff --git a/internal/task/base.go b/internal/task/base.go new file mode 100644 index 00000000000..a30e59876b8 --- /dev/null +++ b/internal/task/base.go @@ -0,0 +1,26 @@ +package task + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/xhofe/tache" +) + +type TaskWithCreator struct { + tache.Base + Creator *model.User +} + +func (t *TaskWithCreator) SetCreator(creator *model.User) { + t.Creator = creator + t.Persist() +} + +func (t *TaskWithCreator) GetCreator() *model.User { + return t.Creator +} + +type TaskInfoWithCreator interface { + tache.TaskWithInfo + SetCreator(creator *model.User) + GetCreator() *model.User +} diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 3d446eda957..42d53d7e7c7 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -2,7 +2,7 @@ package handles import ( "fmt" - "github.com/xhofe/tache" + "github.com/alist-org/alist/v3/internal/task" "io" stdpath "path" @@ -121,7 +121,7 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, err, 403) return } - var addedTasks []tache.TaskWithInfo + var addedTasks []task.TaskInfoWithCreator for i, name := range req.Names { t, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) if t != nil { diff --git a/server/handles/fsup.go b/server/handles/fsup.go index ef9baa11dc5..3a366d49fd0 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -1,17 +1,16 @@ package handles import ( - "github.com/xhofe/tache" + "github.com/alist-org/alist/v3/internal/task" "io" "net/url" stdpath "path" "strconv" "time" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" ) @@ -58,9 +57,9 @@ func FsStream(c *gin.Context) { Mimetype: c.GetHeader("Content-Type"), WebPutAsTask: asTask, } - var t tache.TaskWithInfo + var t task.TaskInfoWithCreator if asTask { - t, err = fs.PutAsTask(dir, s) + t, err = fs.PutAsTask(c, dir, s) } else { err = fs.PutDirectly(c, dir, s, true) } @@ -123,12 +122,12 @@ func FsForm(c *gin.Context) { Mimetype: file.Header.Get("Content-Type"), WebPutAsTask: asTask, } - var t tache.TaskWithInfo + var t task.TaskInfoWithCreator if asTask { s.Reader = struct { io.Reader }{f} - t, err = fs.PutAsTask(dir, &s) + t, err = fs.PutAsTask(c, dir, &s) } else { ss, err := stream.NewSeekableStream(s, nil) if err != nil { diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 1c5f95557ff..ff1fcfa05bc 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -5,9 +5,9 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" - "github.com/xhofe/tache" ) type SetAria2Req struct { @@ -133,7 +133,7 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 403) return } - var tasks []tache.TaskWithInfo + var tasks []task.TaskInfoWithCreator for _, url := range req.Urls { t, err := tool.AddURL(c, &tool.AddURLArgs{ URL: url, diff --git a/server/handles/task.go b/server/handles/task.go index a8b4d21b2b9..71b4c622144 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -1,6 +1,8 @@ package handles import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/task" "math" "github.com/alist-org/alist/v3/internal/fs" @@ -12,15 +14,17 @@ import ( ) type TaskInfo struct { - ID string `json:"id"` - Name string `json:"name"` - State tache.State `json:"state"` - Status string `json:"status"` - Progress float64 `json:"progress"` - Error string `json:"error"` + ID string `json:"id"` + Name string `json:"name"` + Creator string `json:"creator"` + CreatorRole int `json:"creator_role"` + State tache.State `json:"state"` + Status string `json:"status"` + Progress float64 `json:"progress"` + Error string `json:"error"` } -func getTaskInfo[T tache.TaskWithInfo](task T) TaskInfo { +func getTaskInfo[T task.TaskInfoWithCreator](task T) TaskInfo { errMsg := "" if task.GetErr() != nil { errMsg = task.GetErr().Error() @@ -30,62 +34,142 @@ func getTaskInfo[T tache.TaskWithInfo](task T) TaskInfo { if math.IsNaN(progress) { progress = 100 } + creatorName := "" + creatorRole := -1 + if task.GetCreator() != nil { + creatorName = task.GetCreator().Username + creatorRole = task.GetCreator().Role + } return TaskInfo{ - ID: task.GetID(), - Name: task.GetName(), - State: task.GetState(), - Status: task.GetStatus(), - Progress: progress, - Error: errMsg, + ID: task.GetID(), + Name: task.GetName(), + Creator: creatorName, + CreatorRole: creatorRole, + State: task.GetState(), + Status: task.GetStatus(), + Progress: progress, + Error: errMsg, } } -func getTaskInfos[T tache.TaskWithInfo](tasks []T) []TaskInfo { +func getTaskInfos[T task.TaskInfoWithCreator](tasks []T) []TaskInfo { return utils.MustSliceConvert(tasks, getTaskInfo[T]) } -func taskRoute[T tache.TaskWithInfo](g *gin.RouterGroup, manager *tache.Manager[T]) { +func argsContains[T comparable](v T, slice ...T) bool { + return utils.SliceContains(slice, v) +} + +func getUserInfo(c *gin.Context) (bool, uint, bool) { + if user, ok := c.Value("user").(*model.User); ok { + return user.IsAdmin(), user.ID, true + } else { + return false, 0, false + } +} + +func getTargetedHandler[T task.TaskInfoWithCreator](manager *tache.Manager[T], callback func(c *gin.Context, task T)) gin.HandlerFunc { + return func(c *gin.Context) { + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) + return + } + t, ok := manager.GetByID(c.Query("tid")) + if !ok { + common.ErrorStrResp(c, "task not found", 404) + return + } + if !isAdmin && uid != t.GetCreator().ID { + // to avoid an attacker using error messages to guess valid TID, return a 404 rather than a 403 + common.ErrorStrResp(c, "task not found", 404) + return + } + callback(c, t) + } +} + +func taskRoute[T task.TaskInfoWithCreator](g *gin.RouterGroup, manager *tache.Manager[T]) { g.GET("/undone", func(c *gin.Context) { - common.SuccessResp(c, getTaskInfos(manager.GetByState(tache.StatePending, tache.StateRunning, - tache.StateCanceling, tache.StateErrored, tache.StateFailing, tache.StateWaitingRetry, tache.StateBeforeRetry))) + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) + return + } + common.SuccessResp(c, getTaskInfos(manager.GetByCondition(func(task T) bool { + // avoid directly passing the user object into the function to reduce closure size + return (isAdmin || uid == task.GetCreator().ID) && + argsContains(task.GetState(), tache.StatePending, tache.StateRunning, tache.StateCanceling, + tache.StateErrored, tache.StateFailing, tache.StateWaitingRetry, tache.StateBeforeRetry) + }))) }) g.GET("/done", func(c *gin.Context) { - common.SuccessResp(c, getTaskInfos(manager.GetByState(tache.StateCanceled, tache.StateFailed, tache.StateSucceeded))) - }) - g.POST("/info", func(c *gin.Context) { - tid := c.Query("tid") - task, ok := manager.GetByID(tid) + isAdmin, uid, ok := getUserInfo(c) if !ok { - common.ErrorStrResp(c, "task not found", 404) + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) return } - common.SuccessResp(c, getTaskInfo(task)) + common.SuccessResp(c, getTaskInfos(manager.GetByCondition(func(task T) bool { + return (isAdmin || uid == task.GetCreator().ID) && + argsContains(task.GetState(), tache.StateCanceled, tache.StateFailed, tache.StateSucceeded) + }))) }) - g.POST("/cancel", func(c *gin.Context) { - tid := c.Query("tid") - manager.Cancel(tid) + g.POST("/info", getTargetedHandler(manager, func(c *gin.Context, task T) { + common.SuccessResp(c, getTaskInfo(task)) + })) + g.POST("/cancel", getTargetedHandler(manager, func(c *gin.Context, task T) { + manager.Cancel(task.GetID()) common.SuccessResp(c) - }) - g.POST("/delete", func(c *gin.Context) { - tid := c.Query("tid") - manager.Remove(tid) + })) + g.POST("/delete", getTargetedHandler(manager, func(c *gin.Context, task T) { + manager.Remove(task.GetID()) common.SuccessResp(c) - }) - g.POST("/retry", func(c *gin.Context) { - tid := c.Query("tid") - manager.Retry(tid) + })) + g.POST("/retry", getTargetedHandler(manager, func(c *gin.Context, task T) { + manager.Retry(task.GetID()) common.SuccessResp(c) - }) + })) g.POST("/clear_done", func(c *gin.Context) { - manager.RemoveByState(tache.StateCanceled, tache.StateFailed, tache.StateSucceeded) + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) + return + } + manager.RemoveByCondition(func(task T) bool { + return (isAdmin || uid == task.GetCreator().ID) && + argsContains(task.GetState(), tache.StateCanceled, tache.StateFailed, tache.StateSucceeded) + }) common.SuccessResp(c) }) g.POST("/clear_succeeded", func(c *gin.Context) { - manager.RemoveByState(tache.StateSucceeded) + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) + return + } + manager.RemoveByCondition(func(task T) bool { + return (isAdmin || uid == task.GetCreator().ID) && task.GetState() == tache.StateSucceeded + }) common.SuccessResp(c) }) g.POST("/retry_failed", func(c *gin.Context) { - manager.RetryAllFailed() + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) + return + } + tasks := manager.GetByCondition(func(task T) bool { + return (isAdmin || uid == task.GetCreator().ID) && task.GetState() == tache.StateFailed + }) + for _, t := range tasks { + manager.Retry(t.GetID()) + } common.SuccessResp(c) }) } diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 14f186be8bf..d65d1ad648a 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -127,6 +127,16 @@ func Authn(c *gin.Context) { c.Next() } +func AuthNotGuest(c *gin.Context) { + user := c.MustGet("user").(*model.User) + if user.IsGuest() { + common.ErrorStrResp(c, "You are a guest", 403) + c.Abort() + } else { + c.Next() + } +} + func AuthAdmin(c *gin.Context) { user := c.MustGet("user").(*model.User) if !user.IsAdmin() { diff --git a/server/router.go b/server/router.go index 07423f923cd..fffa840e537 100644 --- a/server/router.go +++ b/server/router.go @@ -76,6 +76,7 @@ func Init(e *gin.Engine) { public.Any("/offline_download_tools", handles.OfflineDownloadTools) _fs(auth.Group("/fs")) + _task(auth.Group("/task", middlewares.AuthNotGuest)) admin(auth.Group("/admin", middlewares.AuthAdmin)) if flags.Debug || flags.Dev { debug(g.Group("/debug")) @@ -127,8 +128,8 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_qbit", handles.SetQbittorrent) setting.POST("/set_transmission", handles.SetTransmission) - task := g.Group("/task") - handles.SetupTaskRoute(task) + // retain /admin/task API to ensure compatibility with legacy automation scripts + _task(g.Group("/task")) ms := g.Group("/message") ms.POST("/get", message.HttpInstance.GetHandle) @@ -166,6 +167,10 @@ func _fs(g *gin.RouterGroup) { g.POST("/add_offline_download", handles.AddOfflineDownload) } +func _task(g *gin.RouterGroup) { + handles.SetupTaskRoute(g) +} + func Cors(r *gin.Engine) { config := cors.DefaultConfig() // config.AllowAllOrigins = true From b803b0070ecde83f2868b15902c34fe9543fb2e2 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sat, 2 Nov 2024 16:41:33 +0800 Subject: [PATCH 349/659] fix(115): 20GB file upload restriction (#7452) * fix(115): multipart upload error * feat(115): Modify default page size * fix(115): Replace temporary repair scheme --- drivers/115/meta.go | 2 +- drivers/115/types.go | 18 +++++++++++++- drivers/115/util.go | 51 +++++++++++++++------------------------ drivers/115_share/meta.go | 2 +- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/drivers/115/meta.go b/drivers/115/meta.go index 38c1742a741..d9526775229 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -9,7 +9,7 @@ type Addition struct { Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` - PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` + PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` driver.RootID } diff --git a/drivers/115/types.go b/drivers/115/types.go index 830e347b44e..40b951d80ce 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -1,10 +1,11 @@ package _115 import ( + "time" + "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" - "time" ) var _ model.Obj = (*FileObj)(nil) @@ -20,3 +21,18 @@ func (f *FileObj) CreateTime() time.Time { func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } + +type UploadResult struct { + driver.BasicResp + Data struct { + PickCode string `json:"pick_code"` + FileSize int `json:"file_size"` + FileID string `json:"file_id"` + ThumbURL string `json:"thumb_url"` + Sha1 string `json:"sha1"` + Aid int `json:"aid"` + FileName string `json:"file_name"` + Cid string `json:"cid"` + IsVideo int `json:"is_video"` + } `json:"data"` +} diff --git a/drivers/115/util.go b/drivers/115/util.go index 7d5889af9f9..381ef0bd185 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -10,7 +10,6 @@ import ( "io" "net/http" "net/url" - "path/filepath" "strconv" "strings" "sync" @@ -254,6 +253,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i ossClient *oss.Client bucket *oss.Bucket ossToken *driver115.UploadOSSTokenResp + bodyBytes []byte err error ) @@ -268,12 +268,14 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i f(options) } } + // oss 启用Sequential必须按顺序上传 + options.ThreadsNum = 1 if ossToken, err = d.client.GetOSSToken(); err != nil { return err } - if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret); err != nil { + if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { return err } @@ -294,6 +296,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i if imur, err = bucket.InitiateMultipartUpload(params.Object, oss.SetHeader(driver115.OssSecurityTokenHeaderName, ossToken.SecurityToken), oss.UserAgentHeader(driver115.OSSUserAgent), + oss.EnableSha1(), oss.Sequential(), ); err != nil { return err } @@ -337,8 +340,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i continue } - b := bytes.NewBuffer(buf) - if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { + if part, err = bucket.UploadPart(imur, bytes.NewBuffer(buf), chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { break } } @@ -373,14 +375,20 @@ LOOP: } } - // EOF错误是xml的Unmarshal导致的，响应其实是json格式，所以实际上上传是成功的 - if _, err = bucket.CompleteMultipartUpload(imur, parts, driver115.OssOption(params, ossToken)...); err != nil && !errors.Is(err, io.EOF) { - // 当文件名含有 &< 这两个字符之一时响应的xml解析会出现错误，实际上上传是成功的 - if filename := filepath.Base(stream.GetName()); !strings.ContainsAny(filename, "&<") { - return err - } + // 不知道啥原因，oss那边分片上传不计算sha1，导致115服务器校验错误 + // params.Callback.Callback = strings.ReplaceAll(params.Callback.Callback, "${sha1}", params.SHA1) + if _, err := bucket.CompleteMultipartUpload(imur, parts, append( + driver115.OssOption(params, ossToken), + oss.CallbackResult(&bodyBytes), + )...); err != nil { + return err } - return d.checkUploadStatus(dirID, params.SHA1) + + var uploadResult UploadResult + if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { + return err + } + return uploadResult.Err(string(bodyBytes)) } func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { @@ -389,27 +397,6 @@ func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { } } -func (d *Pan115) checkUploadStatus(dirID, sha1 string) error { - // 验证上传是否成功 - req := d.client.NewRequest().ForceContentType("application/json;charset=UTF-8") - opts := []driver115.GetFileOptions{ - driver115.WithOrder(driver115.FileOrderByTime), - driver115.WithShowDirEnable(false), - driver115.WithAsc(false), - driver115.WithLimit(500), - } - fResp, err := driver115.GetFiles(req, dirID, opts...) - if err != nil { - return err - } - for _, fileInfo := range fResp.Files { - if fileInfo.Sha1 == sha1 { - return nil - } - } - return driver115.ErrUploadFailed -} - func SplitFile(fileSize int64) (chunks []oss.FileChunk, err error) { for i := int64(1); i < 10; i++ { if fileSize < i*utils.GB { // 文件大小小于iGB时分为i*1000片 diff --git a/drivers/115_share/meta.go b/drivers/115_share/meta.go index 1d203b24c2b..3fcc7b92133 100644 --- a/drivers/115_share/meta.go +++ b/drivers/115_share/meta.go @@ -9,7 +9,7 @@ type Addition struct { Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` - PageSize int64 `json:"page_size" type:"number" default:"20" help:"list api per page size of 115 driver"` + PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"` ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"` From e707fa38f1fce5c92922686ac421b37eb2173c23 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sat, 2 Nov 2024 17:05:00 +0800 Subject: [PATCH 350/659] ci: remove specific tag for freebsd action --- .github/workflows/release_freebsd.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_freebsd.yml b/.github/workflows/release_freebsd.yml index 46afb326bf3..70dcecb10f9 100644 --- a/.github/workflows/release_freebsd.yml +++ b/.github/workflows/release_freebsd.yml @@ -31,5 +31,4 @@ jobs: - name: Upload assets uses: softprops/action-gh-release@v2 with: - tag_name: dev files: build/compress/* From 2671c876f1fe7e6de1a3939c9ee5f588d9b5f41b Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sat, 2 Nov 2024 21:08:19 +0800 Subject: [PATCH 351/659] revert: "fix(115): enforce 20GB file size limit on uploadev" This reverts commit 216e3909f3946eb9c1b786c0d82c00f278f0ea25. --- drivers/115/driver.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 4857c1ec05e..f6fb6b05618 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -2,7 +2,6 @@ package _115 import ( "context" - "fmt" "strings" "sync" @@ -122,10 +121,7 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr if err := d.WaitLimit(ctx); err != nil { return err } - if stream.GetSize() > utils.GB*20 { // TODO 由于官方分片上传接口失效，所以使用普通上传小于20GB的文件 - return fmt.Errorf("unsupported file size: 20GB limit exceeded") - } - // 分片上传 + var ( fastInfo *driver115.UploadInitResp dirID = dstDir.GetID() @@ -181,13 +177,11 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } // 闪传失败，上传 - // if stream.GetSize() <= utils.KB{ // 文件大小小于1KB，改用普通模式上传 - if stream.GetSize() <= utils.GB*20 { // TODO 由于官方分片上传接口失效，所以使用普通上传小于20GB的文件 + if stream.GetSize() <= utils.KB { // 文件大小小于1KB，改用普通模式上传 return d.client.UploadByOSS(&fastInfo.UploadOSSParams, stream, dirID) } - return driver115.ErrUnexpected // 分片上传 - // return d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID) + return d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID) } func (d *Pan115) OfflineList(ctx context.Context) ([]*driver115.OfflineTask, error) { From f58de9923a75ed21009debe464fec867c854bbfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E7=A8=B3?= Date: Fri, 8 Nov 2024 22:07:35 +0800 Subject: [PATCH 352/659] refactor(aliyunopen,config): Modify default properties (#7476) --- drivers/aliyundrive_open/meta.go | 2 +- internal/conf/config.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index de9b45e01d6..5801314396e 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -6,7 +6,7 @@ import ( ) type Addition struct { - DriveType string `json:"drive_type" type:"select" options:"default,resource,backup" default:"default"` + DriveType string `json:"drive_type" type:"select" options:"default,resource,backup" default:"resource"` driver.RootID RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` diff --git a/internal/conf/config.go b/internal/conf/config.go index c5dc9c521bf..aa29e1f506d 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -131,22 +131,22 @@ func DefaultConfig() *Config { TlsInsecureSkipVerify: true, Tasks: TasksConfig{ Download: TaskConfig{ - Workers: 5, - MaxRetry: 1, - TaskPersistant: true, + Workers: 5, + MaxRetry: 1, + // TaskPersistant: true, }, Transfer: TaskConfig{ - Workers: 5, - MaxRetry: 2, - TaskPersistant: true, + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, }, Upload: TaskConfig{ Workers: 5, }, Copy: TaskConfig{ - Workers: 5, - MaxRetry: 2, - TaskPersistant: true, + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, }, }, Cors: Cors{ From 67c93eed2b1e28e0425b93e5f3f0533de653cc70 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Fri, 8 Nov 2024 22:08:25 +0800 Subject: [PATCH 353/659] feat(baidu_netdisk,baidu_photo): add and fix hashinfo (#7469) --- drivers/baidu_netdisk/types.go | 3 +- drivers/baidu_netdisk/util.go | 57 ++++++++++++++++++++++++---------- drivers/baidu_photo/types.go | 2 +- drivers/baidu_photo/utils.go | 41 ++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 19 deletions(-) diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index cbec0bcfcd6..6f3bf13b3e4 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -6,6 +6,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" ) type TokenErrResp struct { @@ -72,7 +73,7 @@ func fileToObj(f File) *model.ObjThumb { IsFolder: f.Isdir == 1, // 直接获取的MD5是错误的 - // HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), + HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index ac1f06e807e..ca1a6805a04 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -1,11 +1,14 @@ package baidu_netdisk import ( + "encoding/hex" "errors" "fmt" "net/http" "strconv" + "strings" "time" + "unicode" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/errs" @@ -153,8 +156,6 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model u = res.Header().Get("location") //} - updateObjMd5(file, "pan.baidu.com", u) - return &model.Link{ URL: u, Header: http.Header{ @@ -178,8 +179,6 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li return nil, err } - updateObjMd5(file, d.CustomCrackUA, resp.Info[0].Dlink) - return &model.Link{ URL: resp.Info[0].Dlink, Header: http.Header{ @@ -229,19 +228,6 @@ func joinTime(form map[string]string, ctime, mtime int64) { form["local_ctime"] = strconv.FormatInt(ctime, 10) } -func updateObjMd5(obj model.Obj, userAgent, u string) { - object := model.GetRawObject(obj) - if object != nil { - req, _ := http.NewRequest(http.MethodHead, u, nil) - req.Header.Add("User-Agent", userAgent) - resp, _ := base.HttpClient.Do(req) - if resp != nil { - contentMd5 := resp.Header.Get("Content-Md5") - object.HashInfo = utils.NewHashInfo(utils.MD5, contentMd5) - } - } -} - const ( DefaultSliceSize int64 = 4 * utils.MB VipSliceSize = 16 * utils.MB @@ -267,3 +253,40 @@ func (d *BaiduNetdisk) getSliceSize() int64 { // r = strings.ReplaceAll(r, "+", "%20") // return r // } + +func DecryptMd5(encryptMd5 string) string { + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} + +func EncryptMd5(originalMd5 string) string { + reversed := originalMd5[8:16] + originalMd5[:8] + originalMd5[24:32] + originalMd5[16:24] + + var out strings.Builder + out.Grow(len(reversed)) + for i, n := 0, int64(0); i < len(reversed); i++ { + n, _ = strconv.ParseInt(reversed[i:i+1], 16, 64) + n ^= int64(15 & i) + if i == 9 { + out.WriteRune(rune(n) + 'g') + } else { + out.WriteString(strconv.FormatInt(n, 16)) + } + } + return out.String() +} diff --git a/drivers/baidu_photo/types.go b/drivers/baidu_photo/types.go index 2bbacd303f7..0e5cbb2cdd5 100644 --- a/drivers/baidu_photo/types.go +++ b/drivers/baidu_photo/types.go @@ -72,7 +72,7 @@ func (c *File) Thumb() string { } func (c *File) GetHash() utils.HashInfo { - return utils.NewHashInfo(utils.MD5, c.Md5) + return utils.NewHashInfo(utils.MD5, DecryptMd5(c.Md5)) } /*相册部分*/ diff --git a/drivers/baidu_photo/utils.go b/drivers/baidu_photo/utils.go index be0ed1336e6..c8c5b7ee88b 100644 --- a/drivers/baidu_photo/utils.go +++ b/drivers/baidu_photo/utils.go @@ -2,8 +2,12 @@ package baiduphoto import ( "context" + "encoding/hex" "fmt" "net/http" + "strconv" + "strings" + "unicode" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/errs" @@ -476,3 +480,40 @@ func (d *BaiduPhoto) uInfo() (*UInfo, error) { } return &info, nil } + +func DecryptMd5(encryptMd5 string) string { + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} + +func EncryptMd5(originalMd5 string) string { + reversed := originalMd5[8:16] + originalMd5[:8] + originalMd5[24:32] + originalMd5[16:24] + + var out strings.Builder + out.Grow(len(reversed)) + for i, n := 0, int64(0); i < len(reversed); i++ { + n, _ = strconv.ParseInt(reversed[i:i+1], 16, 64) + n ^= int64(15 & i) + if i == 9 { + out.WriteRune(rune(n) + 'g') + } else { + out.WriteString(strconv.FormatInt(n, 16)) + } + } + return out.String() +} From 0a46979c519885465e586da009b70422382c84ac Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Fri, 8 Nov 2024 22:08:50 +0800 Subject: [PATCH 354/659] feat(115): enhance cache (#7479) --- drivers/115/driver.go | 100 +++++++++++++++++++++++++++++++----------- drivers/115/meta.go | 2 +- drivers/115/util.go | 84 +++++++++++++++++++++++++++++------ 3 files changed, 146 insertions(+), 40 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index f6fb6b05618..4f584cd7b51 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -79,28 +79,60 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return link, nil } -func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { +func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err } - if _, err := d.client.Mkdir(parentDir.GetID(), dirName); err != nil { - return err + + result := driver115.MkdirResp{} + form := map[string]string{ + "pid": parentDir.GetID(), + "cname": dirName, } - return nil + req := d.client.NewRequest(). + SetFormData(form). + SetResult(&result). + ForceContentType("application/json;charset=UTF-8") + + resp, err := req.Post(driver115.ApiDirAdd) + + err = driver115.CheckErr(err, &result, resp) + if err != nil { + return nil, err + } + f, err := d.getNewFile(result.FileID) + if err != nil { + return nil, nil + } + return f, nil } -func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err } - return d.client.Move(dstDir.GetID(), srcObj.GetID()) + if err := d.client.Move(dstDir.GetID(), srcObj.GetID()); err != nil { + return nil, err + } + f, err := d.getNewFile(srcObj.GetID()) + if err != nil { + return nil, nil + } + return f, nil } -func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) error { +func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err + } + if err := d.client.Rename(srcObj.GetID(), newName); err != nil { + return nil, err } - return d.client.Rename(srcObj.GetID(), newName) + f, err := d.getNewFile((srcObj.GetID())) + if err != nil { + return nil, nil + } + return f, nil } func (d *Pan115) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -117,9 +149,9 @@ func (d *Pan115) Remove(ctx context.Context, obj model.Obj) error { return d.client.Delete(obj.GetID()) } -func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err } var ( @@ -128,10 +160,10 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr ) if ok, err := d.client.UploadAvailable(); err != nil || !ok { - return err + return nil, err } if stream.GetSize() > d.client.UploadMetaInfo.SizeLimit { - return driver115.ErrUploadTooLarge + return nil, driver115.ErrUploadTooLarge } //if digest, err = d.client.GetDigestResult(stream); err != nil { // return err @@ -144,22 +176,22 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: hashSize}) if err != nil { - return err + return nil, err } preHash, err := utils.HashReader(utils.SHA1, reader) if err != nil { - return err + return nil, err } preHash = strings.ToUpper(preHash) fullHash := stream.GetHash().GetHash(utils.SHA1) if len(fullHash) <= 0 { tmpF, err := stream.CacheFullInTempFile() if err != nil { - return err + return nil, err } fullHash, err = utils.HashFile(utils.SHA1, tmpF) if err != nil { - return err + return nil, err } } fullHash = strings.ToUpper(fullHash) @@ -168,20 +200,36 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr // note that 115 add timeout for rapid-upload, // and "sig invalid" err is thrown even when the hash is correct after timeout. if fastInfo, err = d.rapidUpload(stream.GetSize(), stream.GetName(), dirID, preHash, fullHash, stream); err != nil { - return err + return nil, err } if matched, err := fastInfo.Ok(); err != nil { - return err + return nil, err } else if matched { - return nil + f, err := d.getNewFileByPickCode(fastInfo.PickCode) + if err != nil { + return nil, nil + } + return f, nil } + var uploadResult *UploadResult // 闪传失败，上传 - if stream.GetSize() <= utils.KB { // 文件大小小于1KB，改用普通模式上传 - return d.client.UploadByOSS(&fastInfo.UploadOSSParams, stream, dirID) + if stream.GetSize() <= 10*utils.MB { // 文件大小小于10MB，改用普通模式上传 + if uploadResult, err = d.UploadByOSS(&fastInfo.UploadOSSParams, stream, dirID); err != nil { + return nil, err + } + } else { + // 分片上传 + if uploadResult, err = d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID); err != nil { + return nil, err + } + } + + file, err := d.getNewFile(uploadResult.Data.FileID) + if err != nil { + return nil, nil } - // 分片上传 - return d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID) + return file, nil } func (d *Pan115) OfflineList(ctx context.Context) ([]*driver115.OfflineTask, error) { diff --git a/drivers/115/meta.go b/drivers/115/meta.go index d9526775229..3b192291a43 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -10,7 +10,7 @@ type Addition struct { QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` - LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` + LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate ([limit]r/1s)"` driver.RootID } diff --git a/drivers/115/util.go b/drivers/115/util.go index 381ef0bd185..33e345706d2 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -74,6 +74,34 @@ func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { return res, nil } +func (d *Pan115) getNewFile(fileId string) (*FileObj, error) { + file, err := d.client.GetFile(fileId) + if err != nil { + return nil, err + } + return &FileObj{*file}, nil +} + +func (d *Pan115) getNewFileByPickCode(pickCode string) (*FileObj, error) { + result := driver115.GetFileInfoResponse{} + req := d.client.NewRequest(). + SetQueryParam("pick_code", pickCode). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + resp, err := req.Get(driver115.ApiFileInfo) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + if len(result.Files) == 0 { + return nil, errors.New("not get file info") + } + fileInfo := result.Files[0] + + f := &FileObj{} + f.From(fileInfo) + return f, nil +} + func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } @@ -244,8 +272,38 @@ func UploadDigestRange(stream model.FileStreamer, rangeSpec string) (result stri return } +// UploadByOSS use aliyun sdk to upload +func (c *Pan115) UploadByOSS(params *driver115.UploadOSSParams, r io.Reader, dirID string) (*UploadResult, error) { + ossToken, err := c.client.GetOSSToken() + if err != nil { + return nil, err + } + ossClient, err := oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret) + if err != nil { + return nil, err + } + bucket, err := ossClient.Bucket(params.Bucket) + if err != nil { + return nil, err + } + + var bodyBytes []byte + if err = bucket.PutObject(params.Object, r, append( + driver115.OssOption(params, ossToken), + oss.CallbackResult(&bodyBytes), + )...); err != nil { + return nil, err + } + + var uploadResult UploadResult + if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { + return nil, err + } + return &uploadResult, uploadResult.Err(string(bodyBytes)) +} + // UploadByMultipart upload by mutipart blocks -func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize int64, stream model.FileStreamer, dirID string, opts ...driver115.UploadMultipartOption) error { +func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize int64, stream model.FileStreamer, dirID string, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart @@ -259,7 +317,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i tmpF, err := stream.CacheFullInTempFile() if err != nil { - return err + return nil, err } options := driver115.DefalutUploadMultipartOptions() @@ -272,15 +330,15 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i options.ThreadsNum = 1 if ossToken, err = d.client.GetOSSToken(); err != nil { - return err + return nil, err } if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { - return err + return nil, err } if bucket, err = ossClient.Bucket(params.Bucket); err != nil { - return err + return nil, err } // ossToken一小时后就会失效，所以每50分钟重新获取一次 @@ -290,7 +348,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i timeout := time.NewTimer(options.Timeout) if chunks, err = SplitFile(fileSize); err != nil { - return err + return nil, err } if imur, err = bucket.InitiateMultipartUpload(params.Object, @@ -298,7 +356,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i oss.UserAgentHeader(driver115.OSSUserAgent), oss.EnableSha1(), oss.Sequential(), ); err != nil { - return err + return nil, err } wg := sync.WaitGroup{} @@ -364,14 +422,14 @@ LOOP: case <-ticker.C: // 到时重新获取ossToken if ossToken, err = d.client.GetOSSToken(); err != nil { - return err + return nil, err } case <-quit: break LOOP case <-errCh: - return err + return nil, err case <-timeout.C: - return fmt.Errorf("time out") + return nil, fmt.Errorf("time out") } } @@ -381,14 +439,14 @@ LOOP: driver115.OssOption(params, ossToken), oss.CallbackResult(&bodyBytes), )...); err != nil { - return err + return nil, err } var uploadResult UploadResult if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { - return err + return nil, err } - return uploadResult.Err(string(bodyBytes)) + return &uploadResult, uploadResult.Err(string(bodyBytes)) } func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { From 6c38c5972da06aa2f5b0badaafd3f5fe6683fca9 Mon Sep 17 00:00:00 2001 From: Jason-Fly <869914918@qq.com> Date: Sat, 16 Nov 2024 13:18:49 +0800 Subject: [PATCH 355/659] fix(terabox): big file upload issue (#7498 close #7490) --- drivers/terabox/driver.go | 120 ++++++++++++++++++++------------------ drivers/terabox/types.go | 4 ++ drivers/terabox/util.go | 21 +++++++ 3 files changed, 87 insertions(+), 58 deletions(-) diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index 11db351b75c..362de69e0a0 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -10,8 +10,6 @@ import ( "math" stdpath "path" "strconv" - "strings" - "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" @@ -24,9 +22,9 @@ import ( type Terabox struct { model.Storage Addition - JsToken string + JsToken string url_domain_prefix string - base_url string + base_url string } func (d *Terabox) Config() driver.Config { @@ -145,52 +143,24 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt } log.Debugln(locateupload_resp) - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - var Default int64 = 4 * 1024 * 1024 - defaultByteData := make([]byte, Default) - count := int(math.Ceil(float64(stream.GetSize()) / float64(Default))) - // cal md5 - h1 := md5.New() - h2 := md5.New() - block_list := make([]string, 0) - left := stream.GetSize() - for i := 0; i < count; i++ { - byteSize := Default - var byteData []byte - if left < Default { - byteSize = left - byteData = make([]byte, byteSize) - } else { - byteData = defaultByteData - } - left -= byteSize - _, err = io.ReadFull(tempFile, byteData) - if err != nil { - return err - } - h1.Write(byteData) - h2.Write(byteData) - block_list = append(block_list, fmt.Sprintf("\"%s\"", hex.EncodeToString(h2.Sum(nil)))) - h2.Reset() - } + // precreate file + rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) + path := encodeURIComponent(rawPath) - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err + var precreateBlockListStr string + if stream.GetSize() > initialChunkSize { + precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761","a5fc157d78e6ad1c7e114b056c92821e"]` + } else { + precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761"]` } - rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) - path := encodeURIComponent(rawPath) - block_list_str := fmt.Sprintf("[%s]", strings.Join(block_list, ",")) data := map[string]string{ - "path": rawPath, - "autoinit": "1", - "target_path": dstDir.GetPath(), - "block_list": block_list_str, - "local_mtime": strconv.FormatInt(time.Now().Unix(), 10), + "path": rawPath, + "autoinit": "1", + "target_path": dstDir.GetPath(), + "block_list": precreateBlockListStr, + "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), + "file_limit_switch_v34": "true", } var precreateResp PrecreateResp log.Debugln(data) @@ -206,6 +176,13 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt if precreateResp.ReturnType == 2 { return nil } + + // upload chunks + tempFile, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + params := map[string]string{ "method": "upload", "path": path, @@ -215,24 +192,37 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt "channel": "dubox", "clienttype": "0", } - left = stream.GetSize() - for i, partseq := range precreateResp.BlockList { + + streamSize := stream.GetSize() + chunkSize := calculateChunkSize(streamSize) + chunkByteData := make([]byte, chunkSize) + count := int(math.Ceil(float64(streamSize) / float64(chunkSize))) + left := streamSize + uploadBlockList := make([]string, 0, count) + h := md5.New() + for partseq := 0; partseq < count; partseq++ { if utils.IsCanceled(ctx) { return ctx.Err() } - byteSize := Default + byteSize := chunkSize var byteData []byte - if left < Default { + if left >= chunkSize { + byteData = chunkByteData + } else { byteSize = left byteData = make([]byte, byteSize) - } else { - byteData = defaultByteData } left -= byteSize _, err = io.ReadFull(tempFile, byteData) if err != nil { return err } + + // calculate md5 + h.Write(byteData) + uploadBlockList = append(uploadBlockList, hex.EncodeToString(h.Sum(nil))) + h.Reset() + u := "https://" + locateupload_resp.Host + "/rest/2.0/pcs/superfile2" params["partseq"] = strconv.Itoa(partseq) res, err := base.RestyClient.R(). @@ -245,25 +235,39 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt return err } log.Debugln(res.String()) - if len(precreateResp.BlockList) > 0 { - up(float64(i) * 100 / float64(len(precreateResp.BlockList))) + if count > 0 { + up(float64(partseq) * 100 / float64(count)) } } + + // create file params = map[string]string{ "isdir": "0", "rtype": "1", } + + uploadBlockListStr, err := utils.Json.MarshalToString(uploadBlockList) + if err != nil { + return err + } data = map[string]string{ "path": rawPath, "size": strconv.FormatInt(stream.GetSize(), 10), "uploadid": precreateResp.Uploadid, "target_path": dstDir.GetPath(), - "block_list": block_list_str, - "local_mtime": strconv.FormatInt(time.Now().Unix(), 10), + "block_list": uploadBlockListStr, + "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), } - res, err = d.post_form("/api/create", params, data, nil) + var createResp CreateResp + res, err = d.post_form("/api/create", params, data, &createResp) log.Debugln(string(res)) - return err + if err != nil { + return err + } + if createResp.Errno != 0 { + return fmt.Errorf("[terabox] failed to create file, errno: %d", createResp.Errno) + } + return nil } var _ driver.Driver = (*Terabox)(nil) diff --git a/drivers/terabox/types.go b/drivers/terabox/types.go index 8bdbc6fce1b..f4d50ddef37 100644 --- a/drivers/terabox/types.go +++ b/drivers/terabox/types.go @@ -99,3 +99,7 @@ type CheckLoginResp struct { type LocateUploadResp struct { Host string `json:"host"` } + +type CreateResp struct { + Errno int `json:"errno"` +} diff --git a/drivers/terabox/util.go b/drivers/terabox/util.go index e0f3d74e8f5..002f80b5446 100644 --- a/drivers/terabox/util.go +++ b/drivers/terabox/util.go @@ -17,6 +17,11 @@ import ( log "github.com/sirupsen/logrus" ) +const ( + initialChunkSize int64 = 4 << 20 // 4MB + initialSizeThreshold int64 = 4 << 30 // 4GB +) + func getStrBetween(raw, start, end string) string { regexPattern := fmt.Sprintf(`%s(.*?)%s`, regexp.QuoteMeta(start), regexp.QuoteMeta(end)) regex := regexp.MustCompile(regexPattern) @@ -258,3 +263,19 @@ func encodeURIComponent(str string) string { r = strings.ReplaceAll(r, "+", "%20") return r } + +func calculateChunkSize(streamSize int64) int64 { + chunkSize := initialChunkSize + sizeThreshold := initialSizeThreshold + + if streamSize < chunkSize { + return streamSize + } + + for streamSize > sizeThreshold { + chunkSize <<= 1 + sizeThreshold <<= 1 + } + + return chunkSize +} From c3c5843dce04450a4d03bca22758268895454798 Mon Sep 17 00:00:00 2001 From: Jason-Fly <869914918@qq.com> Date: Sat, 16 Nov 2024 13:19:59 +0800 Subject: [PATCH 356/659] fix(terabox): panic due to slice out of range (#7499 close #7487) --- drivers/terabox/util.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/terabox/util.go b/drivers/terabox/util.go index 002f80b5446..058eecd6085 100644 --- a/drivers/terabox/util.go +++ b/drivers/terabox/util.go @@ -91,11 +91,15 @@ func (d *Terabox) request(rurl string, method string, callback base.ReqCallback, return d.request(rurl, method, callback, resp, true) } } else if errno == -6 { - log.Debugln(res.Header()) - d.url_domain_prefix = res.Header()["Url-Domain-Prefix"][0] - d.base_url = "https://" + d.url_domain_prefix + ".terabox.com" - log.Debugln("Redirect base_url to", d.base_url) - return d.request(rurl, method, callback, resp, noRetry...) + header := res.Header() + log.Debugln(header) + urlDomainPrefix := header.Get("Url-Domain-Prefix") + if len(urlDomainPrefix) > 0 { + d.url_domain_prefix = urlDomainPrefix + d.base_url = "https://" + d.url_domain_prefix + ".terabox.com" + log.Debugln("Redirect base_url to", d.base_url) + return d.request(rurl, method, callback, resp, noRetry...) + } } return res.Body(), nil } From 1c01dc683931f9a71b5f2bd97ce78cc1eecda9ff Mon Sep 17 00:00:00 2001 From: Jason-Fly <869914918@qq.com> Date: Sat, 16 Nov 2024 13:20:49 +0800 Subject: [PATCH 357/659] fix(storage): delete storage fails if a panic occurred during initialization (#7501) * fix(storage): store storages map when init storage panic * fix(drivers): add nil check to drop method --- drivers/chaoxing/driver.go | 4 +++- drivers/vtencent/drive.go | 4 +++- internal/op/storage.go | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index de122c36c4d..360c6e3d01d 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -67,7 +67,9 @@ func (d *ChaoXing) Init(ctx context.Context) error { } func (d *ChaoXing) Drop(ctx context.Context) error { - d.cron.Stop() + if d.cron != nil { + d.cron.Stop() + } return nil } diff --git a/drivers/vtencent/drive.go b/drivers/vtencent/drive.go index 676431439a9..36a9167234e 100644 --- a/drivers/vtencent/drive.go +++ b/drivers/vtencent/drive.go @@ -55,7 +55,9 @@ func (d *Vtencent) Init(ctx context.Context) error { } func (d *Vtencent) Drop(ctx context.Context) error { - d.cron.Stop() + if d.cron != nil { + d.cron.Stop() + } return nil } diff --git a/internal/op/storage.go b/internal/op/storage.go index 6790a8dffa6..7d8831f548e 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -101,7 +101,7 @@ func initStorage(ctx context.Context, storage model.Storage, storageDriver drive log.Errorf("panic init storage: %s", errInfo) driverStorage.SetStatus(errInfo) MustSaveDriverStorage(storageDriver) - storagesMap.Delete(driverStorage.MountPath) + storagesMap.Store(driverStorage.MountPath, storageDriver) } }() // Unmarshal Addition From a4ad98ee3e4647d2222d41cd22588fadc9a7535b Mon Sep 17 00:00:00 2001 From: BlueSkyXN <63384277+BlueSkyXN@users.noreply.github.com> Date: Sun, 17 Nov 2024 20:03:04 +0800 Subject: [PATCH 358/659] fix(pikpak): domain block and change to NET (#7350) --- drivers/pikpak/driver.go | 30 ++++++------- drivers/pikpak/util.go | 82 +++++++++++++++++----------------- drivers/pikpak_share/driver.go | 4 +- drivers/pikpak_share/util.go | 80 ++++++++++++++++----------------- 4 files changed, 98 insertions(+), 98 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 4208bb8765a..24de24d4f59 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -91,8 +91,8 @@ func (d *PikPak) Init(ctx context.Context) (err error) { ClientID: d.ClientID, ClientSecret: d.ClientSecret, Endpoint: oauth2.Endpoint{ - AuthURL: "https://user.mypikpak.com/v1/auth/signin", - TokenURL: "https://user.mypikpak.com/v1/auth/token", + AuthURL: "https://user.mypikpak.net/v1/auth/signin", + TokenURL: "https://user.mypikpak.net/v1/auth/token", AuthStyle: oauth2.AuthStyleInParams, }, } @@ -124,7 +124,7 @@ func (d *PikPak) Init(ctx context.Context) (err error) { } // 获取CaptchaToken - err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/files"), d.Common.GetUserID()) + err = d.RefreshCaptchaTokenAtLogin(GetAction(http.MethodGet, "https://api-drive.mypikpak.net/drive/v1/files"), d.Common.GetUserID()) if err != nil { return err } @@ -174,7 +174,7 @@ func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if !d.DisableMediaLink { queryParams["usage"] = "CACHE" } - _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.com/drive/v1/files/%s", file.GetID()), + _, err := d.request(fmt.Sprintf("https://api-drive.mypikpak.net/drive/v1/files/%s", file.GetID()), http.MethodGet, func(req *resty.Request) { req.SetQueryParams(queryParams) }, &resp) @@ -200,7 +200,7 @@ func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *PikPak) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "kind": "drive#folder", "parent_id": parentDir.GetID(), @@ -211,7 +211,7 @@ func (d *PikPak) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *PikPak) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchMove", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchMove", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{srcObj.GetID()}, "to": base.Json{ @@ -223,7 +223,7 @@ func (d *PikPak) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *PikPak) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files/"+srcObj.GetID(), http.MethodPatch, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files/"+srcObj.GetID(), http.MethodPatch, func(req *resty.Request) { req.SetBody(base.Json{ "name": newName, }) @@ -232,7 +232,7 @@ func (d *PikPak) Rename(ctx context.Context, srcObj model.Obj, newName string) e } func (d *PikPak) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchCopy", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchCopy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{srcObj.GetID()}, "to": base.Json{ @@ -244,7 +244,7 @@ func (d *PikPak) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *PikPak) Remove(ctx context.Context, obj model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchTrash", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchTrash", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{obj.GetID()}, }) @@ -268,7 +268,7 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } var resp UploadTaskData - res, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + res, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "kind": "drive#file", "name": stream.GetName(), @@ -292,9 +292,9 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr params := resp.Resumable.Params //endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") - // web 端上传 返回的endpoint 为 `mypikpak.com` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.com`· + // web 端上传 返回的endpoint 为 `mypikpak.net` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.net`· if d.Addition.Platform == "android" { - params.Endpoint = "mypikpak.com" + params.Endpoint = "mypikpak.net" } if stream.GetSize() <= 10*utils.MB { // 文件大小 小于10MB，改用普通模式上传 @@ -318,7 +318,7 @@ func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir } var resp OfflineDownloadResp - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(requestBody) }, &resp) @@ -336,7 +336,7 @@ PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING */ func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase []string) ([]OfflineTask, error) { res := make([]OfflineTask, 0) - url := "https://api-drive.mypikpak.com/drive/v1/tasks" + url := "https://api-drive.mypikpak.net/drive/v1/tasks" if len(phase) == 0 { phase = []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_COMPLETE", "PHASE_TYPE_PENDING"} @@ -377,7 +377,7 @@ func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase [] } func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { - url := "https://api-drive.mypikpak.com/drive/v1/tasks" + url := "https://api-drive.mypikpak.net/drive/v1/tasks" params := map[string]string{ "task_ids": strings.Join(taskIDs, ","), "delete_files": strconv.FormatBool(deleteFiles), diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 1fd26020a60..6c5c88ad4b2 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -86,51 +86,51 @@ const ( WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" - WebPackageName = "mypikpak.com" + WebPackageName = "mypikpak.net" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" PCClientVersion = "undefined" // 2.5.6.4831 - PCPackageName = "mypikpak.com" + PCPackageName = "mypikpak.net" PCSdkVersion = "8.0.3" ) var DlAddr = []string{ - "dl-a10b-0621.mypikpak.com", - "dl-a10b-0622.mypikpak.com", - "dl-a10b-0623.mypikpak.com", - "dl-a10b-0624.mypikpak.com", - "dl-a10b-0625.mypikpak.com", - "dl-a10b-0858.mypikpak.com", - "dl-a10b-0859.mypikpak.com", - "dl-a10b-0860.mypikpak.com", - "dl-a10b-0861.mypikpak.com", - "dl-a10b-0862.mypikpak.com", - "dl-a10b-0863.mypikpak.com", - "dl-a10b-0864.mypikpak.com", - "dl-a10b-0865.mypikpak.com", - "dl-a10b-0866.mypikpak.com", - "dl-a10b-0867.mypikpak.com", - "dl-a10b-0868.mypikpak.com", - "dl-a10b-0869.mypikpak.com", - "dl-a10b-0870.mypikpak.com", - "dl-a10b-0871.mypikpak.com", - "dl-a10b-0872.mypikpak.com", - "dl-a10b-0873.mypikpak.com", - "dl-a10b-0874.mypikpak.com", - "dl-a10b-0875.mypikpak.com", - "dl-a10b-0876.mypikpak.com", - "dl-a10b-0877.mypikpak.com", - "dl-a10b-0878.mypikpak.com", - "dl-a10b-0879.mypikpak.com", - "dl-a10b-0880.mypikpak.com", - "dl-a10b-0881.mypikpak.com", - "dl-a10b-0882.mypikpak.com", - "dl-a10b-0883.mypikpak.com", - "dl-a10b-0884.mypikpak.com", - "dl-a10b-0885.mypikpak.com", - "dl-a10b-0886.mypikpak.com", - "dl-a10b-0887.mypikpak.com", + "dl-a10b-0621.mypikpak.net", + "dl-a10b-0622.mypikpak.net", + "dl-a10b-0623.mypikpak.net", + "dl-a10b-0624.mypikpak.net", + "dl-a10b-0625.mypikpak.net", + "dl-a10b-0858.mypikpak.net", + "dl-a10b-0859.mypikpak.net", + "dl-a10b-0860.mypikpak.net", + "dl-a10b-0861.mypikpak.net", + "dl-a10b-0862.mypikpak.net", + "dl-a10b-0863.mypikpak.net", + "dl-a10b-0864.mypikpak.net", + "dl-a10b-0865.mypikpak.net", + "dl-a10b-0866.mypikpak.net", + "dl-a10b-0867.mypikpak.net", + "dl-a10b-0868.mypikpak.net", + "dl-a10b-0869.mypikpak.net", + "dl-a10b-0870.mypikpak.net", + "dl-a10b-0871.mypikpak.net", + "dl-a10b-0872.mypikpak.net", + "dl-a10b-0873.mypikpak.net", + "dl-a10b-0874.mypikpak.net", + "dl-a10b-0875.mypikpak.net", + "dl-a10b-0876.mypikpak.net", + "dl-a10b-0877.mypikpak.net", + "dl-a10b-0878.mypikpak.net", + "dl-a10b-0879.mypikpak.net", + "dl-a10b-0880.mypikpak.net", + "dl-a10b-0881.mypikpak.net", + "dl-a10b-0882.mypikpak.net", + "dl-a10b-0883.mypikpak.net", + "dl-a10b-0884.mypikpak.net", + "dl-a10b-0885.mypikpak.net", + "dl-a10b-0886.mypikpak.net", + "dl-a10b-0887.mypikpak.net", } func (d *PikPak) login() error { @@ -139,7 +139,7 @@ func (d *PikPak) login() error { return errors.New("username or password is empty") } - url := "https://user.mypikpak.com/v1/auth/signin" + url := "https://user.mypikpak.net/v1/auth/signin" // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) if d.GetCaptchaToken() == "" { if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { @@ -169,7 +169,7 @@ func (d *PikPak) login() error { } func (d *PikPak) refreshToken(refreshToken string) error { - url := "https://user.mypikpak.com/v1/auth/token" + url := "https://user.mypikpak.net/v1/auth/token" var e ErrResp res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e). SetHeader("user-agent", "").SetBody(base.Json{ @@ -307,7 +307,7 @@ func (d *PikPak) getFiles(id string) ([]File, error) { "page_token": pageToken, } var resp Files - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -473,7 +473,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err } var e ErrResp var resp CaptchaTokenResponse - _, err := d.request("https://user.mypikpak.com/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://user.mypikpak.net/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param).SetQueryParam("client_id", d.ClientID) }, &resp) diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index 91cb45ca1cf..f107ac17ba3 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -80,7 +80,7 @@ func (d *PikPakShare) Init(ctx context.Context) error { } // 获取CaptchaToken - err := d.RefreshCaptchaToken(GetAction(http.MethodGet, "https://api-drive.mypikpak.com/drive/v1/share:batch_file_info"), "") + err := d.RefreshCaptchaToken(GetAction(http.MethodGet, "https://api-drive.mypikpak.net/drive/v1/share:batch_file_info"), "") if err != nil { return err } @@ -113,7 +113,7 @@ func (d *PikPakShare) Link(ctx context.Context, file model.Obj, args model.LinkA "file_id": file.GetID(), "pass_code_token": d.PassCodeToken, } - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share/file_info", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share/file_info", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index f333ca5f706..1b14a65aad6 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -68,51 +68,51 @@ const ( WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" - WebPackageName = "mypikpak.com" + WebPackageName = "mypikpak.net" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" PCClientVersion = "undefined" // 2.5.6.4831 - PCPackageName = "mypikpak.com" + PCPackageName = "mypikpak.net" PCSdkVersion = "8.0.3" ) var DlAddr = []string{ - "dl-a10b-0621.mypikpak.com", - "dl-a10b-0622.mypikpak.com", - "dl-a10b-0623.mypikpak.com", - "dl-a10b-0624.mypikpak.com", - "dl-a10b-0625.mypikpak.com", - "dl-a10b-0858.mypikpak.com", - "dl-a10b-0859.mypikpak.com", - "dl-a10b-0860.mypikpak.com", - "dl-a10b-0861.mypikpak.com", - "dl-a10b-0862.mypikpak.com", - "dl-a10b-0863.mypikpak.com", - "dl-a10b-0864.mypikpak.com", - "dl-a10b-0865.mypikpak.com", - "dl-a10b-0866.mypikpak.com", - "dl-a10b-0867.mypikpak.com", - "dl-a10b-0868.mypikpak.com", - "dl-a10b-0869.mypikpak.com", - "dl-a10b-0870.mypikpak.com", - "dl-a10b-0871.mypikpak.com", - "dl-a10b-0872.mypikpak.com", - "dl-a10b-0873.mypikpak.com", - "dl-a10b-0874.mypikpak.com", - "dl-a10b-0875.mypikpak.com", - "dl-a10b-0876.mypikpak.com", - "dl-a10b-0877.mypikpak.com", - "dl-a10b-0878.mypikpak.com", - "dl-a10b-0879.mypikpak.com", - "dl-a10b-0880.mypikpak.com", - "dl-a10b-0881.mypikpak.com", - "dl-a10b-0882.mypikpak.com", - "dl-a10b-0883.mypikpak.com", - "dl-a10b-0884.mypikpak.com", - "dl-a10b-0885.mypikpak.com", - "dl-a10b-0886.mypikpak.com", - "dl-a10b-0887.mypikpak.com", + "dl-a10b-0621.mypikpak.net", + "dl-a10b-0622.mypikpak.net", + "dl-a10b-0623.mypikpak.net", + "dl-a10b-0624.mypikpak.net", + "dl-a10b-0625.mypikpak.net", + "dl-a10b-0858.mypikpak.net", + "dl-a10b-0859.mypikpak.net", + "dl-a10b-0860.mypikpak.net", + "dl-a10b-0861.mypikpak.net", + "dl-a10b-0862.mypikpak.net", + "dl-a10b-0863.mypikpak.net", + "dl-a10b-0864.mypikpak.net", + "dl-a10b-0865.mypikpak.net", + "dl-a10b-0866.mypikpak.net", + "dl-a10b-0867.mypikpak.net", + "dl-a10b-0868.mypikpak.net", + "dl-a10b-0869.mypikpak.net", + "dl-a10b-0870.mypikpak.net", + "dl-a10b-0871.mypikpak.net", + "dl-a10b-0872.mypikpak.net", + "dl-a10b-0873.mypikpak.net", + "dl-a10b-0874.mypikpak.net", + "dl-a10b-0875.mypikpak.net", + "dl-a10b-0876.mypikpak.net", + "dl-a10b-0877.mypikpak.net", + "dl-a10b-0878.mypikpak.net", + "dl-a10b-0879.mypikpak.net", + "dl-a10b-0880.mypikpak.net", + "dl-a10b-0881.mypikpak.net", + "dl-a10b-0882.mypikpak.net", + "dl-a10b-0883.mypikpak.net", + "dl-a10b-0884.mypikpak.net", + "dl-a10b-0885.mypikpak.net", + "dl-a10b-0886.mypikpak.net", + "dl-a10b-0887.mypikpak.net", } func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { @@ -159,7 +159,7 @@ func (d *PikPakShare) getSharePassToken() error { "limit": "100", } var resp ShareResp - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -187,7 +187,7 @@ func (d *PikPakShare) getFiles(id string) ([]File, error) { "pass_code_token": d.PassCodeToken, } var resp ShareResp - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share/detail", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share/detail", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -345,7 +345,7 @@ func (d *PikPakShare) refreshCaptchaToken(action string, metas map[string]string } var e ErrResp var resp CaptchaTokenResponse - _, err := d.request("https://user.mypikpak.com/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://user.mypikpak.net/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { req.SetError(&e).SetBody(param) }, &resp) From 28d2367a87cfd76de59a24616d68128d3a9cc14a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 17 Nov 2024 22:24:06 +0800 Subject: [PATCH 359/659] fix(ci): no space left on device --- .github/workflows/release.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ef38566143..beed6fcd483 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,23 @@ jobs: name: Release runs-on: ${{ matrix.platform }} steps: + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Prerelease uses: irongut/EditRelease@v1.2.0 with: @@ -32,7 +49,6 @@ jobs: - name: Install dependencies run: | - sudo snap install zig --classic --beta docker pull crazymax/xgo:latest go install github.com/crazy-max/xgo@latest sudo apt install upx From 0ba754fd406041d9a7e1bf3ce0e9949914239a49 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 17 Nov 2024 23:11:03 +0800 Subject: [PATCH 360/659] fix(release): missing installation of zig --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index beed6fcd483..1d42019ad06 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,6 +49,7 @@ jobs: - name: Install dependencies run: | + sudo snap install zig --classic --beta docker pull crazymax/xgo:latest go install github.com/crazy-max/xgo@latest sudo apt install upx From 150dcc21474a05ecd61550b35c58821dec96bb73 Mon Sep 17 00:00:00 2001 From: Mmx Date: Thu, 21 Nov 2024 22:36:41 +0800 Subject: [PATCH 361/659] fix(sso): OIDC compatibility mode (#7524) --- server/handles/ssologin.go | 61 +++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 70298a9c3f0..1acd04764df 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -36,14 +36,21 @@ var opts = totp.ValidateOpts{ Algorithm: otp.AlgorithmSHA1, } +func ssoRedirectUri(c *gin.Context, useCompatibility bool, method string) string { + if useCompatibility { + return common.GetApiUrl(c.Request) + "/api/auth/" + method + } else { + return common.GetApiUrl(c.Request) + "/api/auth/sso_callback" + "?method=" + method + } +} + func SSOLoginRedirect(c *gin.Context) { method := c.Query("method") - usecompatibility := setting.GetBool(conf.SSOCompatibilityMode) + useCompatibility := setting.GetBool(conf.SSOCompatibilityMode) enabled := setting.GetBool(conf.SSOLoginEnabled) clientId := setting.GetStr(conf.SSOClientId) platform := setting.GetStr(conf.SSOLoginPlatform) - var r_url string - var redirect_uri string + var rUrl string if !enabled { common.ErrorStrResp(c, "Single sign-on is not enabled", 403) return @@ -53,37 +60,33 @@ func SSOLoginRedirect(c *gin.Context) { common.ErrorStrResp(c, "no method provided", 400) return } - if usecompatibility { - redirect_uri = common.GetApiUrl(c.Request) + "/api/auth/" + method - } else { - redirect_uri = common.GetApiUrl(c.Request) + "/api/auth/sso_callback" + "?method=" + method - } + redirectUri := ssoRedirectUri(c, useCompatibility, method) urlValues.Add("response_type", "code") - urlValues.Add("redirect_uri", redirect_uri) + urlValues.Add("redirect_uri", redirectUri) urlValues.Add("client_id", clientId) switch platform { case "Github": - r_url = "https://github.com/login/oauth/authorize?" + rUrl = "https://github.com/login/oauth/authorize?" urlValues.Add("scope", "read:user") case "Microsoft": - r_url = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?" + rUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?" urlValues.Add("scope", "user.read") urlValues.Add("response_mode", "query") case "Google": - r_url = "https://accounts.google.com/o/oauth2/v2/auth?" + rUrl = "https://accounts.google.com/o/oauth2/v2/auth?" urlValues.Add("scope", "https://www.googleapis.com/auth/userinfo.profile") case "Dingtalk": - r_url = "https://login.dingtalk.com/oauth2/auth?" + rUrl = "https://login.dingtalk.com/oauth2/auth?" urlValues.Add("scope", "openid") urlValues.Add("prompt", "consent") urlValues.Add("response_type", "code") case "Casdoor": endpoint := strings.TrimSuffix(setting.GetStr(conf.SSOEndpointName), "/") - r_url = endpoint + "/login/oauth/authorize?" + rUrl = endpoint + "/login/oauth/authorize?" urlValues.Add("scope", "profile") urlValues.Add("state", endpoint) case "OIDC": - oauth2Config, err := GetOIDCClient(c) + oauth2Config, err := GetOIDCClient(c, useCompatibility, redirectUri, method) if err != nil { common.ErrorStrResp(c, err.Error(), 400) return @@ -100,22 +103,14 @@ func SSOLoginRedirect(c *gin.Context) { common.ErrorStrResp(c, "invalid platform", 400) return } - c.Redirect(302, r_url+urlValues.Encode()) + c.Redirect(302, rUrl+urlValues.Encode()) } var ssoClient = resty.New().SetRetryCount(3) -func GetOIDCClient(c *gin.Context) (*oauth2.Config, error) { - var redirect_uri string - usecompatibility := setting.GetBool(conf.SSOCompatibilityMode) - argument := c.Query("method") - if usecompatibility { - argument = path.Base(c.Request.URL.Path) - } - if usecompatibility { - redirect_uri = common.GetApiUrl(c.Request) + "/api/auth/" + argument - } else { - redirect_uri = common.GetApiUrl(c.Request) + "/api/auth/sso_callback" + "?method=" + argument +func GetOIDCClient(c *gin.Context, useCompatibility bool, redirectUri, method string) (*oauth2.Config, error) { + if redirectUri == "" { + redirectUri = ssoRedirectUri(c, useCompatibility, method) } endpoint := setting.GetStr(conf.SSOEndpointName) provider, err := oidc.NewProvider(c, endpoint) @@ -127,7 +122,7 @@ func GetOIDCClient(c *gin.Context) (*oauth2.Config, error) { return &oauth2.Config{ ClientID: clientId, ClientSecret: clientSecret, - RedirectURL: redirect_uri, + RedirectURL: redirectUri, // Discovery returns the OAuth2 endpoints. Endpoint: provider.Endpoint(), @@ -181,9 +176,9 @@ func parseJWT(p string) ([]byte, error) { func OIDCLoginCallback(c *gin.Context) { useCompatibility := setting.GetBool(conf.SSOCompatibilityMode) - argument := c.Query("method") + method := c.Query("method") if useCompatibility { - argument = path.Base(c.Request.URL.Path) + method = path.Base(c.Request.URL.Path) } clientId := setting.GetStr(conf.SSOClientId) endpoint := setting.GetStr(conf.SSOEndpointName) @@ -192,7 +187,7 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorResp(c, err, 400) return } - oauth2Config, err := GetOIDCClient(c) + oauth2Config, err := GetOIDCClient(c, useCompatibility, "", method) if err != nil { common.ErrorResp(c, err, 400) return @@ -236,7 +231,7 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorStrResp(c, "cannot get username from OIDC provider", 400) return } - if argument == "get_sso_id" { + if method == "get_sso_id" { if useCompatibility { c.Redirect(302, common.GetApiUrl(c.Request)+"/@manage?sso_id="+userID) return @@ -252,7 +247,7 @@ func OIDCLoginCallback(c *gin.Context) { c.Data(200, "text/html; charset=utf-8", []byte(html)) return } - if argument == "sso_get_token" { + if method == "sso_get_token" { user, err := db.GetUserBySSOID(userID) if err != nil { user, err = autoRegister(userID, userID, err) From 12b429584ecceaf88af3465fc3d5bc580187e6ae Mon Sep 17 00:00:00 2001 From: Mmx Date: Thu, 21 Nov 2024 22:37:19 +0800 Subject: [PATCH 362/659] feat(security): generating random string with crypto rand (#7525) --- pkg/utils/random/random.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/pkg/utils/random/random.go b/pkg/utils/random/random.go index 65fbf14a0d3..c3f3dd48377 100644 --- a/pkg/utils/random/random.go +++ b/pkg/utils/random/random.go @@ -1,20 +1,27 @@ package random import ( - "math/rand" + "crypto/rand" + "math/big" + mathRand "math/rand" "time" "github.com/google/uuid" ) -var Rand *rand.Rand +var Rand *mathRand.Rand const letterBytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" func String(n int) string { b := make([]byte, n) + letterLen := big.NewInt(int64(len(letterBytes))) for i := range b { - b[i] = letterBytes[Rand.Intn(len(letterBytes))] + idx, err := rand.Int(rand.Reader, letterLen) + if err != nil { + panic(err) + } + b[i] = letterBytes[idx.Int64()] } return string(b) } @@ -24,10 +31,10 @@ func Token() string { } func RangeInt64(left, right int64) int64 { - return rand.Int63n(left+right) - left + return mathRand.Int63n(left+right) - left } func init() { - s := rand.NewSource(time.Now().UnixNano()) - Rand = rand.New(s) + s := mathRand.NewSource(time.Now().UnixNano()) + Rand = mathRand.New(s) } From 398c04386ae537d0cb6e79a360c0b155583bf146 Mon Sep 17 00:00:00 2001 From: Mmx Date: Thu, 21 Nov 2024 22:38:04 +0800 Subject: [PATCH 363/659] feat(sso): generate and verify OAuth state with go-cache (#7527) --- server/handles/ssologin.go | 44 ++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 1acd04764df..cb5fc4ca6c4 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -1,10 +1,10 @@ package handles import ( - "encoding/base32" "encoding/base64" "errors" "fmt" + "github.com/Xhofe/go-cache" "net/http" "net/url" "path" @@ -21,19 +21,28 @@ import ( "github.com/coreos/go-oidc" "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" - "github.com/pquerna/otp" - "github.com/pquerna/otp/totp" "golang.org/x/oauth2" "gorm.io/gorm" ) -var opts = totp.ValidateOpts{ - // state verify won't expire in 30 secs, which is quite enough for the callback - Period: 30, - Skew: 1, - // in some OIDC providers(such as Authelia), state parameter must be at least 8 characters - Digits: otp.DigitsEight, - Algorithm: otp.AlgorithmSHA1, +const stateLength = 16 +const stateExpire = time.Minute * 5 + +var stateCache = cache.NewMemCache[string](cache.WithShards[string](stateLength)) + +func _keyState(clientID, state string) string { + return fmt.Sprintf("%s_%s", clientID, state) +} + +func generateState(clientID, ip string) string { + state := random.String(stateLength) + stateCache.Set(_keyState(clientID, state), ip, cache.WithEx[string](stateExpire)) + return state +} + +func verifyState(clientID, ip, state string) bool { + value, ok := stateCache.Get(_keyState(clientID, state)) + return ok && value == ip } func ssoRedirectUri(c *gin.Context, useCompatibility bool, method string) string { @@ -91,12 +100,7 @@ func SSOLoginRedirect(c *gin.Context) { common.ErrorStrResp(c, err.Error(), 400) return } - // generate state parameter - state, err := totp.GenerateCodeCustom(base32.StdEncoding.EncodeToString([]byte(oauth2Config.ClientSecret)), time.Now(), opts) - if err != nil { - common.ErrorStrResp(c, err.Error(), 400) - return - } + state := generateState(clientId, c.ClientIP()) c.Redirect(http.StatusFound, oauth2Config.AuthCodeURL(state)) return default: @@ -192,13 +196,7 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorResp(c, err, 400) return } - // add state verify process - stateVerification, err := totp.ValidateCustom(c.Query("state"), base32.StdEncoding.EncodeToString([]byte(oauth2Config.ClientSecret)), time.Now(), opts) - if err != nil { - common.ErrorResp(c, err, 400) - return - } - if !stateVerification { + if !verifyState(clientId, c.ClientIP(), c.Query("state")) { common.ErrorStrResp(c, "incorrect or expired state parameter", 400) return } From 25c5e075a977f24a35489cb72b665cc440c96255 Mon Sep 17 00:00:00 2001 From: Rirmach Date: Thu, 21 Nov 2024 22:38:41 +0800 Subject: [PATCH 364/659] fix(local): Preserve file owner when copying (#7528) --- drivers/local/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index c39cec10c6b..229c86925fb 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -280,7 +280,7 @@ func (d *Local) Copy(_ context.Context, srcObj, dstDir model.Obj) error { return cp.Copy(srcPath, dstPath, cp.Options{ Sync: true, // Sync file to disk after copy, may have performance penalty in filesystem such as ZFS PreserveTimes: true, - NumOfWorkers: 0, // Serialized copy without using goroutine + PreserveOwner: true, }) } From 4c0cffd29b7d8b078388420c0211da31ec47a577 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Thu, 21 Nov 2024 22:39:14 +0800 Subject: [PATCH 365/659] fix(net): close of closed channel (#7529) --- internal/net/request.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/net/request.go b/internal/net/request.go index 088ff66ab4f..c0f547ba8c3 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "github.com/alist-org/alist/v3/pkg/utils" "io" "math" "net/http" @@ -13,6 +12,8 @@ import ( "sync" "time" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/aws/aws-sdk-go/aws/awsutil" log "github.com/sirupsen/logrus" @@ -168,6 +169,9 @@ func (d *downloader) sendChunkTask() *chunk { // when the final reader Close, we interrupt func (d *downloader) interrupt() error { + if d.chunkChannel == nil { + return nil + } d.cancel() if d.written != d.params.Range.Length { log.Debugf("Downloader interrupt before finish") @@ -177,6 +181,7 @@ func (d *downloader) interrupt() error { } defer func() { close(d.chunkChannel) + d.chunkChannel = nil for _, buf := range d.bufs { buf.Close() } From 2dec756f232e2cd3be5c3bf4df18658dc605a240 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Thu, 21 Nov 2024 22:40:39 +0800 Subject: [PATCH 366/659] fix(pikpak&pikpak_share): captcha_sign error (#7530 close #7481 close #7482) --- drivers/pikpak/driver.go | 16 ---- drivers/pikpak/meta.go | 18 ++--- drivers/pikpak/util.go | 138 +++++++------------------------- drivers/pikpak_share/driver.go | 16 ---- drivers/pikpak_share/meta.go | 12 ++- drivers/pikpak_share/util.go | 139 +++++++-------------------------- 6 files changed, 71 insertions(+), 268 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 24de24d4f59..5640d7652f4 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -14,7 +14,6 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/oauth2" "net/http" - "regexp" "strconv" "strings" ) @@ -49,7 +48,6 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.Common.CaptchaToken = token op.MustSaveDriverStorage(d) }, - LowLatencyAddr: "", } } @@ -138,14 +136,6 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.Addition.RefreshToken = d.RefreshToken op.MustSaveDriverStorage(d) - if d.UseLowLatencyAddress && d.Addition.CustomLowLatencyAddress != "" { - d.Common.LowLatencyAddr = d.Addition.CustomLowLatencyAddress - } else if d.UseLowLatencyAddress { - d.Common.LowLatencyAddr = findLowestLatencyAddress(DlAddr) - d.Addition.CustomLowLatencyAddress = d.Common.LowLatencyAddr - op.MustSaveDriverStorage(d) - } - return nil } @@ -188,12 +178,6 @@ func (d *PikPak) Link(ctx context.Context, file model.Obj, args model.LinkArgs) url = resp.Medias[0].Link.Url } - if d.UseLowLatencyAddress && d.Common.LowLatencyAddr != "" { - // 替换为加速链接 - re := regexp.MustCompile(`https://[^/]+/download/`) - url = re.ReplaceAllString(url, "https://"+d.Common.LowLatencyAddr+"/download/") - } - return &model.Link{ URL: url, }, nil diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index 4d25ecc605e..7e525787814 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -7,16 +7,14 @@ import ( type Addition struct { driver.RootID - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - Platform string `json:"platform" required:"true" type:"select" options:"android,web,pc"` - RefreshToken string `json:"refresh_token" required:"true" default:""` - RefreshTokenMethod string `json:"refresh_token_method" required:"true" type:"select" options:"oauth2,http"` - CaptchaToken string `json:"captcha_token" default:""` - DeviceID string `json:"device_id" required:"false" default:""` - DisableMediaLink bool `json:"disable_media_link" default:"true"` - UseLowLatencyAddress bool `json:"use_low_latency_address" default:"false"` - CustomLowLatencyAddress string `json:"custom_low_latency_address" default:""` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + Platform string `json:"platform" required:"true" default:"web" type:"select" options:"android,web,pc"` + RefreshToken string `json:"refresh_token" required:"true" default:""` + RefreshTokenMethod string `json:"refresh_token_method" required:"true" type:"select" options:"oauth2,http"` + CaptchaToken string `json:"captcha_token" default:""` + DeviceID string `json:"device_id" required:"false" default:""` + DisableMediaLink bool `json:"disable_media_link" default:"true"` } var config = driver.Config{ diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 6c5c88ad4b2..67077fb861e 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -30,32 +30,34 @@ import ( // do others that not defined in Driver interface var AndroidAlgorithms = []string{ - "aDhgaSE3MsjROCmpmsWqP1sJdFJ", - "+oaVkqdd8MJuKT+uMr2AYKcd9tdWge3XPEPR2hcePUknd", - "u/sd2GgT2fTytRcKzGicHodhvIltMntA3xKw2SRv7S48OdnaQIS5mn", - "2WZiae2QuqTOxBKaaqCNHCW3olu2UImelkDzBn", - "/vJ3upic39lgmrkX855Qx", - "yNc9ruCVMV7pGV7XvFeuLMOcy1", - "4FPq8mT3JQ1jzcVxMVfwFftLQm33M7i", - "xozoy5e3Ea", + "7xOq4Z8s", + "QE9/9+IQco", + "WdX5J9CPLZp", + "NmQ5qFAXqH3w984cYhMeC5TJR8j", + "cc44M+l7GDhav", + "KxGjo/wHB+Yx8Lf7kMP+/m9I+", + "wla81BUVSmDkctHDpUT", + "c6wMr1sm1WxiR3i8LDAm3W", + "hRLrEQCFNYi0PFPV", + "o1J41zIraDtJPNuhBu7Ifb/q3", + "U", + "RrbZvV0CTu3gaZJ56PVKki4IeP", + "NNuRbLckJqUp1Do0YlrKCUP", + "UUwnBbipMTvInA0U0E9", + "VzGc", } var WebAlgorithms = []string{ - "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", - "+r6CQVxjzJV6LCV", - "F", - "pFJRC", - "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", - "/750aCr4lm/Sly/c", - "RB+DT/gZCrbV", - "", - "CyLsf7hdkIRxRm215hl", - "7xHvLi2tOYP0Y92b", - "ZGTXXxu8E/MIWaEDB+Sm/", - "1UI3", - "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", - "ihtqpG6FMt65+Xk+tWUH2", - "NhXXU9rg4XXdzo7u5o", + "fyZ4+p77W1U4zcWBUwefAIFhFxvADWtT1wzolCxhg9q7etmGUjXr", + "uSUX02HYJ1IkyLdhINEFcCf7l2", + "iWt97bqD/qvjIaPXB2Ja5rsBWtQtBZZmaHH2rMR41", + "3binT1s/5a1pu3fGsN", + "8YCCU+AIr7pg+yd7CkQEY16lDMwi8Rh4WNp5", + "DYS3StqnAEKdGddRP8CJrxUSFh", + "crquW+4", + "ryKqvW9B9hly+JAymXCIfag5Z", + "Hr08T/NDTX1oSJfHk90c", + "i", } var PCAlgorithms = []string{ @@ -80,59 +82,21 @@ const ( const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - AndroidClientVersion = "1.48.3" + AndroidClientVersion = "1.49.3" AndroidPackageName = "com.pikcloud.pikpak" AndroidSdkVersion = "2.0.4.204101" WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - WebClientVersion = "2.0.0" - WebPackageName = "mypikpak.net" + WebClientVersion = "undefined" + WebPackageName = "drive.mypikpak.com" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" PCClientVersion = "undefined" // 2.5.6.4831 - PCPackageName = "mypikpak.net" + PCPackageName = "mypikpak.com" PCSdkVersion = "8.0.3" ) -var DlAddr = []string{ - "dl-a10b-0621.mypikpak.net", - "dl-a10b-0622.mypikpak.net", - "dl-a10b-0623.mypikpak.net", - "dl-a10b-0624.mypikpak.net", - "dl-a10b-0625.mypikpak.net", - "dl-a10b-0858.mypikpak.net", - "dl-a10b-0859.mypikpak.net", - "dl-a10b-0860.mypikpak.net", - "dl-a10b-0861.mypikpak.net", - "dl-a10b-0862.mypikpak.net", - "dl-a10b-0863.mypikpak.net", - "dl-a10b-0864.mypikpak.net", - "dl-a10b-0865.mypikpak.net", - "dl-a10b-0866.mypikpak.net", - "dl-a10b-0867.mypikpak.net", - "dl-a10b-0868.mypikpak.net", - "dl-a10b-0869.mypikpak.net", - "dl-a10b-0870.mypikpak.net", - "dl-a10b-0871.mypikpak.net", - "dl-a10b-0872.mypikpak.net", - "dl-a10b-0873.mypikpak.net", - "dl-a10b-0874.mypikpak.net", - "dl-a10b-0875.mypikpak.net", - "dl-a10b-0876.mypikpak.net", - "dl-a10b-0877.mypikpak.net", - "dl-a10b-0878.mypikpak.net", - "dl-a10b-0879.mypikpak.net", - "dl-a10b-0880.mypikpak.net", - "dl-a10b-0881.mypikpak.net", - "dl-a10b-0882.mypikpak.net", - "dl-a10b-0883.mypikpak.net", - "dl-a10b-0884.mypikpak.net", - "dl-a10b-0885.mypikpak.net", - "dl-a10b-0886.mypikpak.net", - "dl-a10b-0887.mypikpak.net", -} - func (d *PikPak) login() error { // 检查用户名和密码是否为空 if d.Addition.Username == "" || d.Addition.Password == "" { @@ -338,7 +302,6 @@ type Common struct { UserAgent string // 验证码token刷新成功回调 RefreshCTokenCk func(token string) - LowLatencyAddr string } func generateDeviceSign(deviceID, packageName string) string { @@ -729,46 +692,3 @@ func OssOption(params *S3Params) []oss.Option { } return options } - -type AddressLatency struct { - Address string - Latency time.Duration -} - -func checkLatency(address string, wg *sync.WaitGroup, ch chan<- AddressLatency) { - defer wg.Done() - start := time.Now() - resp, err := http.Get("https://" + address + "/generate_204") - if err != nil { - ch <- AddressLatency{Address: address, Latency: time.Hour} // Set high latency on error - return - } - defer resp.Body.Close() - latency := time.Since(start) - ch <- AddressLatency{Address: address, Latency: latency} -} - -func findLowestLatencyAddress(addresses []string) string { - var wg sync.WaitGroup - ch := make(chan AddressLatency, len(addresses)) - - for _, address := range addresses { - wg.Add(1) - go checkLatency(address, &wg, ch) - } - - wg.Wait() - close(ch) - - var lowestLatencyAddress string - lowestLatency := time.Hour - - for result := range ch { - if result.Latency < lowestLatency { - lowestLatency = result.Latency - lowestLatencyAddress = result.Address - } - } - - return lowestLatencyAddress -} diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index f107ac17ba3..d527a1ab1a4 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -4,7 +4,6 @@ import ( "context" "github.com/alist-org/alist/v3/internal/op" "net/http" - "regexp" "time" "github.com/alist-org/alist/v3/internal/driver" @@ -37,7 +36,6 @@ func (d *PikPakShare) Init(ctx context.Context) error { d.Common.CaptchaToken = token op.MustSaveDriverStorage(d) }, - LowLatencyAddr: "", } } @@ -71,14 +69,6 @@ func (d *PikPakShare) Init(ctx context.Context) error { d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.5.6.4831 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" } - if d.UseLowLatencyAddress && d.Addition.CustomLowLatencyAddress != "" { - d.Common.LowLatencyAddr = d.Addition.CustomLowLatencyAddress - } else if d.UseLowLatencyAddress { - d.Common.LowLatencyAddr = findLowestLatencyAddress(DlAddr) - d.Addition.CustomLowLatencyAddress = d.Common.LowLatencyAddr - op.MustSaveDriverStorage(d) - } - // 获取CaptchaToken err := d.RefreshCaptchaToken(GetAction(http.MethodGet, "https://api-drive.mypikpak.net/drive/v1/share:batch_file_info"), "") if err != nil { @@ -131,12 +121,6 @@ func (d *PikPakShare) Link(ctx context.Context, file model.Obj, args model.LinkA } - if d.UseLowLatencyAddress && d.Common.LowLatencyAddr != "" { - // 替换为加速链接 - re := regexp.MustCompile(`https://[^/]+/download/`) - downloadUrl = re.ReplaceAllString(downloadUrl, "https://"+d.Common.LowLatencyAddr+"/download/") - } - return &model.Link{ URL: downloadUrl, }, nil diff --git a/drivers/pikpak_share/meta.go b/drivers/pikpak_share/meta.go index dc551e028a0..30bccbdce87 100644 --- a/drivers/pikpak_share/meta.go +++ b/drivers/pikpak_share/meta.go @@ -7,13 +7,11 @@ import ( type Addition struct { driver.RootID - ShareId string `json:"share_id" required:"true"` - SharePwd string `json:"share_pwd"` - Platform string `json:"platform" required:"true" type:"select" options:"android,web,pc"` - DeviceID string `json:"device_id" required:"false" default:""` - UseTransCodingAddress bool `json:"use_transcoding_address" required:"true" default:"false"` - UseLowLatencyAddress bool `json:"use_low_latency_address" default:"false"` - CustomLowLatencyAddress string `json:"custom_low_latency_address" default:""` + ShareId string `json:"share_id" required:"true"` + SharePwd string `json:"share_pwd"` + Platform string `json:"platform" default:"web" required:"true" type:"select" options:"android,web,pc"` + DeviceID string `json:"device_id" required:"false" default:""` + UseTransCodingAddress bool `json:"use_transcoding_address" required:"true" default:"false"` } var config = driver.Config{ diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index 1b14a65aad6..172a61487d8 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -10,7 +10,6 @@ import ( "net/http" "regexp" "strings" - "sync" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -18,32 +17,34 @@ import ( ) var AndroidAlgorithms = []string{ - "aDhgaSE3MsjROCmpmsWqP1sJdFJ", - "+oaVkqdd8MJuKT+uMr2AYKcd9tdWge3XPEPR2hcePUknd", - "u/sd2GgT2fTytRcKzGicHodhvIltMntA3xKw2SRv7S48OdnaQIS5mn", - "2WZiae2QuqTOxBKaaqCNHCW3olu2UImelkDzBn", - "/vJ3upic39lgmrkX855Qx", - "yNc9ruCVMV7pGV7XvFeuLMOcy1", - "4FPq8mT3JQ1jzcVxMVfwFftLQm33M7i", - "xozoy5e3Ea", + "7xOq4Z8s", + "QE9/9+IQco", + "WdX5J9CPLZp", + "NmQ5qFAXqH3w984cYhMeC5TJR8j", + "cc44M+l7GDhav", + "KxGjo/wHB+Yx8Lf7kMP+/m9I+", + "wla81BUVSmDkctHDpUT", + "c6wMr1sm1WxiR3i8LDAm3W", + "hRLrEQCFNYi0PFPV", + "o1J41zIraDtJPNuhBu7Ifb/q3", + "U", + "RrbZvV0CTu3gaZJ56PVKki4IeP", + "NNuRbLckJqUp1Do0YlrKCUP", + "UUwnBbipMTvInA0U0E9", + "VzGc", } var WebAlgorithms = []string{ - "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", - "+r6CQVxjzJV6LCV", - "F", - "pFJRC", - "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", - "/750aCr4lm/Sly/c", - "RB+DT/gZCrbV", - "", - "CyLsf7hdkIRxRm215hl", - "7xHvLi2tOYP0Y92b", - "ZGTXXxu8E/MIWaEDB+Sm/", - "1UI3", - "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", - "ihtqpG6FMt65+Xk+tWUH2", - "NhXXU9rg4XXdzo7u5o", + "fyZ4+p77W1U4zcWBUwefAIFhFxvADWtT1wzolCxhg9q7etmGUjXr", + "uSUX02HYJ1IkyLdhINEFcCf7l2", + "iWt97bqD/qvjIaPXB2Ja5rsBWtQtBZZmaHH2rMR41", + "3binT1s/5a1pu3fGsN", + "8YCCU+AIr7pg+yd7CkQEY16lDMwi8Rh4WNp5", + "DYS3StqnAEKdGddRP8CJrxUSFh", + "crquW+4", + "ryKqvW9B9hly+JAymXCIfag5Z", + "Hr08T/NDTX1oSJfHk90c", + "i", } var PCAlgorithms = []string{ @@ -62,59 +63,21 @@ var PCAlgorithms = []string{ const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - AndroidClientVersion = "1.48.3" + AndroidClientVersion = "1.49.3" AndroidPackageName = "com.pikcloud.pikpak" AndroidSdkVersion = "2.0.4.204101" WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - WebClientVersion = "2.0.0" - WebPackageName = "mypikpak.net" + WebClientVersion = "undefined" + WebPackageName = "drive.mypikpak.com" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" PCClientVersion = "undefined" // 2.5.6.4831 - PCPackageName = "mypikpak.net" + PCPackageName = "mypikpak.com" PCSdkVersion = "8.0.3" ) -var DlAddr = []string{ - "dl-a10b-0621.mypikpak.net", - "dl-a10b-0622.mypikpak.net", - "dl-a10b-0623.mypikpak.net", - "dl-a10b-0624.mypikpak.net", - "dl-a10b-0625.mypikpak.net", - "dl-a10b-0858.mypikpak.net", - "dl-a10b-0859.mypikpak.net", - "dl-a10b-0860.mypikpak.net", - "dl-a10b-0861.mypikpak.net", - "dl-a10b-0862.mypikpak.net", - "dl-a10b-0863.mypikpak.net", - "dl-a10b-0864.mypikpak.net", - "dl-a10b-0865.mypikpak.net", - "dl-a10b-0866.mypikpak.net", - "dl-a10b-0867.mypikpak.net", - "dl-a10b-0868.mypikpak.net", - "dl-a10b-0869.mypikpak.net", - "dl-a10b-0870.mypikpak.net", - "dl-a10b-0871.mypikpak.net", - "dl-a10b-0872.mypikpak.net", - "dl-a10b-0873.mypikpak.net", - "dl-a10b-0874.mypikpak.net", - "dl-a10b-0875.mypikpak.net", - "dl-a10b-0876.mypikpak.net", - "dl-a10b-0877.mypikpak.net", - "dl-a10b-0878.mypikpak.net", - "dl-a10b-0879.mypikpak.net", - "dl-a10b-0880.mypikpak.net", - "dl-a10b-0881.mypikpak.net", - "dl-a10b-0882.mypikpak.net", - "dl-a10b-0883.mypikpak.net", - "dl-a10b-0884.mypikpak.net", - "dl-a10b-0885.mypikpak.net", - "dl-a10b-0886.mypikpak.net", - "dl-a10b-0887.mypikpak.net", -} - func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ @@ -227,7 +190,6 @@ type Common struct { UserAgent string // 验证码token刷新成功回调 RefreshCTokenCk func(token string) - LowLatencyAddr string } func (c *Common) SetUserAgent(userAgent string) { @@ -367,46 +329,3 @@ func (d *PikPakShare) refreshCaptchaToken(action string, metas map[string]string d.Common.SetCaptchaToken(resp.CaptchaToken) return nil } - -type AddressLatency struct { - Address string - Latency time.Duration -} - -func checkLatency(address string, wg *sync.WaitGroup, ch chan<- AddressLatency) { - defer wg.Done() - start := time.Now() - resp, err := http.Get("https://" + address + "/generate_204") - if err != nil { - ch <- AddressLatency{Address: address, Latency: time.Hour} // Set high latency on error - return - } - defer resp.Body.Close() - latency := time.Since(start) - ch <- AddressLatency{Address: address, Latency: latency} -} - -func findLowestLatencyAddress(addresses []string) string { - var wg sync.WaitGroup - ch := make(chan AddressLatency, len(addresses)) - - for _, address := range addresses { - wg.Add(1) - go checkLatency(address, &wg, ch) - } - - wg.Wait() - close(ch) - - var lowestLatencyAddress string - lowestLatency := time.Hour - - for result := range ch { - if result.Latency < lowestLatency { - lowestLatency = result.Latency - lowestLatencyAddress = result.Address - } - } - - return lowestLatencyAddress -} From 94915b214838714431a6735ebd4632eeec28568d Mon Sep 17 00:00:00 2001 From: Kuingsmile Date: Thu, 21 Nov 2024 22:41:23 +0800 Subject: [PATCH 367/659] fix(baidu_netdisk): update fileToObj to use ServerCtime and ServerMtime (#7535) --- drivers/baidu_netdisk/types.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 6f3bf13b3e4..728273b8dab 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -56,11 +56,11 @@ func fileToObj(f File) *model.ObjThumb { if f.ServerFilename == "" { f.ServerFilename = path.Base(f.Path) } - if f.LocalCtime == 0 { - f.LocalCtime = f.Ctime + if f.ServerCtime == 0 { + f.ServerCtime = f.Ctime } - if f.LocalMtime == 0 { - f.LocalMtime = f.Mtime + if f.ServerMtime == 0 { + f.ServerMtime = f.Mtime } return &model.ObjThumb{ Object: model.Object{ @@ -68,8 +68,8 @@ func fileToObj(f File) *model.ObjThumb { Path: f.Path, Name: f.ServerFilename, Size: f.Size, - Modified: time.Unix(f.LocalMtime, 0), - Ctime: time.Unix(f.LocalCtime, 0), + Modified: time.Unix(f.ServerMtime, 0), + Ctime: time.Unix(f.ServerCtime, 0), IsFolder: f.Isdir == 1, // 直接获取的MD5是错误的 From 492b49d77af21be06a2ec7e8259b3da3873375e3 Mon Sep 17 00:00:00 2001 From: alist666 Date: Sat, 7 Dec 2024 01:00:25 +0800 Subject: [PATCH 368/659] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bed2eadf160..701bbc2ff3b 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Document - + ## Demo @@ -138,4 +138,4 @@ The `AList` is open-source software licensed under the AGPL-3.0 license. --- -> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/alist-org) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) +> [@GitHub](https://github.com/alist-org) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) From 2d3605c6847554409a148c9ca549c10a1b618859 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sat, 7 Dec 2024 17:02:52 +0800 Subject: [PATCH 369/659] fix(baidu_photo): cookie login fix download error (#7602) --- drivers/baidu_photo/driver.go | 33 ++++++------ drivers/baidu_photo/meta.go | 13 ++--- drivers/baidu_photo/utils.go | 99 +++++++++++++++++------------------ 3 files changed, 70 insertions(+), 75 deletions(-) diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index 94716983e52..d0d69e82222 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -27,9 +27,9 @@ type BaiduPhoto struct { model.Storage Addition - AccessToken string - Uk int64 - root model.Obj + // AccessToken string + Uk int64 + root model.Obj uploadThread int } @@ -48,9 +48,9 @@ func (d *BaiduPhoto) Init(ctx context.Context) error { d.uploadThread, d.UploadThread = 3, "3" } - if err := d.refreshToken(); err != nil { - return err - } + // if err := d.refreshToken(); err != nil { + // return err + // } // root if d.AlbumID != "" { @@ -82,7 +82,7 @@ func (d *BaiduPhoto) GetRoot(ctx context.Context) (model.Obj, error) { } func (d *BaiduPhoto) Drop(ctx context.Context) error { - d.AccessToken = "" + // d.AccessToken = "" d.Uk = 0 d.root = nil return nil @@ -140,14 +140,13 @@ func (d *BaiduPhoto) Link(ctx context.Context, file model.Obj, args model.LinkAr // 处理共享相册 if d.Uk != file.Uk { // 有概率无法获取到链接 - return d.linkAlbum(ctx, file, args) - - // 接口被限制，只能使用cookie - // f, err := d.CopyAlbumFile(ctx, file) - // if err != nil { - // return nil, err - // } - // return d.linkFile(ctx, f, args) + // return d.linkAlbum(ctx, file, args) + + f, err := d.CopyAlbumFile(ctx, file) + if err != nil { + return nil, err + } + return d.linkFile(ctx, f, args) } return d.linkFile(ctx, &file.File, args) } @@ -292,7 +291,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil } // 尝试获取之前的进度 - precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) + precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, strconv.FormatInt(d.Uk, 10), contentMd5) if !ok { _, err = d.Post(FILE_API_URL_V1+"/precreate", func(r *resty.Request) { r.SetContext(ctx) @@ -343,7 +342,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) - base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + base.SaveUploadProgress(d, strconv.FormatInt(d.Uk, 10), contentMd5) } return nil, err } diff --git a/drivers/baidu_photo/meta.go b/drivers/baidu_photo/meta.go index da2229f5424..3bc2f6227c5 100644 --- a/drivers/baidu_photo/meta.go +++ b/drivers/baidu_photo/meta.go @@ -6,13 +6,14 @@ import ( ) type Addition struct { - RefreshToken string `json:"refresh_token" required:"true"` - ShowType string `json:"show_type" type:"select" options:"root,root_only_album,root_only_file" default:"root"` - AlbumID string `json:"album_id"` + // RefreshToken string `json:"refresh_token" required:"true"` + Cookie string `json:"cookie" required:"true"` + ShowType string `json:"show_type" type:"select" options:"root,root_only_album,root_only_file" default:"root"` + AlbumID string `json:"album_id"` //AlbumPassword string `json:"album_password"` - DeleteOrigin bool `json:"delete_origin"` - ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` - ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` + DeleteOrigin bool `json:"delete_origin"` + // ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` + // ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` } diff --git a/drivers/baidu_photo/utils.go b/drivers/baidu_photo/utils.go index c8c5b7ee88b..0b960593bce 100644 --- a/drivers/baidu_photo/utils.go +++ b/drivers/baidu_photo/utils.go @@ -10,9 +10,7 @@ import ( "unicode" "github.com/alist-org/alist/v3/drivers/base" - "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -27,7 +25,8 @@ const ( func (d *BaiduPhoto) Request(client *resty.Client, furl string, method string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { req := client.R(). - SetQueryParam("access_token", d.AccessToken) + // SetQueryParam("access_token", d.AccessToken) + SetHeader("Cookie", d.Cookie) if callback != nil { callback(req) } @@ -49,10 +48,10 @@ func (d *BaiduPhoto) Request(client *resty.Client, furl string, method string, c return nil, fmt.Errorf("no shared albums found") case 50100: return nil, fmt.Errorf("illegal title, only supports 50 characters") - case -6: - if err = d.refreshToken(); err != nil { - return nil, err - } + // case -6: + // if err = d.refreshToken(); err != nil { + // return nil, err + // } default: return nil, fmt.Errorf("errno: %d, refer to https://photo.baidu.com/union/doc", erron) } @@ -67,29 +66,29 @@ func (d *BaiduPhoto) Request(client *resty.Client, furl string, method string, c // return res.Body(), nil //} -func (d *BaiduPhoto) refreshToken() error { - u := "https://openapi.baidu.com/oauth/2.0/token" - var resp base.TokenResp - var e TokenErrResp - _, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetQueryParams(map[string]string{ - "grant_type": "refresh_token", - "refresh_token": d.RefreshToken, - "client_id": d.ClientID, - "client_secret": d.ClientSecret, - }).Get(u) - if err != nil { - return err - } - if e.ErrorMsg != "" { - return &e - } - if resp.RefreshToken == "" { - return errs.EmptyToken - } - d.AccessToken, d.RefreshToken = resp.AccessToken, resp.RefreshToken - op.MustSaveDriverStorage(d) - return nil -} +// func (d *BaiduPhoto) refreshToken() error { +// u := "https://openapi.baidu.com/oauth/2.0/token" +// var resp base.TokenResp +// var e TokenErrResp +// _, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetQueryParams(map[string]string{ +// "grant_type": "refresh_token", +// "refresh_token": d.RefreshToken, +// "client_id": d.ClientID, +// "client_secret": d.ClientSecret, +// }).Get(u) +// if err != nil { +// return err +// } +// if e.ErrorMsg != "" { +// return &e +// } +// if resp.RefreshToken == "" { +// return errs.EmptyToken +// } +// d.AccessToken, d.RefreshToken = resp.AccessToken, resp.RefreshToken +// op.MustSaveDriverStorage(d) +// return nil +// } func (d *BaiduPhoto) Get(furl string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { return d.Request(base.RestyClient, furl, http.MethodGet, callback, resp) @@ -363,10 +362,6 @@ func (d *BaiduPhoto) linkAlbum(ctx context.Context, file *AlbumFile, args model. location := resp.Header().Get("Location") - if err != nil { - return nil, err - } - link := &model.Link{ URL: location, Header: http.Header{ @@ -388,36 +383,36 @@ func (d *BaiduPhoto) linkFile(ctx context.Context, file *File, args model.LinkAr headers["X-Forwarded-For"] = args.IP } - // var downloadUrl struct { - // Dlink string `json:"dlink"` - // } - // _, err := d.Get(FILE_API_URL_V1+"/download", func(r *resty.Request) { - // r.SetContext(ctx) - // r.SetHeaders(headers) - // r.SetQueryParams(map[string]string{ - // "fsid": fmt.Sprint(file.Fsid), - // }) - // }, &downloadUrl) - - resp, err := d.Request(base.NoRedirectClient, FILE_API_URL_V1+"/download", http.MethodHead, func(r *resty.Request) { + var downloadUrl struct { + Dlink string `json:"dlink"` + } + _, err := d.Get(FILE_API_URL_V2+"/download", func(r *resty.Request) { r.SetContext(ctx) r.SetHeaders(headers) r.SetQueryParams(map[string]string{ "fsid": fmt.Sprint(file.Fsid), }) - }, nil) + }, &downloadUrl) + + // resp, err := d.Request(base.NoRedirectClient, FILE_API_URL_V1+"/download", http.MethodHead, func(r *resty.Request) { + // r.SetContext(ctx) + // r.SetHeaders(headers) + // r.SetQueryParams(map[string]string{ + // "fsid": fmt.Sprint(file.Fsid), + // }) + // }, nil) if err != nil { return nil, err } - if resp.StatusCode() != 302 { - return nil, fmt.Errorf("not found 302 redirect") - } + // if resp.StatusCode() != 302 { + // return nil, fmt.Errorf("not found 302 redirect") + // } - location := resp.Header().Get("Location") + // location := resp.Header().Get("Location") link := &model.Link{ - URL: location, + URL: downloadUrl.Dlink, Header: http.Header{ "User-Agent": []string{headers["User-Agent"]}, "Referer": []string{"https://photo.baidu.com/"}, From fa15c576f0fcc2ced45f55986211fda0fe4229a6 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Sat, 7 Dec 2024 17:03:46 +0800 Subject: [PATCH 370/659] fix(pikpak): remove oauth2 method (#7567 close #7545) --- drivers/pikpak/driver.go | 33 +++------------------------ drivers/pikpak/meta.go | 15 ++++++------- drivers/pikpak/util.go | 48 +++------------------------------------- 3 files changed, 13 insertions(+), 83 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 5640d7652f4..3db273d652b 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -12,7 +12,6 @@ import ( hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" - "golang.org/x/oauth2" "net/http" "strconv" "strings" @@ -24,7 +23,6 @@ type PikPak struct { *Common RefreshToken string AccessToken string - oauth2Token oauth2.TokenSource } func (d *PikPak) Config() driver.Config { @@ -84,41 +82,16 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.Addition.DeviceID = d.Common.DeviceID op.MustSaveDriverStorage(d) } - // 初始化 oauth2Config - oauth2Config := &oauth2.Config{ - ClientID: d.ClientID, - ClientSecret: d.ClientSecret, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://user.mypikpak.net/v1/auth/signin", - TokenURL: "https://user.mypikpak.net/v1/auth/token", - AuthStyle: oauth2.AuthStyleInParams, - }, - } - // 如果已经有RefreshToken，直接获取AccessToken if d.Addition.RefreshToken != "" { - if d.RefreshTokenMethod == "oauth2" { - // 使用 oauth2 刷新令牌 - // 初始化 oauth2Token - d.initializeOAuth2Token(ctx, oauth2Config, d.Addition.RefreshToken) - if err := d.refreshTokenByOAuth2(); err != nil { - return err - } - } else { - if err := d.refreshToken(d.Addition.RefreshToken); err != nil { - return err - } + if err = d.refreshToken(d.Addition.RefreshToken); err != nil { + return err } - } else { // 如果没有填写RefreshToken，尝试登录 获取 refreshToken - if err := d.login(); err != nil { + if err = d.login(); err != nil { return err } - if d.RefreshTokenMethod == "oauth2" { - d.initializeOAuth2Token(ctx, oauth2Config, d.RefreshToken) - } - } // 获取CaptchaToken diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index 7e525787814..5abbc8796ca 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -7,14 +7,13 @@ import ( type Addition struct { driver.RootID - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - Platform string `json:"platform" required:"true" default:"web" type:"select" options:"android,web,pc"` - RefreshToken string `json:"refresh_token" required:"true" default:""` - RefreshTokenMethod string `json:"refresh_token_method" required:"true" type:"select" options:"oauth2,http"` - CaptchaToken string `json:"captcha_token" default:""` - DeviceID string `json:"device_id" required:"false" default:""` - DisableMediaLink bool `json:"disable_media_link" default:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + Platform string `json:"platform" required:"true" default:"web" type:"select" options:"android,web,pc"` + RefreshToken string `json:"refresh_token" required:"true" default:""` + CaptchaToken string `json:"captcha_token" default:""` + DeviceID string `json:"device_id" required:"false" default:""` + DisableMediaLink bool `json:"disable_media_link" default:"true"` } var config = driver.Config{ diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 67077fb861e..e8f3c854533 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -2,7 +2,6 @@ package pikpak import ( "bytes" - "context" "crypto/md5" "crypto/sha1" "encoding/hex" @@ -14,7 +13,6 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" - "golang.org/x/oauth2" "io" "net/http" "path/filepath" @@ -27,8 +25,6 @@ import ( "github.com/go-resty/resty/v2" ) -// do others that not defined in Driver interface - var AndroidAlgorithms = []string{ "7xOq4Z8s", "QE9/9+IQco", @@ -171,30 +167,6 @@ func (d *PikPak) refreshToken(refreshToken string) error { return nil } -func (d *PikPak) initializeOAuth2Token(ctx context.Context, oauth2Config *oauth2.Config, refreshToken string) { - d.oauth2Token = oauth2.ReuseTokenSource(nil, utils.TokenSource(func() (*oauth2.Token, error) { - return oauth2Config.TokenSource(ctx, &oauth2.Token{ - RefreshToken: refreshToken, - }).Token() - })) -} - -func (d *PikPak) refreshTokenByOAuth2() error { - token, err := d.oauth2Token.Token() - if err != nil { - return err - } - d.Status = "work" - d.RefreshToken = token.RefreshToken - d.AccessToken = token.AccessToken - // 获取用户ID - userID := token.Extra("sub").(string) - d.Common.SetUserID(userID) - d.Addition.RefreshToken = d.RefreshToken - op.MustSaveDriverStorage(d) - return nil -} - func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ @@ -203,14 +175,7 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r "X-Device-ID": d.GetDeviceID(), "X-Captcha-Token": d.GetCaptchaToken(), }) - if d.RefreshTokenMethod == "oauth2" && d.oauth2Token != nil { - // 使用oauth2 获取 access_token - token, err := d.oauth2Token.Token() - if err != nil { - return nil, err - } - req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) - } else if d.AccessToken != "" { + if d.AccessToken != "" { req.SetHeader("Authorization", "Bearer "+d.AccessToken) } @@ -232,16 +197,9 @@ func (d *PikPak) request(url string, method string, callback base.ReqCallback, r return res.Body(), nil case 4122, 4121, 16: // access_token 过期 - if d.RefreshTokenMethod == "oauth2" { - if err1 := d.refreshTokenByOAuth2(); err1 != nil { - return nil, err1 - } - } else { - if err1 := d.refreshToken(d.RefreshToken); err1 != nil { - return nil, err1 - } + if err1 := d.refreshToken(d.RefreshToken); err1 != nil { + return nil, err1 } - return d.request(url, method, callback, resp) case 9: // 验证码token过期 if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.GetUserID()); err != nil { From 5084d98398144e6ccd9cc627ada5cb54d7dbb526 Mon Sep 17 00:00:00 2001 From: shingyu Date: Sun, 8 Dec 2024 17:06:33 +0800 Subject: [PATCH 371/659] fix(onedrive): fix timeout error (#7551 close #7506) --- drivers/onedrive/util.go | 2 +- drivers/onedrive_app/util.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 9ee2dae9cae..95f92db6433 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -127,7 +127,7 @@ func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, func (d *Onedrive) getFiles(path string) ([]File, error) { var res []File - nextLink := d.GetMetaUrl(false, path) + "/children?$top=5000&$expand=thumbnails($select=medium)&$select=id,name,size,fileSystemInfo,content.downloadUrl,file,parentReference" + nextLink := d.GetMetaUrl(false, path) + "/children?$top=1000&$expand=thumbnails($select=medium)&$select=id,name,size,fileSystemInfo,content.downloadUrl,file,parentReference" for nextLink != "" { var files Files _, err := d.Request(nextLink, http.MethodGet, nil, &files) diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 28b34837806..d036e131757 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -118,7 +118,7 @@ func (d *OnedriveAPP) Request(url string, method string, callback base.ReqCallba func (d *OnedriveAPP) getFiles(path string) ([]File, error) { var res []File - nextLink := d.GetMetaUrl(false, path) + "/children?$top=5000&$expand=thumbnails($select=medium)&$select=id,name,size,lastModifiedDateTime,content.downloadUrl,file,parentReference" + nextLink := d.GetMetaUrl(false, path) + "/children?$top=1000&$expand=thumbnails($select=medium)&$select=id,name,size,lastModifiedDateTime,content.downloadUrl,file,parentReference" for nextLink != "" { var files Files _, err := d.Request(nextLink, http.MethodGet, nil, &files) From aa45a829146aef36dbebd564f155ccfcf115fba2 Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Mon, 9 Dec 2024 23:33:07 +0800 Subject: [PATCH 372/659] fix(115): fix login bug (#7626 close #7614 close #7620) --- drivers/115/util.go | 5 ++--- drivers/115_share/utils.go | 2 +- go.mod | 4 ++-- go.sum | 11 ++++++----- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/drivers/115/util.go b/drivers/115/util.go index 33e345706d2..d7a1adff71c 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -27,8 +27,7 @@ import ( "github.com/pkg/errors" ) -//var UserAgent = driver115.UA115Browser - +// var UserAgent = driver115.UA115Browser func (d *Pan115) login() error { var err error opts := []driver115.Option{ @@ -46,7 +45,7 @@ func (d *Pan115) login() error { if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { return errors.Wrap(err, "failed to login by qrcode") } - d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s", cr.UID, cr.CID, cr.SEID) + d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s;KID=%s", cr.UID, cr.CID, cr.SEID, cr.KID) d.QRCodeToken = "" } else if d.Cookie != "" { if err = cr.FromCookie(d.Cookie); err != nil { diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 812352ef42a..1f9e112deef 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -96,7 +96,7 @@ func (d *Pan115Share) login() error { if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { return errors.Wrap(err, "failed to login by qrcode") } - d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s", cr.UID, cr.CID, cr.SEID) + d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s;KID=%s", cr.UID, cr.CID, cr.SEID, cr.KID) d.QRCodeToken = "" } else if d.Cookie != "" { if err = cr.FromCookie(d.Cookie); err != nil { diff --git a/go.mod b/go.mod index 19bc7c2e627..be63182399b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/alist-org/alist/v3 go 1.22.4 require ( - github.com/SheltonZhu/115driver v1.0.29 + github.com/SheltonZhu/115driver v1.0.32 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 @@ -64,7 +64,7 @@ require ( golang.org/x/image v0.19.0 golang.org/x/net v0.28.0 golang.org/x/oauth2 v0.22.0 - golang.org/x/time v0.6.0 + golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 gorm.io/driver/mysql v1.5.7 diff --git a/go.sum b/go.sum index 78ac273a5bf..f1ff39b3157 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= -github.com/SheltonZhu/115driver v1.0.29 h1:yFBqFDYJyADo3eG2RjJgSovnFd1OrpGHmsHBi6j0+r4= -github.com/SheltonZhu/115driver v1.0.29/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= +github.com/SheltonZhu/115driver v1.0.32 h1:Taw1bnfcPJZW0xTdhDvEbBS1tccif7J7DslRp2NkDyQ= +github.com/SheltonZhu/115driver v1.0.32/go.mod h1:XXFi23pyhAgzUE8dUEKdGvIdUQKi3wv6zR7C1Do40D8= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= @@ -393,6 +393,8 @@ github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= +github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= @@ -512,8 +514,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= -github.com/xhofe/tache v0.1.2 h1:pHrXlrWcbTb4G7hVUDW7Rc+YTUnLJvnLBrdktVE1Fqg= -github.com/xhofe/tache v0.1.2/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/tache v0.1.3 h1:MipxzlljYX29E1YI/SLC7hVomVF+51iP1OUzlsuq1wE= github.com/xhofe/tache v0.1.3/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= @@ -663,8 +663,9 @@ golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 088120df8253623f937fe76cfdf4b471929a4f3b Mon Sep 17 00:00:00 2001 From: Joseph Chris Date: Mon, 9 Dec 2024 07:33:46 -0800 Subject: [PATCH 373/659] feat(sso): add custom extra scope support (#7577) --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 1 + server/handles/ssologin.go | 9 +++++++-- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 920a7a2d118..f1b98a70deb 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -164,6 +164,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SSOApplicationName, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOEndpointName, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOJwtPublicKey, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, + {Key: conf.SSOExtraScopes, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOAutoRegister, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 13787b5e2ac..499e0a4f0c6 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -72,6 +72,7 @@ const ( SSOApplicationName = "sso_application_name" SSOEndpointName = "sso_endpoint_name" SSOJwtPublicKey = "sso_jwt_public_key" + SSOExtraScopes = "sso_extra_scopes" SSOAutoRegister = "sso_auto_register" SSODefaultDir = "sso_default_dir" SSODefaultPermission = "sso_default_permission" diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index cb5fc4ca6c4..62bd4aaa2bf 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -4,13 +4,14 @@ import ( "encoding/base64" "errors" "fmt" - "github.com/Xhofe/go-cache" "net/http" "net/url" "path" "strings" "time" + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" @@ -123,6 +124,10 @@ func GetOIDCClient(c *gin.Context, useCompatibility bool, redirectUri, method st } clientId := setting.GetStr(conf.SSOClientId) clientSecret := setting.GetStr(conf.SSOClientSecret) + extraScopes := []string{} + if setting.GetStr(conf.SSOExtraScopes) != "" { + extraScopes = strings.Split(setting.GetStr(conf.SSOExtraScopes), " ") + } return &oauth2.Config{ ClientID: clientId, ClientSecret: clientSecret, @@ -132,7 +137,7 @@ func GetOIDCClient(c *gin.Context, useCompatibility bool, redirectUri, method st Endpoint: provider.Endpoint(), // "openid" is a required scope for OpenID Connect flows. - Scopes: []string{oidc.ScopeOpenID, "profile"}, + Scopes: append([]string{oidc.ScopeOpenID, "profile"}, extraScopes...), }, nil } From 016e169c41dcc1ce255c86d3c391526080356305 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 9 Dec 2024 23:34:29 +0800 Subject: [PATCH 374/659] feat(139): support multipart upload (close: #7444) (#7630) * feat(139): support multipart upload (close: #7444) * feat(139): add custom upload part size option --- drivers/139/driver.go | 135 +++++++++++++++++++++++++++++++----------- drivers/139/meta.go | 5 +- drivers/139/types.go | 19 ++++++ 3 files changed, 122 insertions(+), 37 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index d33c3d77ebf..2fedc477730 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -357,7 +357,10 @@ const ( TB ) -func getPartSize(size int64) int64 { +func (d *Yun139) getPartSize(size int64) int64 { + if d.CustomUploadPartSize != 0 { + return d.CustomUploadPartSize + } // 网盘对于分片数量存在上限 if size/GB > 30 { return 512 * MB @@ -380,24 +383,51 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } } - // return errs.NotImplement + + partInfos := []PartInfo{} + var partSize = d.getPartSize(stream.GetSize()) + part := (stream.GetSize() + partSize - 1) / partSize + if part == 0 { + part = 1 + } + for i := int64(0); i < part; i++ { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + start := i * partSize + byteSize := stream.GetSize() - start + if byteSize > partSize { + byteSize = partSize + } + partNumber := i + 1 + partInfo := PartInfo{ + PartNumber: partNumber, + PartSize: byteSize, + ParallelHashCtx: ParallelHashCtx{ + PartOffset: start, + }, + } + partInfos = append(partInfos, partInfo) + } + + // 筛选出前 100 个 partInfos + firstPartInfos := partInfos + if len(firstPartInfos) > 100 { + firstPartInfos = firstPartInfos[:100] + } + + // 获取上传信息和前100个分片的上传地址 data := base.Json{ "contentHash": fullHash, "contentHashAlgorithm": "SHA256", "contentType": "application/octet-stream", "parallelUpload": false, - "partInfos": []base.Json{{ - "parallelHashCtx": base.Json{ - "partOffset": 0, - }, - "partNumber": 1, - "partSize": stream.GetSize(), - }}, - "size": stream.GetSize(), - "parentFileId": dstDir.GetID(), - "name": stream.GetName(), - "type": "file", - "fileRenameMode": "auto_rename", + "partInfos": firstPartInfos, + "size": stream.GetSize(), + "parentFileId": dstDir.GetID(), + "name": stream.GetName(), + "type": "file", + "fileRenameMode": "auto_rename", } pathname := "/hcy/file/create" var resp PersonalUploadResp @@ -410,32 +440,67 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return nil } + uploadPartInfos := resp.Data.PartInfos + + // 获取后续分片的上传地址 + for i := 101; i < len(partInfos); i += 100 { + end := i + 100 + if end > len(partInfos) { + end = len(partInfos) + } + batchPartInfos := partInfos[i:end] + + moredata := base.Json{ + "fileId": resp.Data.FileId, + "uploadId": resp.Data.UploadId, + "partInfos": batchPartInfos, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname := "/hcy/file/getUploadUrl" + var moreresp PersonalUploadUrlResp + _, err = d.personalPost(pathname, moredata, &moreresp) + if err != nil { + return err + } + uploadPartInfos = append(uploadPartInfos, moreresp.Data.PartInfos...) + } + // Progress p := driver.NewProgress(stream.GetSize(), up) - // Update Progress - r := io.TeeReader(stream, p) + // 上传所有分片 + for _, uploadPartInfo := range uploadPartInfos { + index := uploadPartInfo.PartNumber - 1 + partSize := partInfos[index].PartSize + log.Debugf("[139] uploading part %+v/%+v", index, len(uploadPartInfos)) + limitReader := io.LimitReader(stream, partSize) - req, err := http.NewRequest("PUT", resp.Data.PartInfos[0].UploadUrl, r) - if err != nil { - return err - } - req = req.WithContext(ctx) - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("Content-Length", fmt.Sprint(stream.GetSize())) - req.Header.Set("Origin", "https://yun.139.com") - req.Header.Set("Referer", "https://yun.139.com/") - req.ContentLength = stream.GetSize() + // Update Progress + r := io.TeeReader(limitReader, p) - res, err := base.HttpClient.Do(req) - if err != nil { - return err - } + req, err := http.NewRequest("PUT", uploadPartInfo.UploadUrl, r) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprint(partSize)) + req.Header.Set("Origin", "https://yun.139.com") + req.Header.Set("Referer", "https://yun.139.com/") + req.ContentLength = partSize - _ = res.Body.Close() - log.Debugf("%+v", res) - if res.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", res.StatusCode) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + _ = res.Body.Close() + log.Debugf("[139] uploaded: %+v", res) + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", res.StatusCode) + } } data = base.Json{ @@ -496,7 +561,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr // Progress p := driver.NewProgress(stream.GetSize(), up) - var partSize = getPartSize(stream.GetSize()) + var partSize = d.getPartSize(stream.GetSize()) part := (stream.GetSize() + partSize - 1) / partSize if part == 0 { part = 1 diff --git a/drivers/139/meta.go b/drivers/139/meta.go index 56a4c1df96b..680e469ded9 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -9,8 +9,9 @@ type Addition struct { //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` driver.RootID - Type string `json:"type" type:"select" options:"personal,family,personal_new" default:"personal"` - CloudID string `json:"cloud_id"` + Type string `json:"type" type:"select" options:"personal,family,personal_new" default:"personal"` + CloudID string `json:"cloud_id"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` } var config = driver.Config{ diff --git a/drivers/139/types.go b/drivers/139/types.go index f797196624b..42b939bf69d 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -196,6 +196,16 @@ type QueryContentListResp struct { } `json:"data"` } +type ParallelHashCtx struct { + PartOffset int64 `json:"partOffset"` +} + +type PartInfo struct { + PartNumber int64 `json:"partNumber"` + PartSize int64 `json:"partSize"` + ParallelHashCtx ParallelHashCtx `json:"parallelHashCtx"` +} + type PersonalThumbnail struct { Style string `json:"style"` Url string `json:"url"` @@ -235,6 +245,15 @@ type PersonalUploadResp struct { } } +type PersonalUploadUrlResp struct { + BaseResp + Data struct { + FileId string `json:"fileId"` + UploadId string `json:"uploadId"` + PartInfos []PersonalPartInfo `json:"partInfos"` + } +} + type RefreshTokenResp struct { XMLName xml.Name `xml:"root"` Return string `xml:"return"` From 2a035302b2e73c87f57945d66300c9a10f1c8127 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 9 Dec 2024 23:35:44 +0800 Subject: [PATCH 375/659] fix(cloudreve): support upload to remote and OneDrive storage (#7632 close #6882) - Add support for remote and OneDrive storage types - Implement new upload methods for different storage types - Update driver to handle various storage policies - Add error handling and session cleanup for failed uploads --- drivers/cloudreve/driver.go | 73 +++++++++++++++++---------- drivers/cloudreve/types.go | 8 +-- drivers/cloudreve/util.go | 99 +++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 30 deletions(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index ec0f6ef2b29..8fc117aca2c 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -10,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -134,6 +135,8 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File if io.ReadCloser(stream) == http.NoBody { return d.create(ctx, dstDir, stream) } + + // 获取存储策略 var r DirectoryResp err := d.request(http.MethodGet, "/directory"+dstDir.GetPath(), nil, &r) if err != nil { @@ -146,6 +149,8 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File "policy_id": r.Policy.Id, "last_modified": stream.ModTime().Unix(), } + + // 获取上传会话信息 var u UploadInfo err = d.request(http.MethodPut, "/file/upload", func(req *resty.Request) { req.SetBody(uploadBody) @@ -153,36 +158,50 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File if err != nil { return err } - var chunkSize = u.ChunkSize - var buf []byte - var chunk int - for { - var n int - buf = make([]byte, chunkSize) - n, err = io.ReadAtLeast(stream, buf, chunkSize) - if err != nil && err != io.ErrUnexpectedEOF { - if err == io.EOF { - return nil - } - return err - } - if n == 0 { - break - } - buf = buf[:n] - err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { - req.SetHeader("Content-Type", "application/octet-stream") - req.SetHeader("Content-Length", strconv.Itoa(n)) - req.SetBody(buf) - }, nil) - if err != nil { - break + // 根据存储方式选择分片上传的方法 + switch r.Policy.Type { + case "onedrive": + err = d.upOneDrive(ctx, stream, u, up) + case "remote": // 从机存储 + err = d.upRemote(ctx, stream, u, up) + case "local": // 本机存储 + var chunkSize = u.ChunkSize + var buf []byte + var chunk int + for { + var n int + buf = make([]byte, chunkSize) + n, err = io.ReadAtLeast(stream, buf, chunkSize) + if err != nil && err != io.ErrUnexpectedEOF { + if err == io.EOF { + return nil + } + return err + } + if n == 0 { + break + } + buf = buf[:n] + err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { + req.SetHeader("Content-Type", "application/octet-stream") + req.SetHeader("Content-Length", strconv.Itoa(n)) + req.SetBody(buf) + }, nil) + if err != nil { + break + } + chunk++ } - chunk++ - + default: + err = errs.NotImplement } - return err + if err != nil { + // 删除失败的会话 + err = d.request(http.MethodDelete, "/file/upload/"+u.SessionID, nil, nil) + return err + } + return nil } func (d *Cloudreve) create(ctx context.Context, dir model.Obj, file model.Obj) error { diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index 241d993ebb8..a7c3919e8a9 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -21,9 +21,11 @@ type Policy struct { } type UploadInfo struct { - SessionID string `json:"sessionID"` - ChunkSize int `json:"chunkSize"` - Expires int `json:"expires"` + SessionID string `json:"sessionID"` + ChunkSize int `json:"chunkSize"` + Expires int `json:"expires"` + UploadURLs []string `json:"uploadURLs"` + Credential string `json:"credential,omitempty"` } type DirectoryResp struct { diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 284e3289dee..b5b71153e12 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -1,16 +1,23 @@ package cloudreve import ( + "bytes" + "context" "encoding/base64" "errors" + "fmt" + "io" "net/http" + "strconv" "strings" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/cookie" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" json "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go" @@ -172,3 +179,95 @@ func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { Thumbnail: resp.Header().Get("Location"), }, nil } + +func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { + uploadUrl := u.UploadURLs[0] + credential := u.Credential + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + for finish < stream.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + utils.Log.Debugf("[Cloudreve-Remote] upload: %d", finish) + var byteSize = DEFAULT + left := stream.GetSize() - finish + if left < DEFAULT { + byteSize = left + } + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), bytes.NewBuffer(byteData)) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.Header.Set("Authorization", fmt.Sprint(credential)) + finish += byteSize + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + res.Body.Close() + up(float64(finish) * 100 / float64(stream.GetSize())) + chunk++ + } + return nil +} + +func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { + uploadUrl := u.UploadURLs[0] + var finish int64 = 0 + DEFAULT := int64(u.ChunkSize) + for finish < stream.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + utils.Log.Debugf("[Cloudreve-OneDrive] upload: %d", finish) + var byteSize = DEFAULT + left := stream.GetSize() - finish + if left < DEFAULT { + byteSize = left + } + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(byteData)) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) + finish += byteSize + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession + if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { + data, _ := io.ReadAll(res.Body) + res.Body.Close() + return errors.New(string(data)) + } + res.Body.Close() + up(float64(finish) * 100 / float64(stream.GetSize())) + } + // 上传成功发送回调请求 + err := d.request(http.MethodPost, "/callback/onedrive/finish/"+u.SessionID, func(req *resty.Request) { + req.SetBody("{}") + }, nil) + if err != nil { + return err + } + return nil +} From a3908fd9a650e93bada0c715af371fc7a7fed33d Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 9 Dec 2024 23:54:21 +0800 Subject: [PATCH 376/659] fix(139): update APIs (#7591 close #7603) * fix(139): update family cloud API * fix(139): update API of familyGetLink * feat(139): support group (close #7603) * docs: add `139 group` to Readme * feat(139): support multipart upload (close: #7444) * feat(139): add custom upload part size option * fix: missing right big quote --------- Co-authored-by: Andy Hsu --- README.md | 2 +- README_cn.md | 2 +- README_ja.md | 2 +- drivers/139/driver.go | 177 +++++++++++++++++++++++++++++++++++++----- drivers/139/types.go | 23 ++++++ drivers/139/util.go | 88 ++++++++++++++++++++- 6 files changed, 268 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 701bbc2ff3b..8140f325a9b 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) - [x] [Mediatrack](https://www.mediatrack.cn/) - - [x] [139yun](https://yun.139.com/) (Personal, Family) + - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) - [x] [Terabox](https://www.terabox.com/main) diff --git a/README_cn.md b/README_cn.md index 7e45d60f757..5c71ccce4c3 100644 --- a/README_cn.md +++ b/README_cn.md @@ -58,7 +58,7 @@ - [x] WebDav(支持无API的OneDrive/SharePoint) - [x] Teambition（[中国](https://www.teambition.com/ )，[国际](https://us.teambition.com/ )） - [x] [分秒帧](https://www.mediatrack.cn/) - - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云) + - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云，共享群组) - [x] [Yandex.Disk](https://disk.yandex.com/) - [x] [百度网盘](http://pan.baidu.com/) - [x] [UC网盘](https://drive.uc.cn) diff --git a/README_ja.md b/README_ja.md index 453e7b9966b..cd4446fab8e 100644 --- a/README_ja.md +++ b/README_ja.md @@ -58,7 +58,7 @@ - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) - [x] [Mediatrack](https://www.mediatrack.cn/) - - [x] [139yun](https://yun.139.com/) (Personal, Family) + - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) - [x] [Terabox](https://www.terabox.com/main) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 2fedc477730..8862983ce5e 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "path" "strconv" "strings" "time" @@ -14,15 +15,16 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" log "github.com/sirupsen/logrus" ) type Yun139 struct { model.Storage Addition - cron *cron.Cron + cron *cron.Cron Account string } @@ -56,6 +58,11 @@ func (d *Yun139) Init(ctx context.Context) error { d.RootFolderID = "root" } fallthrough + case MetaGroup: + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = d.CloudID + } + fallthrough case MetaFamily: decode, err := base64.StdEncoding.DecodeString(d.Authorization) if err != nil { @@ -96,6 +103,8 @@ func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( return d.getFiles(dir.GetID()) case MetaFamily: return d.familyGetFiles(dir.GetID()) + case MetaGroup: + return d.groupGetFiles(dir.GetID()) default: return nil, errs.NotImplement } @@ -108,9 +117,11 @@ func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) case MetaPersonalNew: url, err = d.personalGetLink(file.GetID()) case MetaPersonal: - fallthrough - case MetaFamily: url, err = d.getLink(file.GetID()) + case MetaFamily: + url, err = d.familyGetLink(file.GetID(), file.GetPath()) + case MetaGroup: + url, err = d.groupGetLink(file.GetID(), file.GetPath()) default: return nil, errs.NotImplement } @@ -154,8 +165,22 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin "accountType": 1, }, "docLibName": dirName, + "path": path.Join(parentDir.GetPath(), parentDir.GetID()), + } + pathname := "/orchestration/familyCloud-rebuild/cloudCatalog/v1.0/createCloudDoc" + _, err = d.post(pathname, data, nil) + case MetaGroup: + data := base.Json{ + "catalogName": dirName, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + "groupID": d.CloudID, + "parentFileId": parentDir.GetID(), + "path": path.Join(parentDir.GetPath(), parentDir.GetID()), } - pathname := "/orchestration/familyCloud/cloudCatalog/v1.0/createCloudDoc" + pathname := "/orchestration/group-rebuild/catalog/v1.0/createGroupCatalog" _, err = d.post(pathname, data, nil) default: err = errs.NotImplement @@ -176,6 +201,34 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, return nil, err } return srcObj, nil + case MetaGroup: + var contentList []string + var catalogList []string + if srcObj.IsDir() { + catalogList = append(catalogList, srcObj.GetID()) + } else { + contentList = append(contentList, srcObj.GetID()) + } + data := base.Json{ + "taskType": 3, + "srcType": 2, + "srcGroupID": d.CloudID, + "destType": 2, + "destGroupID": d.CloudID, + "destPath": dstDir.GetPath(), + "contentList": contentList, + "catalogList": catalogList, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname := "/orchestration/group-rebuild/task/v1.0/createBatchOprTask" + _, err := d.post(pathname, data, nil) + if err != nil { + return nil, err + } + return srcObj, nil case MetaPersonal: var contentInfoList []string var catalogInfoList []string @@ -246,6 +299,65 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e pathname = "/orchestration/personalCloud/content/v1.0/updateContentInfo" } _, err = d.post(pathname, data, nil) + case MetaGroup: + var data base.Json + var pathname string + if srcObj.IsDir() { + data = base.Json{ + "groupID": d.CloudID, + "modifyCatalogID": srcObj.GetID(), + "modifyCatalogName": newName, + "path": srcObj.GetPath(), + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname = "/orchestration/group-rebuild/catalog/v1.0/modifyGroupCatalog" + } else { + data = base.Json{ + "groupID": d.CloudID, + "contentID": srcObj.GetID(), + "contentName": newName, + "path": srcObj.GetPath(), + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname = "/orchestration/group-rebuild/content/v1.0/modifyGroupContent" + } + _, err = d.post(pathname, data, nil) + case MetaFamily: + var data base.Json + var pathname string + if srcObj.IsDir() { + // 网页接口不支持重命名家庭云文件夹 + // data = base.Json{ + // "catalogType": 3, + // "catalogID": srcObj.GetID(), + // "catalogName": newName, + // "commonAccountInfo": base.Json{ + // "account": d.Account, + // "accountType": 1, + // }, + // "path": srcObj.GetPath(), + // } + // pathname = "/orchestration/familyCloud-rebuild/photoContent/v1.0/modifyCatalogInfo" + return errs.NotImplement + } else { + data = base.Json{ + "contentID": srcObj.GetID(), + "contentName": newName, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + "path": srcObj.GetPath(), + } + pathname = "/orchestration/familyCloud-rebuild/photoContent/v1.0/modifyContentInfo" + } + _, err = d.post(pathname, data, nil) default: err = errs.NotImplement } @@ -303,6 +415,28 @@ func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { pathname := "/hcy/recyclebin/batchTrash" _, err := d.personalPost(pathname, data, nil) return err + case MetaGroup: + var contentList []string + var catalogList []string + // 必须使用完整路径删除 + if obj.IsDir() { + catalogList = append(catalogList, obj.GetPath()) + } else { + contentList = append(contentList, path.Join(obj.GetPath(), obj.GetID())) + } + data := base.Json{ + "taskType": 2, + "srcGroupID": d.CloudID, + "contentList": contentList, + "catalogList": catalogList, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname := "/orchestration/group-rebuild/task/v1.0/createBatchOprTask" + _, err := d.post(pathname, data, nil) + return err case MetaPersonal: fallthrough case MetaFamily: @@ -337,10 +471,12 @@ func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { "account": d.Account, "accountType": 1, }, + "sourceCloudID": d.CloudID, "sourceCatalogType": 1002, "taskType": 2, + "path": obj.GetPath(), } - pathname = "/orchestration/familyCloud/batchOprTask/v1.0/createBatchOprTask" + pathname = "/orchestration/familyCloud-rebuild/batchOprTask/v1.0/createBatchOprTask" } _, err := d.post(pathname, data, nil) return err @@ -536,21 +672,20 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" if d.isFamily() { - // data = d.newJson(base.Json{ - // "fileCount": 1, - // "manualRename": 2, - // "operation": 0, - // "path": "", - // "seqNo": "", - // "totalSize": 0, - // "uploadContentList": []base.Json{{ - // "contentName": stream.GetName(), - // "contentSize": 0, - // // "digest": "5a3231986ce7a6b46e408612d385bafa" - // }}, - // }) - // pathname = "/orchestration/familyCloud/content/v1.0/getFileUploadURL" - return errs.NotImplement + data = d.newJson(base.Json{ + "fileCount": 1, + "manualRename": 2, + "operation": 0, + "path": path.Join(dstDir.GetPath(), dstDir.GetID()), + "seqNo": random.String(32), //序列号不能为空 + "totalSize": 0, + "uploadContentList": []base.Json{{ + "contentName": stream.GetName(), + "contentSize": 0, + // "digest": "5a3231986ce7a6b46e408612d385bafa" + }}, + }) + pathname = "/orchestration/familyCloud-rebuild/content/v1.0/getFileUploadURL" } var resp UploadResp _, err := d.post(pathname, data, &resp) diff --git a/drivers/139/types.go b/drivers/139/types.go index 42b939bf69d..c34cba0388b 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -7,6 +7,7 @@ import ( const ( MetaPersonal string = "personal" MetaFamily string = "family" + MetaGroup string = "group" MetaPersonalNew string = "personal_new" ) @@ -54,6 +55,7 @@ type Content struct { //ContentDesc string `json:"contentDesc"` //ContentType int `json:"contentType"` //ContentOrigin int `json:"contentOrigin"` + CreateTime string `json:"createTime"` UpdateTime string `json:"updateTime"` //CommentCount int `json:"commentCount"` ThumbnailURL string `json:"thumbnailURL"` @@ -196,6 +198,27 @@ type QueryContentListResp struct { } `json:"data"` } +type QueryGroupContentListResp struct { + BaseResp + Data struct { + Result struct { + ResultCode string `json:"resultCode"` + ResultDesc string `json:"resultDesc"` + } `json:"result"` + GetGroupContentResult struct { + ParentCatalogID string `json:"parentCatalogID"` // 根目录是"0" + CatalogList []struct { + Catalog + Path string `json:"path"` + } `json:"catalogList"` + ContentList []Content `json:"contentList"` + NodeCount int `json:"nodeCount"` // 文件+文件夹数量 + CtlgCnt int `json:"ctlgCnt"` // 文件夹数量 + ContCnt int `json:"contCnt"` // 文件数量 + } `json:"getGroupContentResult"` + } `json:"data"` +} + type ParallelHashCtx struct { PartOffset int64 `json:"partOffset"` } diff --git a/drivers/139/util.go b/drivers/139/util.go index 5918e4c5305..ccb6a912f32 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -13,9 +13,9 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" - "github.com/alist-org/alist/v3/internal/op" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" @@ -220,10 +220,11 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { "sortDirection": 1, }) var resp QueryContentListResp - _, err := d.post("/orchestration/familyCloud/content/v1.0/queryContentList", data, &resp) + _, err := d.post("/orchestration/familyCloud-rebuild/content/v1.2/queryContentList", data, &resp) if err != nil { return nil, err } + path := resp.Data.Path for _, catalog := range resp.Data.CloudCatalogList { f := model.Object{ ID: catalog.CatalogID, @@ -232,6 +233,7 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { IsFolder: true, Modified: getTime(catalog.LastUpdateTime), Ctime: getTime(catalog.CreateTime), + Path: path, // 文件夹上一级的Path } files = append(files, &f) } @@ -243,6 +245,7 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { Size: content.ContentSize, Modified: getTime(content.LastUpdateTime), Ctime: getTime(content.CreateTime), + Path: path, // 文件所在目录的Path }, Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL}, //Thumbnail: content.BigthumbnailURL, @@ -257,6 +260,61 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { return files, nil } +func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) { + pageNum := 1 + files := make([]model.Obj, 0) + for { + data := d.newJson(base.Json{ + "groupID": d.CloudID, + "catalogID": catalogID, + "contentSortType": 0, + "sortDirection": 1, + "startNumber": pageNum, + "endNumber": pageNum + 99, + "path": catalogID, + }) + + var resp QueryGroupContentListResp + _, err := d.post("/orchestration/group-rebuild/content/v1.0/queryGroupContentList", data, &resp) + if err != nil { + return nil, err + } + path := resp.Data.GetGroupContentResult.ParentCatalogID + for _, catalog := range resp.Data.GetGroupContentResult.CatalogList { + f := model.Object{ + ID: catalog.CatalogID, + Name: catalog.CatalogName, + Size: 0, + IsFolder: true, + Modified: getTime(catalog.UpdateTime), + Ctime: getTime(catalog.CreateTime), + Path: catalog.Path, // 文件夹的真实Path， root:/开头 + } + files = append(files, &f) + } + for _, content := range resp.Data.GetGroupContentResult.ContentList { + f := model.ObjThumb{ + Object: model.Object{ + ID: content.ContentID, + Name: content.ContentName, + Size: content.ContentSize, + Modified: getTime(content.UpdateTime), + Ctime: getTime(content.CreateTime), + Path: path, // 文件所在目录的Path + }, + Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL}, + //Thumbnail: content.BigthumbnailURL, + } + files = append(files, &f) + } + if pageNum > resp.Data.GetGroupContentResult.NodeCount { + break + } + pageNum = pageNum + 100 + } + return files, nil +} + func (d *Yun139) getLink(contentId string) (string, error) { data := base.Json{ "appName": "", @@ -273,6 +331,32 @@ func (d *Yun139) getLink(contentId string) (string, error) { } return jsoniter.Get(res, "data", "downloadURL").ToString(), nil } +func (d *Yun139) familyGetLink(contentId string, path string) (string, error) { + data := d.newJson(base.Json{ + "contentID": contentId, + "path": path, + }) + res, err := d.post("/orchestration/familyCloud-rebuild/content/v1.0/getFileDownLoadURL", + data, nil) + if err != nil { + return "", err + } + return jsoniter.Get(res, "data", "downloadURL").ToString(), nil +} + +func (d *Yun139) groupGetLink(contentId string, path string) (string, error) { + data := d.newJson(base.Json{ + "contentID": contentId, + "groupID": d.CloudID, + "path": path, + }) + res, err := d.post("/orchestration/group-rebuild/groupManage/v1.0/getGroupFileDownLoadURL", + data, nil) + if err != nil { + return "", err + } + return jsoniter.Get(res, "data", "downloadURL").ToString(), nil +} func unicode(str string) string { textQuoted := strconv.QuoteToASCII(str) From 734184649991a7319bc857a258e4e2e65e941ec6 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Tue, 10 Dec 2024 19:30:50 +0800 Subject: [PATCH 377/659] perf(task): merge requests of operating selected (#7637) --- server/handles/task.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/server/handles/task.go b/server/handles/task.go index 71b4c622144..5f9965053b9 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -90,6 +90,31 @@ func getTargetedHandler[T task.TaskInfoWithCreator](manager *tache.Manager[T], c } } +func getBatchHandler[T task.TaskInfoWithCreator](manager *tache.Manager[T], callback func(task T)) gin.HandlerFunc { + return func(c *gin.Context) { + isAdmin, uid, ok := getUserInfo(c) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + var tids []string + if err := c.ShouldBind(&tids); err != nil { + common.ErrorStrResp(c, "invalid request format", 400) + return + } + retErrs := make(map[string]string) + for _, tid := range tids { + t, ok := manager.GetByID(tid) + if !ok || (!isAdmin && uid != t.GetCreator().ID) { + retErrs[tid] = "task not found" + continue + } + callback(t) + } + common.SuccessResp(c, retErrs) + } +} + func taskRoute[T task.TaskInfoWithCreator](g *gin.RouterGroup, manager *tache.Manager[T]) { g.GET("/undone", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) @@ -132,6 +157,15 @@ func taskRoute[T task.TaskInfoWithCreator](g *gin.RouterGroup, manager *tache.Ma manager.Retry(task.GetID()) common.SuccessResp(c) })) + g.POST("/cancel_some", getBatchHandler(manager, func(task T) { + manager.Cancel(task.GetID()) + })) + g.POST("/delete_some", getBatchHandler(manager, func(task T) { + manager.Remove(task.GetID()) + })) + g.POST("/retry_some", getBatchHandler(manager, func(task T) { + manager.Retry(task.GetID()) + })) g.POST("/clear_done", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { From 650b03aeb1feea8b51ebc6b4b48ffb885fdc41d6 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Tue, 10 Dec 2024 20:17:46 +0800 Subject: [PATCH 378/659] feat: ftp server support (#7634 close #1898) * feat: ftp server support * fix(ftp): incorrect mode for dirs in LIST returns --- cmd/server.go | 29 +++ go.mod | 7 +- go.sum | 17 ++ internal/bootstrap/data/setting.go | 10 + internal/conf/config.go | 26 +++ internal/conf/const.go | 9 + internal/model/setting.go | 1 + internal/model/user.go | 8 + server/ftp.go | 285 +++++++++++++++++++++++++++++ server/ftp/afero.go | 91 +++++++++ server/ftp/fsmanage.go | 75 ++++++++ server/ftp/fsread.go | 188 +++++++++++++++++++ server/ftp/fsup.go | 91 +++++++++ 13 files changed, 835 insertions(+), 2 deletions(-) create mode 100644 server/ftp.go create mode 100644 server/ftp/afero.go create mode 100644 server/ftp/fsmanage.go create mode 100644 server/ftp/fsread.go create mode 100644 server/ftp/fsup.go diff --git a/cmd/server.go b/cmd/server.go index 8a7beafa7fd..66b57952b49 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" "net" "net/http" "os" @@ -112,6 +113,24 @@ the address is defined in config file`, } }() } + var ftpDriver *server.FtpMainDriver + var ftpServer *ftpserver.FtpServer + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable { + var err error + ftpDriver, err = server.NewMainDriver() + if err != nil { + utils.Log.Fatalf("failed to start ftp driver: %s", err.Error()) + } else { + utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen) + go func() { + ftpServer = ftpserver.NewFtpServer(ftpDriver) + err = ftpServer.ListenAndServe() + if err != nil { + utils.Log.Fatalf("problem ftp server listening: %s", err.Error()) + } + }() + } + } // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -152,6 +171,16 @@ the address is defined in config file`, } }() } + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil && ftpDriver != nil { + wg.Add(1) + go func() { + defer wg.Done() + ftpDriver.Stop() + if err := ftpServer.Stop(); err != nil { + utils.Log.Fatal("FTP server shutdown err: ", err) + } + }() + } wg.Wait() utils.Log.Println("Server exit") }, diff --git a/go.mod b/go.mod index be63182399b..259521e9f7b 100644 --- a/go.mod +++ b/go.mod @@ -50,8 +50,9 @@ require ( github.com/pquerna/otp v1.4.0 github.com/rclone/rclone v1.67.0 github.com/sirupsen/logrus v1.9.3 + github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.9.0 + github.com/stretchr/testify v1.10.0 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 @@ -75,6 +76,7 @@ require ( require ( github.com/BurntSushi/toml v0.3.1 // indirect + github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2 // indirect github.com/blevesearch/go-faiss v1.0.20 // indirect github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -83,6 +85,7 @@ require ( github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fclairamb/go-log v0.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect @@ -221,7 +224,7 @@ require ( go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect + golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/tools v0.24.0 // indirect diff --git a/go.sum b/go.sum index f1ff39b3157..dcad05c9a64 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,11 @@ +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2 h1:P3MoQ1kDfbCjL6+MPd5K7wPdKB4nqMuLU6Mv0+tdWDA= +github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= @@ -144,6 +147,8 @@ github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJL github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= +github.com/fclairamb/go-log v0.5.0/go.mod h1:XoRO1dYezpsGmLLkZE9I+sHqpqY65p8JA+Vqblb7k40= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= @@ -168,6 +173,10 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -441,6 +450,8 @@ github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= @@ -459,6 +470,8 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:s github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -481,6 +494,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= @@ -634,6 +649,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index f1b98a70deb..206273b41ac 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -185,6 +185,16 @@ func InitialSettings() []model.SettingItem { {Key: conf.S3AccessKeyId, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3SecretAccessKey, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3Buckets, Value: "[]", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, + + //ftp settings + {Key: conf.FTPPublicHost, Value: "127.0.0.1", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPPasvPortMap, Value: "", Type: conf.TypeText, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPProxyUserAgent, Value: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + + "Chrome/87.0.4280.88 Safari/537.36", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPMandatoryTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPImplicitTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPTLSPrivateKeyPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPTLSPublicCertPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, } initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { diff --git a/internal/conf/config.go b/internal/conf/config.go index aa29e1f506d..df6c0544e1e 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -71,6 +71,19 @@ type S3 struct { SSL bool `json:"ssl" env:"SSL"` } +type FTP struct { + Enable bool `json:"enable" env:"ENABLE"` + Listen string `json:"listen" env:"LISTEN"` + FindPasvPortAttempts int `json:"find_pasv_port_attempts" env:"FIND_PASV_PORT_ATTEMPTS"` + ActiveTransferPortNon20 bool `json:"active_transfer_port_non_20" env:"ACTIVE_TRANSFER_PORT_NON_20"` + IdleTimeout int `json:"idle_timeout" env:"IDLE_TIMEOUT"` + ConnectionTimeout int `json:"connection_timeout" env:"CONNECTION_TIMEOUT"` + DisableActiveMode bool `json:"disable_active_mode" env:"DISABLE_ACTIVE_MODE"` + DefaultTransferBinary bool `json:"default_transfer_binary" env:"DEFAULT_TRANSFER_BINARY"` + EnableActiveConnIPCheck bool `json:"enable_active_conn_ip_check" env:"ENABLE_ACTIVE_CONN_IP_CHECK"` + EnablePasvConnIPCheck bool `json:"enable_pasv_conn_ip_check" env:"ENABLE_PASV_CONN_IP_CHECK"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -90,6 +103,7 @@ type Config struct { Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"` Cors Cors `json:"cors" envPrefix:"CORS_"` S3 S3 `json:"s3" envPrefix:"S3_"` + FTP FTP `json:"ftp" envPrefix:"FTP_"` } func DefaultConfig() *Config { @@ -159,5 +173,17 @@ func DefaultConfig() *Config { Port: 5246, SSL: false, }, + FTP: FTP{ + Enable: true, + Listen: ":5221", + FindPasvPortAttempts: 50, + ActiveTransferPortNon20: false, + IdleTimeout: 900, + ConnectionTimeout: 30, + DisableActiveMode: false, + DefaultTransferBinary: false, + EnableActiveConnIPCheck: true, + EnablePasvConnIPCheck: true, + }, } } diff --git a/internal/conf/const.go b/internal/conf/const.go index 499e0a4f0c6..99e8c868931 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -97,6 +97,15 @@ const ( // qbittorrent QbittorrentUrl = "qbittorrent_url" QbittorrentSeedtime = "qbittorrent_seedtime" + + // ftp + FTPPublicHost = "ftp_public_host" + FTPPasvPortMap = "ftp_pasv_port_map" + FTPProxyUserAgent = "ftp_proxy_user_agent" + FTPMandatoryTLS = "ftp_mandatory_tls" + FTPImplicitTLS = "ftp_implicit_tls" + FTPTLSPrivateKeyPath = "ftp_tls_private_key_path" + FTPTLSPublicCertPath = "ftp_tls_public_cert_path" ) const ( diff --git a/internal/model/setting.go b/internal/model/setting.go index c474935ed49..9b60d98a76e 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -11,6 +11,7 @@ const ( SSO LDAP S3 + FTP ) const ( diff --git a/internal/model/user.go b/internal/model/user.go index 2d61a971c3d..b4e876a47ab 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -117,6 +117,14 @@ func (u *User) CanWebdavManage() bool { return u.IsAdmin() || (u.Permission>>9)&1 == 1 } +func (u *User) CanFTPAccess() bool { + return (u.Permission>>10)&1 == 1 +} + +func (u *User) CanFTPManage() bool { + return (u.Permission>>11)&1 == 1 +} + func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } diff --git a/server/ftp.go b/server/ftp.go new file mode 100644 index 00000000000..161ea63c5b1 --- /dev/null +++ b/server/ftp.go @@ -0,0 +1,285 @@ +package server + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/ftp" + "math/rand" + "net" + "net/http" + "os" + "strconv" + "strings" + "sync" +) + +type FtpMainDriver struct { + settings *ftpserver.Settings + proxyHeader *http.Header + clients map[uint32]ftpserver.ClientContext + shutdownLock sync.RWMutex + isShutdown bool + tlsConfig *tls.Config +} + +func NewMainDriver() (*FtpMainDriver, error) { + header := &http.Header{} + header.Add("User-Agent", setting.GetStr(conf.FTPProxyUserAgent)) + transferType := ftpserver.TransferTypeASCII + if conf.Conf.FTP.DefaultTransferBinary { + transferType = ftpserver.TransferTypeBinary + } + activeConnCheck := ftpserver.IPMatchDisabled + if conf.Conf.FTP.EnableActiveConnIPCheck { + activeConnCheck = ftpserver.IPMatchRequired + } + pasvConnCheck := ftpserver.IPMatchDisabled + if conf.Conf.FTP.EnablePasvConnIPCheck { + pasvConnCheck = ftpserver.IPMatchRequired + } + tlsRequired := ftpserver.ClearOrEncrypted + if setting.GetBool(conf.FTPImplicitTLS) { + tlsRequired = ftpserver.ImplicitEncryption + } else if setting.GetBool(conf.FTPMandatoryTLS) { + tlsRequired = ftpserver.MandatoryEncryption + } + tlsConf, err := getTlsConf(setting.GetStr(conf.FTPTLSPrivateKeyPath), setting.GetStr(conf.FTPTLSPublicCertPath)) + if err != nil && tlsRequired != ftpserver.ClearOrEncrypted { + return nil, fmt.Errorf("FTP mandatory TLS has been enabled, but the certificate failed to load: %w", err) + } + return &FtpMainDriver{ + settings: &ftpserver.Settings{ + ListenAddr: conf.Conf.FTP.Listen, + PublicHost: lookupIP(setting.GetStr(conf.FTPPublicHost)), + PassiveTransferPortGetter: newPortMapper(setting.GetStr(conf.FTPPasvPortMap)), + FindPasvPortAttempts: conf.Conf.FTP.FindPasvPortAttempts, + ActiveTransferPortNon20: conf.Conf.FTP.ActiveTransferPortNon20, + IdleTimeout: conf.Conf.FTP.IdleTimeout, + ConnectionTimeout: conf.Conf.FTP.ConnectionTimeout, + DisableMLSD: false, + DisableMLST: false, + DisableMFMT: true, + Banner: setting.GetStr(conf.Announcement), + TLSRequired: tlsRequired, + DisableLISTArgs: false, + DisableSite: true, + DisableActiveMode: conf.Conf.FTP.DisableActiveMode, + EnableHASH: false, + DisableSTAT: false, + DisableSYST: false, + EnableCOMB: false, + DefaultTransferType: transferType, + ActiveConnectionsCheck: activeConnCheck, + PasvConnectionsCheck: pasvConnCheck, + }, + proxyHeader: header, + clients: make(map[uint32]ftpserver.ClientContext), + shutdownLock: sync.RWMutex{}, + isShutdown: false, + tlsConfig: tlsConf, + }, nil +} + +func (d *FtpMainDriver) GetSettings() (*ftpserver.Settings, error) { + return d.settings, nil +} + +func (d *FtpMainDriver) ClientConnected(cc ftpserver.ClientContext) (string, error) { + if d.isShutdown || !d.shutdownLock.TryRLock() { + return "", errors.New("server has shutdown") + } + defer d.shutdownLock.RUnlock() + d.clients[cc.ID()] = cc + return "AList FTP Endpoint", nil +} + +func (d *FtpMainDriver) ClientDisconnected(cc ftpserver.ClientContext) { + err := cc.Close() + if err != nil { + utils.Log.Errorf("failed to close client: %v", err) + } + delete(d.clients, cc.ID()) +} + +func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) { + var userObj *model.User + var err error + if user == "anonymous" || user == "guest" { + userObj, err = op.GetGuest() + if err != nil { + return nil, err + } + } else { + userObj, err = op.GetUserByName(user) + if err != nil { + return nil, err + } + passHash := model.StaticHash(pass) + if err = userObj.ValidatePwdStaticHash(passHash); err != nil { + return nil, err + } + } + if userObj.Disabled || !userObj.CanFTPAccess() { + return nil, errors.New("user not allowed to access FTP") + } + + ctx := context.Background() + ctx = context.WithValue(ctx, "user", userObj) + if user == "anonymous" || user == "guest" { + ctx = context.WithValue(ctx, "meta_pass", pass) + } else { + ctx = context.WithValue(ctx, "meta_pass", "") + } + ctx = context.WithValue(ctx, "client_ip", cc.RemoteAddr().String()) + ctx = context.WithValue(ctx, "proxy_header", d.proxyHeader) + return ftp.NewAferoAdapter(ctx), nil +} + +func (d *FtpMainDriver) GetTLSConfig() (*tls.Config, error) { + if d.tlsConfig == nil { + return nil, errors.New("TLS config not provided") + } + return d.tlsConfig, nil +} + +func (d *FtpMainDriver) Stop() { + d.isShutdown = true + d.shutdownLock.Lock() + defer d.shutdownLock.Unlock() + for _, value := range d.clients { + _ = value.Close() + } +} + +func lookupIP(host string) string { + if host == "" || net.ParseIP(host) != nil { + return host + } + ips, err := net.LookupIP(host) + if err != nil || len(ips) == 0 { + utils.Log.Fatalf("given FTP public host is invalid, and the default value will be used: %v", err) + return "" + } + for _, ip := range ips { + if ip.To4() != nil { + return ip.String() + } + } + v6 := ips[0].String() + utils.Log.Warnf("no IPv4 record looked up, %s will be used as public host, and it might do not work.", v6) + return v6 +} + +func newPortMapper(str string) ftpserver.PasvPortGetter { + if str == "" { + return nil + } + pasvPortMappers := strings.Split(strings.Replace(str, "\n", ",", -1), ",") + type group struct { + ExposedStart int + ListenedStart int + Length int + } + groups := make([]group, len(pasvPortMappers)) + totalLength := 0 + convertToPorts := func(str string) (int, int, error) { + start, end, multi := strings.Cut(str, "-") + if multi { + si, err := strconv.Atoi(start) + if err != nil { + return 0, 0, err + } + ei, err := strconv.Atoi(end) + if err != nil { + return 0, 0, err + } + if ei < si || ei < 1024 || si < 1024 || ei > 65535 || si > 65535 { + return 0, 0, errors.New("invalid port") + } + return si, ei - si + 1, nil + } else { + ret, err := strconv.Atoi(str) + if err != nil { + return 0, 0, err + } else { + return ret, 1, nil + } + } + } + for i, mapper := range pasvPortMappers { + var err error + exposed, listened, mapped := strings.Cut(mapper, ":") + for { + if mapped { + var es, ls, el, ll int + es, el, err = convertToPorts(exposed) + if err != nil { + break + } + ls, ll, err = convertToPorts(listened) + if err != nil { + break + } + if el != ll { + err = errors.New("the number of exposed ports and listened ports does not match") + break + } + groups[i].ExposedStart = es + groups[i].ListenedStart = ls + groups[i].Length = el + totalLength += el + } else { + var start, length int + start, length, err = convertToPorts(mapper) + groups[i].ExposedStart = start + groups[i].ListenedStart = start + groups[i].Length = length + totalLength += length + } + break + } + if err != nil { + utils.Log.Fatalf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err) + return nil + } + } + return func() (int, int, bool) { + idxPort := rand.Intn(totalLength) + for _, g := range groups { + if idxPort >= g.Length { + idxPort -= g.Length + } else { + return g.ExposedStart + idxPort, g.ListenedStart + idxPort, true + } + } + // unreachable + return 0, 0, false + } +} + +func getTlsConf(keyPath, certPath string) (*tls.Config, error) { + if keyPath == "" || certPath == "" { + return nil, errors.New("private key or certificate is not provided") + } + cert, err := os.ReadFile(certPath) + if err != nil { + return nil, err + } + key, err := os.ReadFile(keyPath) + if err != nil { + return nil, err + } + tlsCert, err := tls.X509KeyPair(cert, key) + if err != nil { + return nil, err + } + return &tls.Config{Certificates: []tls.Certificate{tlsCert}}, nil +} diff --git a/server/ftp/afero.go b/server/ftp/afero.go new file mode 100644 index 00000000000..6eb4bf8e4e4 --- /dev/null +++ b/server/ftp/afero.go @@ -0,0 +1,91 @@ +package ftp + +import ( + "context" + "errors" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/spf13/afero" + "os" + "time" +) + +type AferoAdapter struct { + ctx context.Context +} + +func NewAferoAdapter(ctx context.Context) *AferoAdapter { + return &AferoAdapter{ctx: ctx} +} + +func (a *AferoAdapter) Create(_ string) (afero.File, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (a *AferoAdapter) Mkdir(name string, _ os.FileMode) error { + return Mkdir(a.ctx, name) +} + +func (a *AferoAdapter) MkdirAll(path string, perm os.FileMode) error { + return a.Mkdir(path, perm) +} + +func (a *AferoAdapter) Open(_ string) (afero.File, error) { + // See also GetHandle and ReadDir + return nil, errs.NotImplement +} + +func (a *AferoAdapter) OpenFile(_ string, _ int, _ os.FileMode) (afero.File, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (a *AferoAdapter) Remove(name string) error { + return Remove(a.ctx, name) +} + +func (a *AferoAdapter) RemoveAll(path string) error { + return a.Remove(path) +} + +func (a *AferoAdapter) Rename(oldName, newName string) error { + return Rename(a.ctx, oldName, newName) +} + +func (a *AferoAdapter) Stat(name string) (os.FileInfo, error) { + return Stat(a.ctx, name) +} + +func (a *AferoAdapter) Name() string { + return "AList FTP Endpoint" +} + +func (a *AferoAdapter) Chmod(_ string, _ os.FileMode) error { + return errs.NotSupport +} + +func (a *AferoAdapter) Chown(_ string, _, _ int) error { + return errs.NotSupport +} + +func (a *AferoAdapter) Chtimes(_ string, _ time.Time, _ time.Time) error { + return errs.NotSupport +} + +func (a *AferoAdapter) ReadDir(name string) ([]os.FileInfo, error) { + return List(a.ctx, name) +} + +func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) { + if offset != 0 { + return nil, errors.New("offset") + } + if (flags & os.O_APPEND) > 0 { + return nil, errors.New("append") + } + if (flags & os.O_WRONLY) > 0 { + return OpenUpload(a.ctx, name) + } + return OpenDownload(a.ctx, name) +} diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go new file mode 100644 index 00000000000..5199a473b5b --- /dev/null +++ b/server/ftp/fsmanage.go @@ -0,0 +1,75 @@ +package ftp + +import ( + "context" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + stdpath "path" +) + +func Mkdir(ctx context.Context, path string) error { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return err + } + if !user.CanWrite() || !user.CanFTPManage() { + meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + } + if !common.CanWrite(meta, reqPath) { + return errs.PermissionDenied + } + } + return fs.MakeDir(ctx, reqPath) +} + +func Remove(ctx context.Context, path string) error { + user := ctx.Value("user").(*model.User) + if !user.CanRemove() || !user.CanFTPManage() { + return errs.PermissionDenied + } + reqPath, err := user.JoinPath(path) + if err != nil { + return err + } + return fs.Remove(ctx, reqPath) +} + +func Rename(ctx context.Context, oldPath, newPath string) error { + user := ctx.Value("user").(*model.User) + srcPath, err := user.JoinPath(oldPath) + if err != nil { + return err + } + dstPath, err := user.JoinPath(newPath) + if err != nil { + return err + } + srcDir, srcBase := stdpath.Split(srcPath) + dstDir, dstBase := stdpath.Split(dstPath) + if srcDir == dstDir { + if !user.CanRename() || !user.CanFTPManage() { + return errs.PermissionDenied + } + return fs.Rename(ctx, srcPath, dstBase) + } else { + if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { + return errs.PermissionDenied + } + if err := fs.Move(ctx, srcPath, dstDir); err != nil { + return err + } + if srcBase != dstBase { + return fs.Rename(ctx, stdpath.Join(dstDir, srcBase), dstBase) + } + return nil + } +} diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go new file mode 100644 index 00000000000..6a9ba2ebb2a --- /dev/null +++ b/server/ftp/fsread.go @@ -0,0 +1,188 @@ +package ftp + +import ( + "context" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/net" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + "io" + fs2 "io/fs" + "net/http" + "os" + "time" +) + +type FileDownloadProxy struct { + ftpserver.FileTransfer + reader io.ReadCloser + closers *utils.Closers +} + +func OpenDownload(ctx context.Context, path string) (*FileDownloadProxy, error) { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + + // directly use proxy + header := *(ctx.Value("proxy_header").(*http.Header)) + link, obj, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: ctx.Value("client_ip").(string), + Header: header, + }) + if err != nil { + return nil, err + } + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + return nil, err + } + if storage.GetStorage().ProxyRange { + common.ProxyRange(link, obj.GetSize()) + } + reader, closers, err := proxy(link) + if err != nil { + return nil, err + } + return &FileDownloadProxy{reader: reader, closers: closers}, nil +} + +func proxy(link *model.Link) (io.ReadCloser, *utils.Closers, error) { + if link.MFile != nil { + return link.MFile, nil, nil + } else if link.RangeReadCloser != nil { + rc, err := link.RangeReadCloser.RangeRead(context.Background(), http_range.Range{Length: -1}) + if err != nil { + return nil, nil, err + } + closers := link.RangeReadCloser.GetClosers() + return rc, &closers, nil + } else { + res, err := net.RequestHttp(context.Background(), http.MethodGet, link.Header, link.URL) + if err != nil { + return nil, nil, err + } + return res.Body, nil, nil + } +} + +func (f *FileDownloadProxy) Read(p []byte) (n int, err error) { + return f.reader.Read(p) +} + +func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { + return 0, errs.NotSupport +} + +func (f *FileDownloadProxy) Seek(offset int64, whence int) (int64, error) { + return 0, errs.NotSupport +} + +func (f *FileDownloadProxy) Close() error { + defer func() { + if f.closers != nil { + _ = f.closers.Close() + } + }() + return f.reader.Close() +} + +type OsFileInfoAdapter struct { + obj model.Obj +} + +func (o *OsFileInfoAdapter) Name() string { + return o.obj.GetName() +} + +func (o *OsFileInfoAdapter) Size() int64 { + return o.obj.GetSize() +} + +func (o *OsFileInfoAdapter) Mode() fs2.FileMode { + var mode fs2.FileMode = 0755 + if o.IsDir() { + mode |= fs2.ModeDir + } + return mode +} + +func (o *OsFileInfoAdapter) ModTime() time.Time { + return o.obj.ModTime() +} + +func (o *OsFileInfoAdapter) IsDir() bool { + return o.obj.IsDir() +} + +func (o *OsFileInfoAdapter) Sys() any { + return o.obj +} + +func Stat(ctx context.Context, path string) (os.FileInfo, error) { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + if err != nil { + return nil, err + } + return &OsFileInfoAdapter{obj: obj}, nil +} + +func List(ctx context.Context, path string) ([]os.FileInfo, error) { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{}) + if err != nil { + return nil, err + } + ret := make([]os.FileInfo, len(objs)) + for i, obj := range objs { + ret[i] = &OsFileInfoAdapter{obj: obj} + } + return ret, nil +} diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go new file mode 100644 index 00000000000..3042a3d2cea --- /dev/null +++ b/server/ftp/fsup.go @@ -0,0 +1,91 @@ +package ftp + +import ( + "context" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + "io" + "net/http" + "os" + stdpath "path" + "time" +) + +type FileUploadProxy struct { + ftpserver.FileTransfer + buffer *os.File + path string + ctx context.Context +} + +func OpenUpload(ctx context.Context, path string) (*FileUploadProxy, error) { + user := ctx.Value("user").(*model.User) + path, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(stdpath.Dir(path)) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + if !(common.CanAccess(user, meta, path, ctx.Value("meta_pass").(string)) && + ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { + return nil, errs.PermissionDenied + } + tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + return &FileUploadProxy{buffer: tmpFile, path: path, ctx: ctx}, nil +} + +func (f *FileUploadProxy) Read(p []byte) (n int, err error) { + return 0, errs.NotSupport +} + +func (f *FileUploadProxy) Write(p []byte) (n int, err error) { + return f.buffer.Write(p) +} + +func (f *FileUploadProxy) Seek(offset int64, whence int) (int64, error) { + return 0, errs.NotSupport +} + +func (f *FileUploadProxy) Close() error { + dir, name := stdpath.Split(f.path) + size, err := f.buffer.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { + return err + } + arr := make([]byte, 512) + if _, err := f.buffer.Read(arr); err != nil { + return err + } + contentType := http.DetectContentType(arr) + if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { + return err + } + s := &stream.FileStream{ + Obj: &model.Object{ + Name: name, + Size: size, + Modified: time.Now(), + }, + Mimetype: contentType, + WebPutAsTask: false, + } + s.SetTmpFile(f.buffer) + return fs.PutDirectly(f.ctx, dir, s, true) +} From ecefa5e0eb61bff2df66cfcbdbf97a14c73cb19f Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 10 Dec 2024 20:21:51 +0800 Subject: [PATCH 379/659] ci: fix desktop beta release trigger --- .github/workflows/beta_release.yml | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 90c2836fd2d..c9cb7475780 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -111,14 +111,23 @@ jobs: name: Beta Release Desktop runs-on: ubuntu-latest steps: - - uses: peter-evans/create-or-update-comment@v4 + - name: Checkout repo + uses: actions/checkout@v4 + with: + repository: alist-org/desktop-release + ref: main + persist-credentials: false + fetch-depth: 0 + + - name: Commit + run: | + git config --local user.email "bot@nn.ci" + git config --local user.name "IlaBot" + git commit --allow-empty -m "Trigger build for ${{ github.sha }}" + + - name: Push commit + uses: ad-m/github-push-action@master with: - issue-number: 69 - body: | - /release-beta - - triggered by @${{ github.actor }} - - commit sha: ${{ github.sha }} - - view files: https://github.com/alist-org/alist/tree/${{ github.sha }} - reactions: 'rocket' - token: ${{ secrets.MY_TOKEN }} + github_token: ${{ secrets.MY_TOKEN }} + branch: main repository: alist-org/desktop-release \ No newline at end of file From 201e25c17fa00e5b8ea1989eebc63a0c7efdefbc Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Thu, 12 Dec 2024 20:50:00 +0800 Subject: [PATCH 380/659] fix(ftp-server): large transfer leads to client timeout (#7639) * fix(ftp-server): client timeout to wait a large file upload to netdisk * fix(ftp-server): driver alist v3 upload failed and temp files do not be deleted --- server/ftp/fsup.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index 3042a3d2cea..356522712d5 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -87,5 +87,7 @@ func (f *FileUploadProxy) Close() error { WebPutAsTask: false, } s.SetTmpFile(f.buffer) - return fs.PutDirectly(f.ctx, dir, s, true) + s.Closers.Add(f.buffer) + _, err = fs.PutAsTask(f.ctx, dir, s) + return err } From 33ba7f152198c11a1ab0bb2eed192e637c835d02 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Thu, 12 Dec 2024 20:51:43 +0800 Subject: [PATCH 381/659] feat: sftp server support (#7643) * feat: sftp server support * fix(sftp-server): try fix build failed * fix: sftp download lack --- cmd/common.go | 1 + cmd/server.go | 28 +++++++++ go.mod | 12 ++-- go.sum | 27 ++++---- internal/bootstrap/ssh.go | 101 ++++++++++++++++++++++++++++++ internal/conf/config.go | 10 +++ internal/conf/var.go | 3 + server/ftp.go | 7 ++- server/ftp/afero.go | 36 +++++++++-- server/ftp/const.go | 11 ++++ server/ftp/fsup.go | 128 ++++++++++++++++++++++++++++++++++++-- server/ftp/sftp.go | 122 ++++++++++++++++++++++++++++++++++++ server/ftp/site.go | 21 +++++++ server/sftp.go | 109 ++++++++++++++++++++++++++++++++ 14 files changed, 584 insertions(+), 32 deletions(-) create mode 100644 internal/bootstrap/ssh.go create mode 100644 server/ftp/const.go create mode 100644 server/ftp/sftp.go create mode 100644 server/ftp/site.go create mode 100644 server/sftp.go diff --git a/cmd/common.go b/cmd/common.go index b4a7081c33f..fabc3a90f1b 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -15,6 +15,7 @@ import ( func Init() { bootstrap.InitConfig() bootstrap.Log() + bootstrap.InitHostKey() bootstrap.InitDB() data.InitData() bootstrap.InitIndex() diff --git a/cmd/server.go b/cmd/server.go index 66b57952b49..3112a6a9905 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/KirCute/sftpd-alist" "net" "net/http" "os" @@ -131,6 +132,24 @@ the address is defined in config file`, }() } } + var sftpDriver *server.SftpDriver + var sftpServer *sftpd.SftpServer + if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable { + var err error + sftpDriver, err = server.NewSftpDriver() + if err != nil { + utils.Log.Fatalf("failed to start sftp driver: %s", err.Error()) + } else { + utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen) + go func() { + sftpServer = sftpd.NewSftpServer(sftpDriver) + err = sftpServer.RunServer() + if err != nil { + utils.Log.Fatalf("problem sftp server listening: %s", err.Error()) + } + }() + } + } // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -181,6 +200,15 @@ the address is defined in config file`, } }() } + if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil && sftpDriver != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := sftpServer.Close(); err != nil { + utils.Log.Fatal("SFTP server shutdown err: ", err) + } + }() + } wg.Wait() utils.Log.Println("Server exit") }, diff --git a/go.mod b/go.mod index 259521e9f7b..1deaa1d5565 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/alist-org/alist/v3 go 1.22.4 require ( + github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 + github.com/KirCute/sftpd-alist v0.0.11 github.com/SheltonZhu/115driver v1.0.32 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 @@ -60,7 +62,7 @@ require ( github.com/xhofe/tache v0.1.3 github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.30.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 golang.org/x/net v0.28.0 @@ -76,7 +78,6 @@ require ( require ( github.com/BurntSushi/toml v0.3.1 // indirect - github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2 // indirect github.com/blevesearch/go-faiss v1.0.20 // indirect github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect @@ -90,6 +91,7 @@ require ( github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect ) require ( @@ -223,10 +225,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.8.0 // indirect + golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.24.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.24.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect diff --git a/go.sum b/go.sum index dcad05c9a64..a4e8e12d7b4 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,10 @@ cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2Qx cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2 h1:P3MoQ1kDfbCjL6+MPd5K7wPdKB4nqMuLU6Mv0+tdWDA= -github.com/KirCute/ftpserverlib-pasvportmap v0.0.0-20241208190057-c9a7bf2571e2/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= +github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 h1:ikwCzeqoqN6wvBHOB9OI6dde/jbV7EoTMpUcxtYl5Po= +github.com/KirCute/ftpserverlib-pasvportmap v1.25.0/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= +github.com/KirCute/sftpd-alist v0.0.11 h1:BGInXmmLBI+v6S9WZCwvY0DRK1vDprGNcTv/57p2GSo= +github.com/KirCute/sftpd-alist v0.0.11/go.mod h1:pPFzr6GrKqXvFXLr46ZpoqmtSpwH8DKTYloSp/ybzKQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= @@ -492,12 +494,13 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= +github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= +github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= @@ -571,8 +574,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= -golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= +golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -614,8 +617,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -647,8 +650,6 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= @@ -661,8 +662,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= -golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -676,8 +677,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= -golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= diff --git a/internal/bootstrap/ssh.go b/internal/bootstrap/ssh.go new file mode 100644 index 00000000000..ec4a07ac6e3 --- /dev/null +++ b/internal/bootstrap/ssh.go @@ -0,0 +1,101 @@ +package bootstrap + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "github.com/alist-org/alist/v3/cmd/flags" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/pkg/utils" + "golang.org/x/crypto/ssh" + "os" + "path/filepath" +) + +func InitHostKey() { + sshPath := filepath.Join(flags.DataDir, "ssh") + if !utils.Exists(sshPath) { + err := utils.CreateNestedDirectory(sshPath) + if err != nil { + utils.Log.Fatalf("failed to create ssh directory: %+v", err) + return + } + } + conf.SSHSigners = make([]ssh.Signer, 0, 4) + if rsaKey, ok := LoadOrGenerateRSAHostKey(sshPath); ok { + conf.SSHSigners = append(conf.SSHSigners, rsaKey) + } + // TODO Add keys for other encryption algorithms +} + +func LoadOrGenerateRSAHostKey(parentDir string) (ssh.Signer, bool) { + privateKeyPath := filepath.Join(parentDir, "ssh_host_rsa_key") + publicKeyPath := filepath.Join(parentDir, "ssh_host_rsa_key.pub") + privateKeyBytes, err := os.ReadFile(privateKeyPath) + if err == nil { + var privateKey *rsa.PrivateKey + privateKey, err = rsaDecodePrivateKey(privateKeyBytes) + if err == nil { + var ret ssh.Signer + ret, err = ssh.NewSignerFromKey(privateKey) + if err == nil { + return ret, true + } + } + } + _ = os.Remove(privateKeyPath) + _ = os.Remove(publicKeyPath) + privateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + utils.Log.Fatalf("failed to generate RSA private key: %+v", err) + return nil, false + } + publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + utils.Log.Fatalf("failed to generate RSA public key: %+v", err) + return nil, false + } + ret, err := ssh.NewSignerFromKey(privateKey) + if err != nil { + utils.Log.Fatalf("failed to generate RSA signer: %+v", err) + return nil, false + } + privateBytes := rsaEncodePrivateKey(privateKey) + publicBytes := ssh.MarshalAuthorizedKey(publicKey) + err = os.WriteFile(privateKeyPath, privateBytes, 0600) + if err != nil { + utils.Log.Fatalf("failed to write RSA private key to file: %+v", err) + return nil, false + } + err = os.WriteFile(publicKeyPath, publicBytes, 0644) + if err != nil { + _ = os.Remove(privateKeyPath) + utils.Log.Fatalf("failed to write RSA public key to file: %+v", err) + return nil, false + } + return ret, true +} + +func rsaEncodePrivateKey(privateKey *rsa.PrivateKey) []byte { + privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey) + privateBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: privateKeyBytes, + } + return pem.EncodeToMemory(privateBlock) +} + +func rsaDecodePrivateKey(bytes []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(bytes) + if block == nil { + return nil, fmt.Errorf("failed to parse PEM block containing the key") + } + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, err + } + return privateKey, nil +} diff --git a/internal/conf/config.go b/internal/conf/config.go index df6c0544e1e..6c0ccb2adc2 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -84,6 +84,11 @@ type FTP struct { EnablePasvConnIPCheck bool `json:"enable_pasv_conn_ip_check" env:"ENABLE_PASV_CONN_IP_CHECK"` } +type SFTP struct { + Enable bool `json:"enable" env:"ENABLE"` + Listen string `json:"listen" env:"LISTEN"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -104,6 +109,7 @@ type Config struct { Cors Cors `json:"cors" envPrefix:"CORS_"` S3 S3 `json:"s3" envPrefix:"S3_"` FTP FTP `json:"ftp" envPrefix:"FTP_"` + SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` } func DefaultConfig() *Config { @@ -185,5 +191,9 @@ func DefaultConfig() *Config { EnableActiveConnIPCheck: true, EnablePasvConnIPCheck: true, }, + SFTP: SFTP{ + Enable: true, + Listen: ":5222", + }, } } diff --git a/internal/conf/var.go b/internal/conf/var.go index 0a8eb16fcd1..b7277e4112c 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -1,6 +1,7 @@ package conf import ( + "golang.org/x/crypto/ssh" "net/url" "regexp" ) @@ -32,3 +33,5 @@ var ( ManageHtml string IndexHtml string ) + +var SSHSigners []ssh.Signer diff --git a/server/ftp.go b/server/ftp.go index 161ea63c5b1..4d507b684b4 100644 --- a/server/ftp.go +++ b/server/ftp.go @@ -70,7 +70,7 @@ func NewMainDriver() (*FtpMainDriver, error) { Banner: setting.GetStr(conf.Announcement), TLSRequired: tlsRequired, DisableLISTArgs: false, - DisableSite: true, + DisableSite: false, DisableActiveMode: conf.Conf.FTP.DisableActiveMode, EnableHASH: false, DisableSTAT: false, @@ -79,6 +79,9 @@ func NewMainDriver() (*FtpMainDriver, error) { DefaultTransferType: transferType, ActiveConnectionsCheck: activeConnCheck, PasvConnectionsCheck: pasvConnCheck, + SiteHandlers: map[string]ftpserver.SiteHandler{ + "SIZE": ftp.HandleSIZE, + }, }, proxyHeader: header, clients: make(map[uint32]ftpserver.ClientContext), @@ -128,7 +131,7 @@ func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) } } if userObj.Disabled || !userObj.CanFTPAccess() { - return nil, errors.New("user not allowed to access FTP") + return nil, errors.New("user is not allowed to access via FTP") } ctx := context.Background() diff --git a/server/ftp/afero.go b/server/ftp/afero.go index 6eb4bf8e4e4..866ad8c0231 100644 --- a/server/ftp/afero.go +++ b/server/ftp/afero.go @@ -5,13 +5,15 @@ import ( "errors" ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" "github.com/spf13/afero" "os" "time" ) type AferoAdapter struct { - ctx context.Context + ctx context.Context + nextFileSize int64 } func NewAferoAdapter(ctx context.Context) *AferoAdapter { @@ -78,14 +80,36 @@ func (a *AferoAdapter) ReadDir(name string) ([]os.FileInfo, error) { } func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) { + fileSize := a.nextFileSize + a.nextFileSize = 0 if offset != 0 { - return nil, errors.New("offset") + return nil, errs.NotSupport } - if (flags & os.O_APPEND) > 0 { - return nil, errors.New("append") + if (flags & os.O_SYNC) != 0 { + return nil, errs.NotSupport } - if (flags & os.O_WRONLY) > 0 { - return OpenUpload(a.ctx, name) + if (flags & os.O_APPEND) != 0 { + return nil, errs.NotSupport + } + _, err := fs.Get(a.ctx, name, &fs.GetArgs{}) + exists := err == nil + if (flags&os.O_CREATE) == 0 && !exists { + return nil, errs.ObjectNotFound + } + if (flags&os.O_EXCL) != 0 && exists { + return nil, errors.New("file already exists") + } + if (flags & os.O_WRONLY) != 0 { + trunc := (flags & os.O_TRUNC) != 0 + if fileSize > 0 { + return OpenUploadWithLength(a.ctx, name, trunc, fileSize) + } else { + return OpenUpload(a.ctx, name, trunc) + } } return OpenDownload(a.ctx, name) } + +func (a *AferoAdapter) SetNextFileSize(size int64) { + a.nextFileSize = size +} diff --git a/server/ftp/const.go b/server/ftp/const.go new file mode 100644 index 00000000000..1fd14e82d97 --- /dev/null +++ b/server/ftp/const.go @@ -0,0 +1,11 @@ +package ftp + +// From leffss/sftpd +const ( + SSH_FXF_READ = 0x00000001 + SSH_FXF_WRITE = 0x00000002 + SSH_FXF_APPEND = 0x00000004 + SSH_FXF_CREAT = 0x00000008 + SSH_FXF_TRUNC = 0x00000010 + SSH_FXF_EXCL = 0x00000020 +) diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index 356522712d5..f18c13c24b6 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -1,6 +1,7 @@ package ftp import ( + "bytes" "context" ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" "github.com/alist-org/alist/v3/internal/conf" @@ -23,29 +24,38 @@ type FileUploadProxy struct { buffer *os.File path string ctx context.Context + trunc bool } -func OpenUpload(ctx context.Context, path string) (*FileUploadProxy, error) { +func uploadAuth(ctx context.Context, path string) error { user := ctx.Value("user").(*model.User) path, err := user.JoinPath(path) if err != nil { - return nil, err + return err } meta, err := op.GetNearestMeta(stdpath.Dir(path)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - return nil, err + return err } } if !(common.CanAccess(user, meta, path, ctx.Value("meta_pass").(string)) && ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { - return nil, errs.PermissionDenied + return errs.PermissionDenied + } + return nil +} + +func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy, error) { + err := uploadAuth(ctx, path) + if err != nil { + return nil, err } tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") if err != nil { return nil, err } - return &FileUploadProxy{buffer: tmpFile, path: path, ctx: ctx}, nil + return &FileUploadProxy{buffer: tmpFile, path: path, ctx: ctx, trunc: trunc}, nil } func (f *FileUploadProxy) Read(p []byte) (n int, err error) { @@ -77,6 +87,9 @@ func (f *FileUploadProxy) Close() error { if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { return err } + if f.trunc { + _ = fs.Remove(f.ctx, f.path) + } s := &stream.FileStream{ Obj: &model.Object{ Name: name, @@ -84,10 +97,113 @@ func (f *FileUploadProxy) Close() error { Modified: time.Now(), }, Mimetype: contentType, - WebPutAsTask: false, + WebPutAsTask: true, } s.SetTmpFile(f.buffer) s.Closers.Add(f.buffer) _, err = fs.PutAsTask(f.ctx, dir, s) return err } + +type FileUploadWithLengthProxy struct { + ftpserver.FileTransfer + ctx context.Context + path string + length int64 + first512Bytes [512]byte + pFirst int + pipeWriter io.WriteCloser + errChan chan error +} + +func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length int64) (*FileUploadWithLengthProxy, error) { + err := uploadAuth(ctx, path) + if err != nil { + return nil, err + } + if trunc { + _ = fs.Remove(ctx, path) + } + return &FileUploadWithLengthProxy{ctx: ctx, path: path, length: length}, nil +} + +func (f *FileUploadWithLengthProxy) Read(p []byte) (n int, err error) { + return 0, errs.NotSupport +} + +func (f *FileUploadWithLengthProxy) Write(p []byte) (n int, err error) { + if f.pipeWriter != nil { + select { + case e := <-f.errChan: + return 0, e + default: + return f.pipeWriter.Write(p) + } + } else if len(p) < 512-f.pFirst { + copy(f.first512Bytes[f.pFirst:], p) + f.pFirst += len(p) + return len(p), nil + } else { + copy(f.first512Bytes[f.pFirst:], p[:512-f.pFirst]) + contentType := http.DetectContentType(f.first512Bytes[:]) + dir, name := stdpath.Split(f.path) + reader, writer := io.Pipe() + f.errChan = make(chan error, 1) + s := &stream.FileStream{ + Obj: &model.Object{ + Name: name, + Size: f.length, + Modified: time.Now(), + }, + Mimetype: contentType, + WebPutAsTask: false, + Reader: reader, + } + go func() { + e := fs.PutDirectly(f.ctx, dir, s, true) + f.errChan <- e + close(f.errChan) + }() + f.pipeWriter = writer + n, err = writer.Write(f.first512Bytes[:]) + if err != nil { + return n, err + } + n1, err := writer.Write(p[512-f.pFirst:]) + if err != nil { + return n1 + 512 - f.pFirst, err + } + f.pFirst = 512 + return len(p), nil + } +} + +func (f *FileUploadWithLengthProxy) Seek(offset int64, whence int) (int64, error) { + return 0, errs.NotSupport +} + +func (f *FileUploadWithLengthProxy) Close() error { + if f.pipeWriter != nil { + err := f.pipeWriter.Close() + if err != nil { + return err + } + err = <-f.errChan + return err + } else { + data := f.first512Bytes[:f.pFirst] + contentType := http.DetectContentType(data) + dir, name := stdpath.Split(f.path) + s := &stream.FileStream{ + Obj: &model.Object{ + Name: name, + Size: int64(f.pFirst), + Modified: time.Now(), + }, + Mimetype: contentType, + WebPutAsTask: false, + Reader: bytes.NewReader(data), + } + return fs.PutDirectly(f.ctx, dir, s, true) + } +} diff --git a/server/ftp/sftp.go b/server/ftp/sftp.go new file mode 100644 index 00000000000..0a11ee1862d --- /dev/null +++ b/server/ftp/sftp.go @@ -0,0 +1,122 @@ +package ftp + +import ( + "github.com/KirCute/sftpd-alist" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "os" +) + +type SftpDriverAdapter struct { + FtpDriver *AferoAdapter +} + +func (s *SftpDriverAdapter) OpenFile(_ string, _ uint32, _ *sftpd.Attr) (sftpd.File, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (s *SftpDriverAdapter) OpenDir(_ string) (sftpd.Dir, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (s *SftpDriverAdapter) Remove(name string) error { + return s.FtpDriver.Remove(name) +} + +func (s *SftpDriverAdapter) Rename(old, new string, _ uint32) error { + return s.FtpDriver.Rename(old, new) +} + +func (s *SftpDriverAdapter) Mkdir(name string, attr *sftpd.Attr) error { + return s.FtpDriver.Mkdir(name, attr.Mode) +} + +func (s *SftpDriverAdapter) Rmdir(name string) error { + return s.Remove(name) +} + +func (s *SftpDriverAdapter) Stat(name string, _ bool) (*sftpd.Attr, error) { + stat, err := s.FtpDriver.Stat(name) + if err != nil { + return nil, err + } + return fileInfoToSftpAttr(stat), nil +} + +func (s *SftpDriverAdapter) SetStat(_ string, _ *sftpd.Attr) error { + return errs.NotSupport +} + +func (s *SftpDriverAdapter) ReadLink(_ string) (string, error) { + return "", errs.NotSupport +} + +func (s *SftpDriverAdapter) CreateLink(_, _ string, _ uint32) error { + return errs.NotSupport +} + +func (s *SftpDriverAdapter) RealPath(path string) (string, error) { + return utils.FixAndCleanPath(path), nil +} + +func (s *SftpDriverAdapter) GetHandle(name string, flags uint32, _ *sftpd.Attr, offset uint64) (sftpd.FileTransfer, error) { + return s.FtpDriver.GetHandle(name, sftpFlagToOpenMode(flags), int64(offset)) +} + +func (s *SftpDriverAdapter) ReadDir(name string) ([]sftpd.NamedAttr, error) { + dir, err := s.FtpDriver.ReadDir(name) + if err != nil { + return nil, err + } + ret := make([]sftpd.NamedAttr, len(dir)) + for i, d := range dir { + ret[i] = *fileInfoToSftpNamedAttr(d) + } + return ret, nil +} + +// From leffss/sftpd +func sftpFlagToOpenMode(flags uint32) int { + mode := 0 + if (flags & SSH_FXF_READ) != 0 { + mode |= os.O_RDONLY + } + if (flags & SSH_FXF_WRITE) != 0 { + mode |= os.O_WRONLY + } + if (flags & SSH_FXF_APPEND) != 0 { + mode |= os.O_APPEND + } + if (flags & SSH_FXF_CREAT) != 0 { + mode |= os.O_CREATE + } + if (flags & SSH_FXF_TRUNC) != 0 { + mode |= os.O_TRUNC + } + if (flags & SSH_FXF_EXCL) != 0 { + mode |= os.O_EXCL + } + return mode +} + +func fileInfoToSftpAttr(stat os.FileInfo) *sftpd.Attr { + ret := &sftpd.Attr{} + ret.Flags |= sftpd.ATTR_SIZE + ret.Size = uint64(stat.Size()) + ret.Flags |= sftpd.ATTR_MODE + ret.Mode = stat.Mode() + ret.Flags |= sftpd.ATTR_TIME + ret.ATime = stat.Sys().(model.Obj).CreateTime() + ret.MTime = stat.ModTime() + return ret +} + +func fileInfoToSftpNamedAttr(stat os.FileInfo) *sftpd.NamedAttr { + return &sftpd.NamedAttr{ + Name: stat.Name(), + Attr: *fileInfoToSftpAttr(stat), + } +} diff --git a/server/ftp/site.go b/server/ftp/site.go new file mode 100644 index 00000000000..8ea667d8e8d --- /dev/null +++ b/server/ftp/site.go @@ -0,0 +1,21 @@ +package ftp + +import ( + "fmt" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "strconv" +) + +func HandleSIZE(param string, client ftpserver.ClientDriver) (int, string) { + fs, ok := client.(*AferoAdapter) + if !ok { + return ftpserver.StatusNotLoggedIn, "Unexpected exception (driver is nil)" + } + size, err := strconv.ParseInt(param, 10, 64) + if err != nil { + return ftpserver.StatusSyntaxErrorParameters, fmt.Sprintf( + "Couldn't parse file size, given: %s, err: %v", param, err) + } + fs.SetNextFileSize(size) + return ftpserver.StatusOK, "Accepted next file size" +} diff --git a/server/sftp.go b/server/sftp.go new file mode 100644 index 00000000000..3b07d472c37 --- /dev/null +++ b/server/sftp.go @@ -0,0 +1,109 @@ +package server + +import ( + "context" + "github.com/KirCute/sftpd-alist" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/ftp" + "github.com/pkg/errors" + "golang.org/x/crypto/ssh" + "net/http" +) + +type SftpDriver struct { + proxyHeader *http.Header + config *sftpd.Config +} + +func NewSftpDriver() (*SftpDriver, error) { + header := &http.Header{} + header.Add("User-Agent", setting.GetStr(conf.FTPProxyUserAgent)) + return &SftpDriver{ + proxyHeader: header, + }, nil +} + +func (d *SftpDriver) GetConfig() *sftpd.Config { + if d.config != nil { + return d.config + } + serverConfig := ssh.ServerConfig{ + NoClientAuth: true, + NoClientAuthCallback: d.NoClientAuth, + PasswordCallback: d.PasswordAuth, + AuthLogCallback: d.AuthLogCallback, + BannerCallback: d.GetBanner, + } + for _, k := range conf.SSHSigners { + serverConfig.AddHostKey(k) + } + d.config = &sftpd.Config{ + ServerConfig: serverConfig, + HostPort: conf.Conf.SFTP.Listen, + ErrorLogFunc: utils.Log.Error, + //DebugLogFunc: utils.Log.Debugf, + } + return d.config +} + +func (d *SftpDriver) GetFileSystem(sc *ssh.ServerConn) (sftpd.FileSystem, error) { + userObj, err := op.GetUserByName(sc.User()) + if err != nil { + return nil, err + } + ctx := context.Background() + ctx = context.WithValue(ctx, "user", userObj) + ctx = context.WithValue(ctx, "meta_pass", "") + ctx = context.WithValue(ctx, "client_ip", sc.RemoteAddr().String()) + ctx = context.WithValue(ctx, "proxy_header", d.proxyHeader) + return &ftp.SftpDriverAdapter{FtpDriver: ftp.NewAferoAdapter(ctx)}, nil +} + +func (d *SftpDriver) Close() { +} + +func (d *SftpDriver) NoClientAuth(conn ssh.ConnMetadata) (*ssh.Permissions, error) { + if conn.User() != "guest" { + return nil, errors.New("only guest is allowed to login without authorization") + } + guest, err := op.GetGuest() + if err != nil { + return nil, err + } + if guest.Disabled || !guest.CanFTPAccess() { + return nil, errors.New("user is not allowed to access via SFTP") + } + return nil, nil +} + +func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) { + userObj, err := op.GetUserByName(conn.User()) + if err != nil { + return nil, err + } + passHash := model.StaticHash(string(password)) + if err = userObj.ValidatePwdStaticHash(passHash); err != nil { + return nil, err + } + if userObj.Disabled || !userObj.CanFTPAccess() { + return nil, errors.New("user is not allowed to access via SFTP") + } + return nil, nil +} + +func (d *SftpDriver) AuthLogCallback(conn ssh.ConnMetadata, method string, err error) { + ip := conn.RemoteAddr().String() + if err == nil { + utils.Log.Infof("[SFTP] %s(%s) logged in via %s", conn.User(), ip, method) + } else if method != "none" { + utils.Log.Infof("[SFTP] %s(%s) tries logging in via %s but with error: %s", conn.User(), ip, method, err) + } +} + +func (d *SftpDriver) GetBanner(_ ssh.ConnMetadata) string { + return setting.GetStr(conf.Announcement) +} From cf58ab3a78aea2e97053fc3fbf8b0050f420604a Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 12 Dec 2024 21:04:14 +0800 Subject: [PATCH 382/659] chore(config): disable FTP and SFTP by default --- internal/conf/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/conf/config.go b/internal/conf/config.go index 6c0ccb2adc2..a9b38242e25 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -180,7 +180,7 @@ func DefaultConfig() *Config { SSL: false, }, FTP: FTP{ - Enable: true, + Enable: false, Listen: ":5221", FindPasvPortAttempts: 50, ActiveTransferPortNon20: false, @@ -192,7 +192,7 @@ func DefaultConfig() *Config { EnablePasvConnIPCheck: true, }, SFTP: SFTP{ - Enable: true, + Enable: false, Listen: ":5222", }, } From 331885ed64860c58b7556f7ac3d46a5eae875ce6 Mon Sep 17 00:00:00 2001 From: hshpy Date: Tue, 17 Dec 2024 22:04:27 +0800 Subject: [PATCH 383/659] fix(net): close of closed channel (#7580) --- internal/net/request.go | 5 +---- internal/net/serve.go | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/net/request.go b/internal/net/request.go index c0f547ba8c3..1a7405e4260 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -169,9 +169,7 @@ func (d *downloader) sendChunkTask() *chunk { // when the final reader Close, we interrupt func (d *downloader) interrupt() error { - if d.chunkChannel == nil { - return nil - } + d.cancel() if d.written != d.params.Range.Length { log.Debugf("Downloader interrupt before finish") @@ -181,7 +179,6 @@ func (d *downloader) interrupt() error { } defer func() { close(d.chunkChannel) - d.chunkChannel = nil for _, buf := range d.bufs { buf.Close() } diff --git a/internal/net/serve.go b/internal/net/serve.go index 0eb8cbb8866..e85f61a8950 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -174,7 +174,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time pw.Close() }() } - defer sendContent.Close() + //defer sendContent.Close() w.Header().Set("Accept-Ranges", "bytes") if w.Header().Get("Content-Encoding") == "" { From b8bd14f99b3cccdddbd7f8d3b841655892ffa89c Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Tue, 17 Dec 2024 22:05:52 +0800 Subject: [PATCH 384/659] fix(lanzou): missing parameter (#7678 close #7210) --- drivers/lanzou/help.go | 4 ++-- drivers/lanzou/util.go | 22 ++++++++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index 31a558e9c75..81d7c567d5c 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -120,9 +120,9 @@ var findKVReg = regexp.MustCompile(`'(.+?)':('?([^' },]*)'?)`) // 拆分kv func findJSVarFunc(key, data string) string { var values []string if key != "sasign" { - values = regexp.MustCompile(`var ` + key + ` = '(.+?)';`).FindStringSubmatch(data) + values = regexp.MustCompile(`var ` + key + `\s*=\s*['"]?(.+?)['"]?;`).FindStringSubmatch(data) } else { - matches := regexp.MustCompile(`var `+key+` = '(.+?)';`).FindAllStringSubmatch(data, -1) + matches := regexp.MustCompile(`var `+key+`\s*=\s*['"]?(.+?)['"]?;`).FindAllStringSubmatch(data, -1) if len(matches) == 3 { values = matches[1] } else { diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index abc2c400119..4b9959ad53d 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -264,6 +264,9 @@ var findSubFolderReg = regexp.MustCompile(`(?i)(?:folderlink|mbxfolder).+href="/ // 获取下载页面链接 var findDownPageParamReg = regexp.MustCompile(` 1 { + fileID = fileIDs[1] + } else { + return nil, fmt.Errorf("not find file id") + } var resp FileShareInfoAndUrlResp[string] - _, err = d.post(d.ShareUrl+"/ajaxm.php", func(req *resty.Request) { req.SetFormData(param) }, &resp) + _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } @@ -381,8 +392,15 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( return nil, err } + fileIDs := findFileIDReg.FindStringSubmatch(nextPageData) + var fileID string + if len(fileIDs) > 1 { + fileID = fileIDs[1] + } else { + return nil, fmt.Errorf("not find file id") + } var resp FileShareInfoAndUrlResp[int] - _, err = d.post(d.ShareUrl+"/ajaxm.php", func(req *resty.Request) { req.SetFormData(param) }, &resp) + _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } From db9922412611f6d546f723586ec67ce587666478 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Wed, 25 Dec 2024 21:08:22 +0800 Subject: [PATCH 385/659] =?UTF-8?q?perf:=20Speed=20=E2=80=8B=E2=80=8Bof=20?= =?UTF-8?q?database=20initialization=20(#7694)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: 优化非sqlite3数据库时初始化慢的问题 * refactor --- internal/bootstrap/data/setting.go | 48 ++++++++++++++++++------------ internal/bootstrap/db.go | 18 +++++------ internal/op/setting.go | 8 ++--- server/common/base.go | 8 ++--- 4 files changed, 46 insertions(+), 36 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 206273b41ac..bcb64f792d7 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -3,6 +3,7 @@ package data import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" @@ -21,17 +22,19 @@ func initSettings() { if err != nil { utils.Log.Fatalf("failed get settings: %+v", err) } - for i := range settings { - if !isActive(settings[i].Key) && settings[i].Flag != model.DEPRECATED { - settings[i].Flag = model.DEPRECATED - err = op.SaveSettingItem(&settings[i]) + settingMap := map[string]*model.SettingItem{} + for _, v := range settings { + if !isActive(v.Key) && v.Flag != model.DEPRECATED { + v.Flag = model.DEPRECATED + err = op.SaveSettingItem(&v) if err != nil { utils.Log.Fatalf("failed save setting: %+v", err) } } + settingMap[v.Key] = &v } - // create or save setting + save := false for i := range initialSettingItems { item := &initialSettingItems[i] item.Index = uint(i) @@ -39,26 +42,33 @@ func initSettings() { item.PreDefault = item.Value } // err - stored, err := op.GetSettingItemByKey(item.Key) - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - utils.Log.Fatalf("failed get setting: %+v", err) - continue + stored, ok := settingMap[item.Key] + if !ok { + stored, err = op.GetSettingItemByKey(item.Key) + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + utils.Log.Fatalf("failed get setting: %+v", err) + continue + } } - // save if stored != nil && item.Key != conf.VERSION && stored.Value != item.PreDefault { item.Value = stored.Value } + _, err = op.HandleSettingItemHook(item) + if err != nil { + utils.Log.Errorf("failed to execute hook on %s: %+v", item.Key, err) + continue + } + // save if stored == nil || *item != *stored { - err = op.SaveSettingItem(item) - if err != nil { - utils.Log.Fatalf("failed save setting: %+v", err) - } + save = true + } + } + if save { + err = db.SaveSettingItems(initialSettingItems) + if err != nil { + utils.Log.Fatalf("failed save setting: %+v", err) } else { - // Not save so needs to execute hook - _, err = op.HandleSettingItemHook(item) - if err != nil { - utils.Log.Errorf("failed to execute hook on %s: %+v", item.Key, err) - } + op.SettingCacheUpdate() } } } diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index 5dfa2820d18..39b659b78f1 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -56,20 +56,20 @@ func InitDB() { } case "mysql": { - //[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] - dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local&tls=%s", - database.User, database.Password, database.Host, database.Port, database.Name, database.SSLMode) - if database.DSN != "" { - dsn = database.DSN + dsn := database.DSN + if dsn == "" { + //[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] + dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local&tls=%s", + database.User, database.Password, database.Host, database.Port, database.Name, database.SSLMode) } dB, err = gorm.Open(mysql.Open(dsn), gormConfig) } case "postgres": { - dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", - database.Host, database.User, database.Password, database.Name, database.Port, database.SSLMode) - if database.DSN != "" { - dsn = database.DSN + dsn := database.DSN + if dsn == "" { + dsn = fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", + database.Host, database.User, database.Password, database.Name, database.Port, database.SSLMode) } dB, err = gorm.Open(postgres.Open(dsn), gormConfig) } diff --git a/internal/op/setting.go b/internal/op/setting.go index 83d19c12fbe..50eba3f744e 100644 --- a/internal/op/setting.go +++ b/internal/op/setting.go @@ -26,7 +26,7 @@ var settingGroupCacheF = func(key string, item []model.SettingItem) { settingGroupCache.Set(key, item, cache.WithEx[[]model.SettingItem](time.Hour)) } -func settingCacheUpdate() { +func SettingCacheUpdate() { settingCache.Clear() settingGroupCache.Clear() } @@ -167,7 +167,7 @@ func SaveSettingItems(items []model.SettingItem) error { } } if len(errs) < len(items)-len(noHookItems)+1 { - settingCacheUpdate() + SettingCacheUpdate() } return utils.MergeErrors(errs...) } @@ -181,7 +181,7 @@ func SaveSettingItem(item *model.SettingItem) (err error) { if err = db.SaveSettingItem(item); err != nil { return err } - settingCacheUpdate() + SettingCacheUpdate() return nil } @@ -193,6 +193,6 @@ func DeleteSettingItemByKey(key string) error { if !old.IsDeprecated() { return errors.Errorf("setting [%s] is not deprecated", key) } - settingCacheUpdate() + SettingCacheUpdate() return db.DeleteSettingItemByKey(key) } diff --git a/server/common/base.go b/server/common/base.go index eb6ef2b8ac2..11a28d25039 100644 --- a/server/common/base.go +++ b/server/common/base.go @@ -12,16 +12,16 @@ import ( func GetApiUrl(r *http.Request) string { api := conf.Conf.SiteURL if strings.HasPrefix(api, "http") { - return api + return strings.TrimSuffix(api, "/") } if r != nil { protocol := "http" if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { protocol = "https" } - host := r.Host - if r.Header.Get("X-Forwarded-Host") != "" { - host = r.Header.Get("X-Forwarded-Host") + host := r.Header.Get("X-Forwarded-Host") + if host == "" { + host = r.Host } api = fmt.Sprintf("%s://%s", protocol, stdpath.Join(host, api)) } From d7aa1608ac2f3834af87b4a81b8f0970b3dadbc0 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Wed, 25 Dec 2024 21:09:54 +0800 Subject: [PATCH 386/659] feat(task): add speed monitor (#7655) --- internal/fs/copy.go | 17 ++++--- internal/fs/fs.go | 4 +- internal/fs/put.go | 11 +++-- internal/offline_download/115/client.go | 1 + internal/offline_download/aria2/aria2.go | 7 +-- internal/offline_download/http/client.go | 1 + internal/offline_download/pikpak/pikpak.go | 5 ++ internal/offline_download/qbit/qbit.go | 1 + internal/offline_download/tool/add.go | 4 +- internal/offline_download/tool/base.go | 11 +++-- internal/offline_download/tool/download.go | 16 +++++-- internal/offline_download/tool/transfer.go | 6 ++- .../offline_download/transmission/client.go | 1 + internal/task/base.go | 46 ++++++++++++++++--- server/handles/fsmanage.go | 2 +- server/handles/fsup.go | 4 +- server/handles/offline_download.go | 2 +- server/handles/task.go | 17 +++++-- 18 files changed, 114 insertions(+), 42 deletions(-) diff --git a/internal/fs/copy.go b/internal/fs/copy.go index d4ad452b169..c3fadaab8fa 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" stdpath "path" + "time" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" @@ -18,7 +19,7 @@ import ( ) type CopyTask struct { - task.TaskWithCreator + task.TaskExtension Status string `json:"-"` //don't save status to save space SrcObjPath string `json:"src_path"` DstDirPath string `json:"dst_path"` @@ -37,6 +38,9 @@ func (t *CopyTask) GetStatus() string { } func (t *CopyTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() var err error if t.srcStorage == nil { t.srcStorage, err = op.GetStorageByMountPath(t.SrcStorageMp) @@ -54,7 +58,7 @@ var CopyTaskManager *tache.Manager[*CopyTask] // Copy if in the same storage, call move method // if not, add copy task -func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskInfoWithCreator, error) { +func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { return nil, errors.WithMessage(err, "failed get src storage") @@ -93,9 +97,9 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool } } // not in the same storage - taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed + taskCreator, _ := ctx.Value("user").(*model.User) t := &CopyTask{ - TaskWithCreator: task.TaskWithCreator{ + TaskExtension: task.TaskExtension{ Creator: taskCreator, }, srcStorage: srcStorage, @@ -128,8 +132,8 @@ func copyBetween2Storages(t *CopyTask, srcStorage, dstStorage driver.Driver, src srcObjPath := stdpath.Join(srcObjPath, obj.GetName()) dstObjPath := stdpath.Join(dstDirPath, srcObj.GetName()) CopyTaskManager.Add(&CopyTask{ - TaskWithCreator: task.TaskWithCreator{ - Creator: t.Creator, + TaskExtension: task.TaskExtension{ + Creator: t.GetCreator(), }, srcStorage: srcStorage, dstStorage: dstStorage, @@ -150,6 +154,7 @@ func copyFileBetween2Storages(tsk *CopyTask, srcStorage, dstStorage driver.Drive if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", srcFilePath) } + tsk.SetTotalBytes(srcFile.GetSize()) link, _, err := op.Link(tsk.Ctx(), srcStorage, srcFilePath, model.LinkArgs{ Header: http.Header{}, }) diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 65e5a2c264a..24f1d47fa5e 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -69,7 +69,7 @@ func Move(ctx context.Context, srcPath, dstDirPath string, lazyCache ...bool) er return err } -func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskInfoWithCreator, error) { +func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { res, err := _copy(ctx, srcObjPath, dstDirPath, lazyCache...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) @@ -101,7 +101,7 @@ func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer return err } -func PutAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskInfoWithCreator, error) { +func PutAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskExtensionInfo, error) { t, err := putAsTask(ctx, dstDirPath, file) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) diff --git a/internal/fs/put.go b/internal/fs/put.go index 23197f5ba54..bc33a3ac102 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -10,10 +10,11 @@ import ( "github.com/alist-org/alist/v3/internal/task" "github.com/pkg/errors" "github.com/xhofe/tache" + "time" ) type UploadTask struct { - task.TaskWithCreator + task.TaskExtension storage driver.Driver dstDirActualPath string file model.FileStreamer @@ -28,13 +29,16 @@ func (t *UploadTask) GetStatus() string { } func (t *UploadTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() return op.Put(t.Ctx(), t.storage, t.dstDirActualPath, t.file, t.SetProgress, true) } var UploadTaskManager *tache.Manager[*UploadTask] // putAsTask add as a put task and return immediately -func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskInfoWithCreator, error) { +func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskExtensionInfo, error) { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { return nil, errors.WithMessage(err, "failed get storage") @@ -52,13 +56,14 @@ func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) } taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed t := &UploadTask{ - TaskWithCreator: task.TaskWithCreator{ + TaskExtension: task.TaskExtension{ Creator: taskCreator, }, storage: storage, dstDirActualPath: dstDirActualPath, file: file, } + t.SetTotalBytes(file.GetSize()) UploadTaskManager.Add(t) return t, nil } diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index 0ebf38ffced..45f147db06d 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -107,6 +107,7 @@ func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Progress = t.Percent s.Status = t.GetStatus() s.Completed = t.IsDone() + s.TotalBytes = t.Size if t.IsFailed() { s.Err = fmt.Errorf(t.GetStatus()) } diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go index d22b32f9d55..fb212b35990 100644 --- a/internal/offline_download/aria2/aria2.go +++ b/internal/offline_download/aria2/aria2.go @@ -82,7 +82,7 @@ func (a *Aria2) Status(task *tool.DownloadTask) (*tool.Status, error) { if err != nil { return nil, err } - total, err := strconv.ParseUint(info.TotalLength, 10, 64) + total, err := strconv.ParseInt(info.TotalLength, 10, 64) if err != nil { total = 0 } @@ -91,8 +91,9 @@ func (a *Aria2) Status(task *tool.DownloadTask) (*tool.Status, error) { downloaded = 0 } s := &tool.Status{ - Completed: info.Status == "complete", - Err: err, + Completed: info.Status == "complete", + Err: err, + TotalBytes: total, } s.Progress = float64(downloaded) / float64(total) * 100 if len(info.FollowedBy) != 0 { diff --git a/internal/offline_download/http/client.go b/internal/offline_download/http/client.go index 6f22fcf7b98..9b83400ea34 100644 --- a/internal/offline_download/http/client.go +++ b/internal/offline_download/http/client.go @@ -83,6 +83,7 @@ func (s SimpleHttp) Run(task *tool.DownloadTask) error { } defer file.Close() fileSize := resp.ContentLength + task.SetTotalBytes(fileSize) err = utils.CopyWithCtx(task.Ctx(), file, resp.Body, fileSize, task.SetProgress) return err } diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index 618b1442b8a..f07b3de8aa0 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -3,6 +3,7 @@ package pikpak import ( "context" "fmt" + "strconv" "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/internal/errs" @@ -105,6 +106,10 @@ func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { s.Progress = float64(t.Progress) s.Status = t.Message s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE") + s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) + if err != nil { + s.TotalBytes = 0 + } if t.Phase == "PHASE_TYPE_ERROR" { s.Err = fmt.Errorf(t.Message) } diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go index 807ebfef2dc..458de03f02f 100644 --- a/internal/offline_download/qbit/qbit.go +++ b/internal/offline_download/qbit/qbit.go @@ -64,6 +64,7 @@ func (a *QBittorrent) Status(task *tool.DownloadTask) (*tool.Status, error) { return nil, err } s := &tool.Status{} + s.TotalBytes = info.Size s.Progress = float64(info.Completed) / float64(info.Size) * 100 switch info.State { case qbittorrent.UPLOADING, qbittorrent.PAUSEDUP, qbittorrent.QUEUEDUP, qbittorrent.STALLEDUP, qbittorrent.FORCEDUP, qbittorrent.CHECKINGUP: diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 1c9da1467b5..42349e2e397 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -29,7 +29,7 @@ type AddURLArgs struct { DeletePolicy DeletePolicy } -func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskInfoWithCreator, error) { +func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, error) { // get tool tool, err := Tools.Get(args.Tool) if err != nil { @@ -81,7 +81,7 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskInfoWithCreator, er taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed t := &DownloadTask{ - TaskWithCreator: task.TaskWithCreator{ + TaskExtension: task.TaskExtension{ Creator: taskCreator, }, Url: args.URL, diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go index 3b9fb07a999..ae9eac2624b 100644 --- a/internal/offline_download/tool/base.go +++ b/internal/offline_download/tool/base.go @@ -16,11 +16,12 @@ type AddUrlArgs struct { } type Status struct { - Progress float64 - NewGID string - Completed bool - Status string - Err error + TotalBytes int64 + Progress float64 + NewGID string + Completed bool + Status string + Err error } type Tool interface { diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 038baf9690b..a0f1a81b3b5 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -14,7 +14,7 @@ import ( ) type DownloadTask struct { - task.TaskWithCreator + task.TaskExtension Url string `json:"url"` DstDirPath string `json:"dst_dir_path"` TempDir string `json:"temp_dir"` @@ -28,6 +28,9 @@ type DownloadTask struct { } func (t *DownloadTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() if t.tool == nil { tool, err := Tools.Get(t.Toolname) if err != nil { @@ -131,6 +134,7 @@ func (t *DownloadTask) Update() (bool, error) { } t.callStatusRetried = 0 t.SetProgress(info.Progress) + t.SetTotalBytes(info.TotalBytes) t.Status = fmt.Sprintf("[%s]: %s", t.tool.Name(), info.Status) if info.NewGID != "" { log.Debugf("followen by: %+v", info.NewGID) @@ -171,16 +175,18 @@ func (t *DownloadTask) Complete() error { // upload files for i := range files { file := files[i] - TransferTaskManager.Add(&TransferTask{ - TaskWithCreator: task.TaskWithCreator{ - Creator: t.Creator, + tsk := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: t.GetCreator(), }, file: file, DstDirPath: t.DstDirPath, TempDir: t.TempDir, DeletePolicy: t.DeletePolicy, FileDir: file.Path, - }) + } + tsk.SetTotalBytes(file.Size) + TransferTaskManager.Add(tsk) } return nil } diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 085b4a66afa..a77c4822f83 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" @@ -16,7 +17,7 @@ import ( ) type TransferTask struct { - task.TaskWithCreator + task.TaskExtension FileDir string `json:"file_dir"` DstDirPath string `json:"dst_dir_path"` TempDir string `json:"temp_dir"` @@ -25,6 +26,9 @@ type TransferTask struct { } func (t *TransferTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() // check dstDir again var err error if (t.file == File{}) { diff --git a/internal/offline_download/transmission/client.go b/internal/offline_download/transmission/client.go index a6075414814..4131f3e1c53 100644 --- a/internal/offline_download/transmission/client.go +++ b/internal/offline_download/transmission/client.go @@ -150,6 +150,7 @@ func (t *Transmission) Status(task *tool.DownloadTask) (*tool.Status, error) { Err: err, } s.Progress = *info.PercentDone * 100 + s.TotalBytes = int64(*info.SizeWhenDone / 8) switch *info.Status { case transmissionrpc.TorrentStatusCheckWait, diff --git a/internal/task/base.go b/internal/task/base.go index a30e59876b8..93f413a7111 100644 --- a/internal/task/base.go +++ b/internal/task/base.go @@ -3,24 +3,58 @@ package task import ( "github.com/alist-org/alist/v3/internal/model" "github.com/xhofe/tache" + "time" ) -type TaskWithCreator struct { +type TaskExtension struct { tache.Base - Creator *model.User + Creator *model.User + startTime *time.Time + endTime *time.Time + totalBytes int64 } -func (t *TaskWithCreator) SetCreator(creator *model.User) { +func (t *TaskExtension) SetCreator(creator *model.User) { t.Creator = creator t.Persist() } -func (t *TaskWithCreator) GetCreator() *model.User { +func (t *TaskExtension) GetCreator() *model.User { return t.Creator } -type TaskInfoWithCreator interface { +func (t *TaskExtension) SetStartTime(startTime time.Time) { + t.startTime = &startTime +} + +func (t *TaskExtension) GetStartTime() *time.Time { + return t.startTime +} + +func (t *TaskExtension) SetEndTime(endTime time.Time) { + t.endTime = &endTime +} + +func (t *TaskExtension) GetEndTime() *time.Time { + return t.endTime +} + +func (t *TaskExtension) ClearEndTime() { + t.endTime = nil +} + +func (t *TaskExtension) SetTotalBytes(totalBytes int64) { + t.totalBytes = totalBytes +} + +func (t *TaskExtension) GetTotalBytes() int64 { + return t.totalBytes +} + +type TaskExtensionInfo interface { tache.TaskWithInfo - SetCreator(creator *model.User) GetCreator() *model.User + GetStartTime() *time.Time + GetEndTime() *time.Time + GetTotalBytes() int64 } diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 42d53d7e7c7..9877b1278d7 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -121,7 +121,7 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, err, 403) return } - var addedTasks []task.TaskInfoWithCreator + var addedTasks []task.TaskExtensionInfo for i, name := range req.Names { t, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) if t != nil { diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 3a366d49fd0..a17c50f08ee 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -57,7 +57,7 @@ func FsStream(c *gin.Context) { Mimetype: c.GetHeader("Content-Type"), WebPutAsTask: asTask, } - var t task.TaskInfoWithCreator + var t task.TaskExtensionInfo if asTask { t, err = fs.PutAsTask(c, dir, s) } else { @@ -122,7 +122,7 @@ func FsForm(c *gin.Context) { Mimetype: file.Header.Get("Content-Type"), WebPutAsTask: asTask, } - var t task.TaskInfoWithCreator + var t task.TaskExtensionInfo if asTask { s.Reader = struct { io.Reader diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index ff1fcfa05bc..9e26030a04d 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -133,7 +133,7 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 403) return } - var tasks []task.TaskInfoWithCreator + var tasks []task.TaskExtensionInfo for _, url := range req.Urls { t, err := tool.AddURL(c, &tool.AddURLArgs{ URL: url, diff --git a/server/handles/task.go b/server/handles/task.go index 5f9965053b9..c7d9ef4842d 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -4,6 +4,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/task" "math" + "time" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/offline_download/tool" @@ -21,10 +22,13 @@ type TaskInfo struct { State tache.State `json:"state"` Status string `json:"status"` Progress float64 `json:"progress"` + StartTime *time.Time `json:"start_time"` + EndTime *time.Time `json:"end_time"` + TotalBytes int64 `json:"total_bytes"` Error string `json:"error"` } -func getTaskInfo[T task.TaskInfoWithCreator](task T) TaskInfo { +func getTaskInfo[T task.TaskExtensionInfo](task T) TaskInfo { errMsg := "" if task.GetErr() != nil { errMsg = task.GetErr().Error() @@ -48,11 +52,14 @@ func getTaskInfo[T task.TaskInfoWithCreator](task T) TaskInfo { State: task.GetState(), Status: task.GetStatus(), Progress: progress, + StartTime: task.GetStartTime(), + EndTime: task.GetEndTime(), + TotalBytes: task.GetTotalBytes(), Error: errMsg, } } -func getTaskInfos[T task.TaskInfoWithCreator](tasks []T) []TaskInfo { +func getTaskInfos[T task.TaskExtensionInfo](tasks []T) []TaskInfo { return utils.MustSliceConvert(tasks, getTaskInfo[T]) } @@ -68,7 +75,7 @@ func getUserInfo(c *gin.Context) (bool, uint, bool) { } } -func getTargetedHandler[T task.TaskInfoWithCreator](manager *tache.Manager[T], callback func(c *gin.Context, task T)) gin.HandlerFunc { +func getTargetedHandler[T task.TaskExtensionInfo](manager *tache.Manager[T], callback func(c *gin.Context, task T)) gin.HandlerFunc { return func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { @@ -90,7 +97,7 @@ func getTargetedHandler[T task.TaskInfoWithCreator](manager *tache.Manager[T], c } } -func getBatchHandler[T task.TaskInfoWithCreator](manager *tache.Manager[T], callback func(task T)) gin.HandlerFunc { +func getBatchHandler[T task.TaskExtensionInfo](manager *tache.Manager[T], callback func(task T)) gin.HandlerFunc { return func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { @@ -115,7 +122,7 @@ func getBatchHandler[T task.TaskInfoWithCreator](manager *tache.Manager[T], call } } -func taskRoute[T task.TaskInfoWithCreator](g *gin.RouterGroup, manager *tache.Manager[T]) { +func taskRoute[T task.TaskExtensionInfo](g *gin.RouterGroup, manager *tache.Manager[T]) { g.GET("/undone", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { From bb2aec20e4b611ab2242c6b1dc793c337cf47d95 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Wed, 25 Dec 2024 21:11:05 +0800 Subject: [PATCH 387/659] fix(139): handle upload file conflicts (#7692) --- drivers/139/driver.go | 191 ++++++++++++++++++++++++++++-------------- drivers/139/types.go | 1 + 2 files changed, 131 insertions(+), 61 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 8862983ce5e..dd154efe42a 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -552,7 +552,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr firstPartInfos = firstPartInfos[:100] } - // 获取上传信息和前100个分片的上传地址 + // 创建任务，获取上传信息和前100个分片的上传地址 data := base.Json{ "contentHash": fullHash, "contentHashAlgorithm": "SHA256", @@ -572,87 +572,156 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } - if resp.Data.Exist || resp.Data.RapidUpload { + // 判断文件是否已存在 + // resp.Data.Exist: true 已存在同名文件且校验相同，云端不会重复增加文件，无需手动处理冲突 + if resp.Data.Exist { return nil } - uploadPartInfos := resp.Data.PartInfos + // 判断文件是否支持快传 + // resp.Data.RapidUpload: true 支持快传，但此处直接检测是否返回分片的上传地址 + // 快传的情况下同样需要手动处理冲突 + if resp.Data.PartInfos != nil { + // 读取前100个分片的上传地址 + uploadPartInfos := resp.Data.PartInfos + + // 获取后续分片的上传地址 + for i := 101; i < len(partInfos); i += 100 { + end := i + 100 + if end > len(partInfos) { + end = len(partInfos) + } + batchPartInfos := partInfos[i:end] + + moredata := base.Json{ + "fileId": resp.Data.FileId, + "uploadId": resp.Data.UploadId, + "partInfos": batchPartInfos, + "commonAccountInfo": base.Json{ + "account": d.Account, + "accountType": 1, + }, + } + pathname := "/hcy/file/getUploadUrl" + var moreresp PersonalUploadUrlResp + _, err = d.personalPost(pathname, moredata, &moreresp) + if err != nil { + return err + } + uploadPartInfos = append(uploadPartInfos, moreresp.Data.PartInfos...) + } - // 获取后续分片的上传地址 - for i := 101; i < len(partInfos); i += 100 { - end := i + 100 - if end > len(partInfos) { - end = len(partInfos) + // Progress + p := driver.NewProgress(stream.GetSize(), up) + + // 上传所有分片 + for _, uploadPartInfo := range uploadPartInfos { + index := uploadPartInfo.PartNumber - 1 + partSize := partInfos[index].PartSize + log.Debugf("[139] uploading part %+v/%+v", index, len(uploadPartInfos)) + limitReader := io.LimitReader(stream, partSize) + + // Update Progress + r := io.TeeReader(limitReader, p) + + req, err := http.NewRequest("PUT", uploadPartInfo.UploadUrl, r) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprint(partSize)) + req.Header.Set("Origin", "https://yun.139.com") + req.Header.Set("Referer", "https://yun.139.com/") + req.ContentLength = partSize + + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + _ = res.Body.Close() + log.Debugf("[139] uploaded: %+v", res) + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", res.StatusCode) + } } - batchPartInfos := partInfos[i:end] - moredata := base.Json{ - "fileId": resp.Data.FileId, - "uploadId": resp.Data.UploadId, - "partInfos": batchPartInfos, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, + data = base.Json{ + "contentHash": fullHash, + "contentHashAlgorithm": "SHA256", + "fileId": resp.Data.FileId, + "uploadId": resp.Data.UploadId, } - pathname := "/hcy/file/getUploadUrl" - var moreresp PersonalUploadUrlResp - _, err = d.personalPost(pathname, moredata, &moreresp) + _, err = d.personalPost("/hcy/file/complete", data, nil) if err != nil { return err } - uploadPartInfos = append(uploadPartInfos, moreresp.Data.PartInfos...) } - // Progress - p := driver.NewProgress(stream.GetSize(), up) - - // 上传所有分片 - for _, uploadPartInfo := range uploadPartInfos { - index := uploadPartInfo.PartNumber - 1 - partSize := partInfos[index].PartSize - log.Debugf("[139] uploading part %+v/%+v", index, len(uploadPartInfos)) - limitReader := io.LimitReader(stream, partSize) - - // Update Progress - r := io.TeeReader(limitReader, p) - - req, err := http.NewRequest("PUT", uploadPartInfo.UploadUrl, r) + // 处理冲突 + if resp.Data.FileName != stream.GetName() { + log.Debugf("[139] conflict detected: %s != %s", resp.Data.FileName, stream.GetName()) + // 给服务器一定时间处理数据，避免无法刷新文件列表 + time.Sleep(time.Millisecond * 500) + // 刷新并获取文件列表 + files, err := d.List(ctx, dstDir, model.ListArgs{Refresh: true}) if err != nil { return err } - req = req.WithContext(ctx) - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("Content-Length", fmt.Sprint(partSize)) - req.Header.Set("Origin", "https://yun.139.com") - req.Header.Set("Referer", "https://yun.139.com/") - req.ContentLength = partSize - - res, err := base.HttpClient.Do(req) - if err != nil { - return err + // 删除旧文件 + for _, file := range files { + if file.GetName() == stream.GetName() { + log.Debugf("[139] conflict: removing old: %s", file.GetName()) + // 删除前重命名旧文件，避免仍旧冲突 + err = d.Rename(ctx, file, stream.GetName()+random.String(4)) + if err != nil { + return err + } + err = d.Remove(ctx, file) + if err != nil { + return err + } + break + } } - _ = res.Body.Close() - log.Debugf("[139] uploaded: %+v", res) - if res.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", res.StatusCode) + // 重命名新文件 + for _, file := range files { + if file.GetName() == resp.Data.FileName { + log.Debugf("[139] conflict: renaming new: %s => %s", file.GetName(), stream.GetName()) + err = d.Rename(ctx, file, stream.GetName()) + if err != nil { + return err + } + break + } } } - - data = base.Json{ - "contentHash": fullHash, - "contentHashAlgorithm": "SHA256", - "fileId": resp.Data.FileId, - "uploadId": resp.Data.UploadId, - } - _, err = d.personalPost("/hcy/file/complete", data, nil) - if err != nil { - return err - } return nil case MetaPersonal: fallthrough case MetaFamily: + // 处理冲突 + // 获取文件列表 + files, err := d.List(ctx, dstDir, model.ListArgs{}) + if err != nil { + return err + } + // 删除旧文件 + for _, file := range files { + if file.GetName() == stream.GetName() { + log.Debugf("[139] conflict: removing old: %s", file.GetName()) + // 删除前重命名旧文件，避免仍旧冲突 + err = d.Rename(ctx, file, stream.GetName()+random.String(4)) + if err != nil { + return err + } + err = d.Remove(ctx, file) + if err != nil { + return err + } + break + } + } data := base.Json{ "manualRename": 2, "operation": 0, @@ -688,7 +757,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr pathname = "/orchestration/familyCloud-rebuild/content/v1.0/getFileUploadURL" } var resp UploadResp - _, err := d.post(pathname, data, &resp) + _, err = d.post(pathname, data, &resp) if err != nil { return err } diff --git a/drivers/139/types.go b/drivers/139/types.go index c34cba0388b..ac7079d8d18 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -261,6 +261,7 @@ type PersonalUploadResp struct { BaseResp Data struct { FileId string `json:"fileId"` + FileName string `json:"fileName"` PartInfos []PersonalPartInfo `json:"partInfos"` Exist bool `json:"exist"` RapidUpload bool `json:"rapidUpload"` From 6aaf5975c6651df36329860ab5be5a2a058396d7 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Wed, 25 Dec 2024 21:11:36 +0800 Subject: [PATCH 388/659] fix(ftp-server): work unproperly when base url is not root (#7693) * fix(ftp-server): work unproperly when base url is not root * fix: avoid merge conflict --- server/ftp/afero.go | 14 ++++++++++---- server/ftp/fsread.go | 6 +----- server/ftp/fsup.go | 4 ---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/server/ftp/afero.go b/server/ftp/afero.go index 866ad8c0231..448744b1c01 100644 --- a/server/ftp/afero.go +++ b/server/ftp/afero.go @@ -6,6 +6,7 @@ import ( ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" "github.com/spf13/afero" "os" "time" @@ -91,7 +92,12 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve if (flags & os.O_APPEND) != 0 { return nil, errs.NotSupport } - _, err := fs.Get(a.ctx, name, &fs.GetArgs{}) + user := a.ctx.Value("user").(*model.User) + path, err := user.JoinPath(name) + if err != nil { + return nil, err + } + _, err = fs.Get(a.ctx, path, &fs.GetArgs{}) exists := err == nil if (flags&os.O_CREATE) == 0 && !exists { return nil, errs.ObjectNotFound @@ -102,12 +108,12 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve if (flags & os.O_WRONLY) != 0 { trunc := (flags & os.O_TRUNC) != 0 if fileSize > 0 { - return OpenUploadWithLength(a.ctx, name, trunc, fileSize) + return OpenUploadWithLength(a.ctx, path, trunc, fileSize) } else { - return OpenUpload(a.ctx, name, trunc) + return OpenUpload(a.ctx, path, trunc) } } - return OpenDownload(a.ctx, name) + return OpenDownload(a.ctx, path) } func (a *AferoAdapter) SetNextFileSize(size int64) { diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index 6a9ba2ebb2a..91f87bf4163 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -25,12 +25,8 @@ type FileDownloadProxy struct { closers *utils.Closers } -func OpenDownload(ctx context.Context, path string) (*FileDownloadProxy, error) { +func OpenDownload(ctx context.Context, reqPath string) (*FileDownloadProxy, error) { user := ctx.Value("user").(*model.User) - reqPath, err := user.JoinPath(path) - if err != nil { - return nil, err - } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index f18c13c24b6..96c8468143c 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -29,10 +29,6 @@ type FileUploadProxy struct { func uploadAuth(ctx context.Context, path string) error { user := ctx.Value("user").(*model.User) - path, err := user.JoinPath(path) - if err != nil { - return err - } meta, err := op.GetNearestMeta(stdpath.Dir(path)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { From b72e85a73a44537678d59e34820a0ec694a52eec Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Wed, 25 Dec 2024 21:11:45 +0800 Subject: [PATCH 389/659] fix(ftp-server): rewrite download in a more appropriate method (#7656) --- server/ftp/fsread.go | 44 +++++++------------------------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index 91f87bf4163..74d184b6535 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -6,10 +6,8 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/server/common" "github.com/pkg/errors" "io" @@ -21,8 +19,7 @@ import ( type FileDownloadProxy struct { ftpserver.FileTransfer - reader io.ReadCloser - closers *utils.Closers + reader io.ReadCloser } func OpenDownload(ctx context.Context, reqPath string) (*FileDownloadProxy, error) { @@ -47,37 +44,15 @@ func OpenDownload(ctx context.Context, reqPath string) (*FileDownloadProxy, erro if err != nil { return nil, err } - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) - if err != nil { - return nil, err - } - if storage.GetStorage().ProxyRange { - common.ProxyRange(link, obj.GetSize()) + fileStream := stream.FileStream{ + Obj: obj, + Ctx: ctx, } - reader, closers, err := proxy(link) + ss, err := stream.NewSeekableStream(fileStream, link) if err != nil { return nil, err } - return &FileDownloadProxy{reader: reader, closers: closers}, nil -} - -func proxy(link *model.Link) (io.ReadCloser, *utils.Closers, error) { - if link.MFile != nil { - return link.MFile, nil, nil - } else if link.RangeReadCloser != nil { - rc, err := link.RangeReadCloser.RangeRead(context.Background(), http_range.Range{Length: -1}) - if err != nil { - return nil, nil, err - } - closers := link.RangeReadCloser.GetClosers() - return rc, &closers, nil - } else { - res, err := net.RequestHttp(context.Background(), http.MethodGet, link.Header, link.URL) - if err != nil { - return nil, nil, err - } - return res.Body, nil, nil - } + return &FileDownloadProxy{reader: ss}, nil } func (f *FileDownloadProxy) Read(p []byte) (n int, err error) { @@ -93,11 +68,6 @@ func (f *FileDownloadProxy) Seek(offset int64, whence int) (int64, error) { } func (f *FileDownloadProxy) Close() error { - defer func() { - if f.closers != nil { - _ = f.closers.Close() - } - }() return f.reader.Close() } From 40b0e66efec91b08b3ea09fdaff8943ae0bdbb5f Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Wed, 25 Dec 2024 21:12:30 +0800 Subject: [PATCH 390/659] feat(ftp-server): treat moving across file systems as copying (#7704 close #7701) * feat(ftp-server): treat moving across file systems as copying * fix: ensure compatibility across different fs on the same driver --- server/ftp/fsmanage.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go index 5199a473b5b..fb03c1b95cb 100644 --- a/server/ftp/fsmanage.go +++ b/server/ftp/fsmanage.go @@ -2,6 +2,7 @@ package ftp import ( "context" + "fmt" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" @@ -64,8 +65,14 @@ func Rename(ctx context.Context, oldPath, newPath string) error { if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { return errs.PermissionDenied } - if err := fs.Move(ctx, srcPath, dstDir); err != nil { - return err + if err = fs.Move(ctx, srcPath, dstDir); err != nil { + if srcBase != dstBase { + return err + } + if _, err1 := fs.Copy(ctx, srcPath, dstDir); err1 != nil { + return fmt.Errorf("failed move for %+v, and failed try copying for %+v", err, err1) + } + return nil } if srcBase != dstBase { return fs.Rename(ctx, stdpath.Join(dstDir, srcBase), dstBase) From 221cdf3611ae317f9c5a7071a8bee28fac0b176e Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Wed, 25 Dec 2024 21:13:23 +0800 Subject: [PATCH 391/659] feat(s3): support custom host presign (#7699 close #7696) --- drivers/s3/driver.go | 8 ++++++-- drivers/s3/meta.go | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 2b72d78980f..82c050a1fe8 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -99,8 +99,12 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo var link model.Link var err error if d.CustomHost != "" { - err = req.Build() - link.URL = req.HTTPRequest.URL.String() + if d.EnableCustomHostPresign { + link.URL, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + } else { + err = req.Build() + link.URL = req.HTTPRequest.URL.String() + } if d.RemoveBucket { link.URL = strings.Replace(link.URL, "/"+d.Bucket, "", 1) } diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 4436c61508e..4de4b60a690 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -14,6 +14,7 @@ type Addition struct { SecretAccessKey string `json:"secret_access_key" required:"true"` SessionToken string `json:"session_token"` CustomHost string `json:"custom_host"` + EnableCustomHostPresign bool `json:"enable_custom_host_presign"` SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` Placeholder string `json:"placeholder"` ForcePathStyle bool `json:"force_path_style"` From db5c601cfe3c816735bcbc034b66757772a6b9e0 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Wed, 25 Dec 2024 21:13:54 +0800 Subject: [PATCH 392/659] fix(crypt): add sign to thumbnail (#6611) --- drivers/crypt/driver.go | 7 ++++++- drivers/local/driver.go | 8 ++++---- server/common/common.go | 9 +++++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index b0325db4956..b6115896b98 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -13,6 +13,7 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" @@ -160,7 +161,11 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ // discarding hash as it's encrypted } if d.Thumbnail && thumb == "" { - thumb = utils.EncodePath(common.GetApiUrl(nil)+stdpath.Join("/d", args.ReqPath, ".thumbnails", name+".webp"), true) + thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") + thumb = fmt.Sprintf("%s/d%s?sign=%s", + common.GetApiUrl(common.GetHttpReq(ctx)), + utils.EncodePath(thumbPath, true), + sign.Sign(thumbPath)) } if !ok && !d.Thumbnail { result = append(result, &objRes) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 229c86925fb..2519232e7d6 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -101,17 +101,17 @@ func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if !d.ShowHidden && strings.HasPrefix(f.Name(), ".") { continue } - file := d.FileInfoToObj(f, args.ReqPath, fullPath) + file := d.FileInfoToObj(ctx, f, args.ReqPath, fullPath) files = append(files, file) } return files, nil } -func (d *Local) FileInfoToObj(f fs.FileInfo, reqPath string, fullPath string) model.Obj { +func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string, fullPath string) model.Obj { thumb := "" if d.Thumbnail { typeName := utils.GetFileType(f.Name()) if typeName == conf.IMAGE || typeName == conf.VIDEO { - thumb = common.GetApiUrl(nil) + stdpath.Join("/d", reqPath, f.Name()) + thumb = common.GetApiUrl(common.GetHttpReq(ctx)) + stdpath.Join("/d", reqPath, f.Name()) thumb = utils.EncodePath(thumb, true) thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) } @@ -149,7 +149,7 @@ func (d *Local) GetMeta(ctx context.Context, path string) (model.Obj, error) { if err != nil { return nil, err } - file := d.FileInfoToObj(f, path, path) + file := d.FileInfoToObj(ctx, f, path, path) //h := "123123" //if s, ok := f.(model.SetHash); ok && file.GetHash() == ("","") { // s.SetHash(h,"SHA1") diff --git a/server/common/common.go b/server/common/common.go index 28d2da4443d..e231ffe6e88 100644 --- a/server/common/common.go +++ b/server/common/common.go @@ -1,6 +1,8 @@ package common import ( + "context" + "net/http" "strings" "github.com/alist-org/alist/v3/cmd/flags" @@ -80,3 +82,10 @@ func SuccessResp(c *gin.Context, data ...interface{}) { Data: data[0], }) } + +func GetHttpReq(ctx context.Context) *http.Request { + if c, ok := ctx.(*gin.Context); ok { + return c.Request + } + return nil +} From 77d0c78bfd0e7040db459940312372e9a4813b05 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Wed, 25 Dec 2024 21:15:06 +0800 Subject: [PATCH 393/659] feat(sftp-server): public key login (#7668) --- internal/db/db.go | 2 +- internal/db/sshkey.go | 57 ++++++++++++++++++ internal/model/sshkey.go | 28 +++++++++ internal/op/sshkey.go | 48 +++++++++++++++ server/handles/sshkey.go | 124 +++++++++++++++++++++++++++++++++++++++ server/router.go | 5 ++ server/sftp.go | 27 ++++++++- 7 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 internal/db/sshkey.go create mode 100644 internal/model/sshkey.go create mode 100644 internal/op/sshkey.go create mode 100644 server/handles/sshkey.go diff --git a/internal/db/db.go b/internal/db/db.go index 2df58d3760b..2cd18050da9 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/sshkey.go b/internal/db/sshkey.go new file mode 100644 index 00000000000..f51dbfdc453 --- /dev/null +++ b/internal/db/sshkey.go @@ -0,0 +1,57 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" +) + +func GetSSHPublicKeyByUserId(userId uint, pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { + keyDB := db.Model(&model.SSHPublicKey{}) + query := model.SSHPublicKey{UserId: userId} + if err := keyDB.Where(query).Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get user's keys count") + } + if err := keyDB.Where(query).Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&keys).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get find user's keys") + } + return keys, count, nil +} + +func GetSSHPublicKeyById(id uint) (*model.SSHPublicKey, error) { + var k model.SSHPublicKey + if err := db.First(&k, id).Error; err != nil { + return nil, errors.Wrapf(err, "failed get old key") + } + return &k, nil +} + +func GetSSHPublicKeyByUserTitle(userId uint, title string) (*model.SSHPublicKey, error) { + key := model.SSHPublicKey{UserId: userId, Title: title} + if err := db.Where(key).First(&key).Error; err != nil { + return nil, errors.Wrapf(err, "failed find key with title of user") + } + return &key, nil +} + +func CreateSSHPublicKey(k *model.SSHPublicKey) error { + return errors.WithStack(db.Create(k).Error) +} + +func UpdateSSHPublicKey(k *model.SSHPublicKey) error { + return errors.WithStack(db.Save(k).Error) +} + +func GetSSHPublicKeys(pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { + keyDB := db.Model(&model.SSHPublicKey{}) + if err := keyDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get keys count") + } + if err := keyDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&keys).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get find keys") + } + return keys, count, nil +} + +func DeleteSSHPublicKeyById(id uint) error { + return errors.WithStack(db.Delete(&model.SSHPublicKey{}, id).Error) +} diff --git a/internal/model/sshkey.go b/internal/model/sshkey.go new file mode 100644 index 00000000000..6e97c103017 --- /dev/null +++ b/internal/model/sshkey.go @@ -0,0 +1,28 @@ +package model + +import ( + "golang.org/x/crypto/ssh" + "time" +) + +type SSHPublicKey struct { + ID uint `json:"id" gorm:"primaryKey"` + UserId uint `json:"-"` + Title string `json:"title"` + Fingerprint string `json:"fingerprint"` + KeyStr string `gorm:"type:text" json:"-"` + AddedTime time.Time `json:"added_time"` + LastUsedTime time.Time `json:"last_used_time"` +} + +func (k *SSHPublicKey) GetKey() (ssh.PublicKey, error) { + pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k.KeyStr)) + if err != nil { + return nil, err + } + return pubKey, nil +} + +func (k *SSHPublicKey) UpdateLastUsedTime() { + k.LastUsedTime = time.Now() +} diff --git a/internal/op/sshkey.go b/internal/op/sshkey.go new file mode 100644 index 00000000000..6ed55658abf --- /dev/null +++ b/internal/op/sshkey.go @@ -0,0 +1,48 @@ +package op + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "golang.org/x/crypto/ssh" + "time" +) + +func CreateSSHPublicKey(k *model.SSHPublicKey) (error, bool) { + _, err := db.GetSSHPublicKeyByUserTitle(k.UserId, k.Title) + if err == nil { + return errors.New("key with the same title already exists"), true + } + pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k.KeyStr)) + if err != nil { + return err, false + } + k.KeyStr = string(pubKey.Marshal()) + k.Fingerprint = ssh.FingerprintSHA256(pubKey) + k.AddedTime = time.Now() + k.LastUsedTime = k.AddedTime + return db.CreateSSHPublicKey(k), true +} + +func GetSSHPublicKeyByUserId(userId uint, pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { + return db.GetSSHPublicKeyByUserId(userId, pageIndex, pageSize) +} + +func GetSSHPublicKeyByIdAndUserId(id uint, userId uint) (*model.SSHPublicKey, error) { + key, err := db.GetSSHPublicKeyById(id) + if err != nil { + return nil, err + } + if key.UserId != userId { + return nil, errors.Wrapf(err, "failed get old key") + } + return key, nil +} + +func UpdateSSHPublicKey(k *model.SSHPublicKey) error { + return db.UpdateSSHPublicKey(k) +} + +func DeleteSSHPublicKeyById(keyId uint) error { + return db.DeleteSSHPublicKeyById(keyId) +} diff --git a/server/handles/sshkey.go b/server/handles/sshkey.go new file mode 100644 index 00000000000..c53b46f2932 --- /dev/null +++ b/server/handles/sshkey.go @@ -0,0 +1,124 @@ +package handles + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "strconv" +) + +type SSHKeyAddReq struct { + Title string `json:"title" binding:"required"` + Key string `json:"key" binding:"required"` +} + +func AddMyPublicKey(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok || userObj.IsGuest() { + common.ErrorStrResp(c, "user invalid", 401) + return + } + var req SSHKeyAddReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorStrResp(c, "request invalid", 400) + return + } + if req.Title == "" { + common.ErrorStrResp(c, "request invalid", 400) + return + } + key := &model.SSHPublicKey{ + Title: req.Title, + KeyStr: req.Key, + UserId: userObj.ID, + } + err, parsed := op.CreateSSHPublicKey(key) + if !parsed { + common.ErrorStrResp(c, "provided key invalid", 400) + return + } else if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func ListMyPublicKey(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok || userObj.IsGuest() { + common.ErrorStrResp(c, "user invalid", 401) + return + } + list(c, userObj) +} + +func DeleteMyPublicKey(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok || userObj.IsGuest() { + common.ErrorStrResp(c, "user invalid", 401) + return + } + keyId, err := strconv.Atoi(c.Query("id")) + if err != nil { + common.ErrorStrResp(c, "id format invalid", 400) + return + } + key, err := op.GetSSHPublicKeyByIdAndUserId(uint(keyId), userObj.ID) + if err != nil { + common.ErrorStrResp(c, "failed to get public key", 404) + return + } + err = op.DeleteSSHPublicKeyById(key.ID) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func ListPublicKeys(c *gin.Context) { + userId, err := strconv.Atoi(c.Query("uid")) + if err != nil { + common.ErrorStrResp(c, "user id format invalid", 400) + return + } + userObj, err := op.GetUserById(uint(userId)) + if err != nil { + common.ErrorStrResp(c, "user invalid", 404) + return + } + list(c, userObj) +} + +func DeletePublicKey(c *gin.Context) { + keyId, err := strconv.Atoi(c.Query("id")) + if err != nil { + common.ErrorStrResp(c, "id format invalid", 400) + return + } + err = op.DeleteSSHPublicKeyById(uint(keyId)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func list(c *gin.Context, userObj *model.User) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + keys, total, err := op.GetSSHPublicKeyByUserId(userObj.ID, req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, common.PageResp{ + Content: keys, + Total: total, + }) +} diff --git a/server/router.go b/server/router.go index fffa840e537..9ff50365a7a 100644 --- a/server/router.go +++ b/server/router.go @@ -52,6 +52,9 @@ func Init(e *gin.Engine) { api.POST("/auth/login/ldap", handles.LoginLdap) auth.GET("/me", handles.CurrentUser) auth.POST("/me/update", handles.UpdateCurrent) + auth.GET("/me/sshkey/list", handles.ListMyPublicKey) + auth.POST("/me/sshkey/add", handles.AddMyPublicKey) + auth.POST("/me/sshkey/delete", handles.DeleteMyPublicKey) auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) auth.GET("/auth/logout", handles.LogOut) @@ -102,6 +105,8 @@ func admin(g *gin.RouterGroup) { user.POST("/cancel_2fa", handles.Cancel2FAById) user.POST("/delete", handles.DeleteUser) user.POST("/del_cache", handles.DelUserCache) + user.GET("/sshkey/list", handles.ListPublicKeys) + user.POST("/sshkey/delete", handles.DeletePublicKey) storage := g.Group("/storage") storage.GET("/list", handles.ListStorages) diff --git a/server/sftp.go b/server/sftp.go index 3b07d472c37..d44046a42d7 100644 --- a/server/sftp.go +++ b/server/sftp.go @@ -12,6 +12,7 @@ import ( "github.com/pkg/errors" "golang.org/x/crypto/ssh" "net/http" + "time" ) type SftpDriver struct { @@ -35,6 +36,7 @@ func (d *SftpDriver) GetConfig() *sftpd.Config { NoClientAuth: true, NoClientAuthCallback: d.NoClientAuth, PasswordCallback: d.PasswordAuth, + PublicKeyCallback: d.PublicKeyAuth, AuthLogCallback: d.AuthLogCallback, BannerCallback: d.GetBanner, } @@ -85,14 +87,37 @@ func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh. if err != nil { return nil, err } + if userObj.Disabled || !userObj.CanFTPAccess() { + return nil, errors.New("user is not allowed to access via SFTP") + } passHash := model.StaticHash(string(password)) if err = userObj.ValidatePwdStaticHash(passHash); err != nil { return nil, err } + return nil, nil +} + +func (d *SftpDriver) PublicKeyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { + userObj, err := op.GetUserByName(conn.User()) + if err != nil { + return nil, err + } if userObj.Disabled || !userObj.CanFTPAccess() { return nil, errors.New("user is not allowed to access via SFTP") } - return nil, nil + keys, _, err := op.GetSSHPublicKeyByUserId(userObj.ID, 1, -1) + if err != nil { + return nil, err + } + marshal := string(key.Marshal()) + for _, sk := range keys { + if marshal == sk.KeyStr { + sk.LastUsedTime = time.Now() + _ = op.UpdateSSHPublicKey(&sk) + return nil, nil + } + } + return nil, errors.New("public key refused") } func (d *SftpDriver) AuthLogCallback(conn ssh.ConnMetadata, method string, err error) { From c218b5701e72a941958aa2b681a5f2e7e1d10519 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Wed, 25 Dec 2024 21:16:03 +0800 Subject: [PATCH 394/659] fix(115): support float QPS (#7677) --- drivers/115/meta.go | 2 +- drivers/115_share/meta.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/115/meta.go b/drivers/115/meta.go index 3b192291a43..bcea174922c 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -10,7 +10,7 @@ type Addition struct { QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` - LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate ([limit]r/1s)"` + LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"` driver.RootID } diff --git a/drivers/115_share/meta.go b/drivers/115_share/meta.go index 3fcc7b92133..b3d2cc1fad7 100644 --- a/drivers/115_share/meta.go +++ b/drivers/115_share/meta.go @@ -10,7 +10,7 @@ type Addition struct { QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` - LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` + LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"` ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"` driver.RootID From 5ecf5e823c6410766843e47c116a101c15df088d Mon Sep 17 00:00:00 2001 From: "Feng.YJ" <32027253+huiyifyj@users.noreply.github.com> Date: Wed, 25 Dec 2024 21:16:34 +0800 Subject: [PATCH 395/659] fix(webauthn): handle error when removing webauthn credential (#7689) --- server/handles/webauthn.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index 1bd1884ef11..c6a7650c991 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -207,6 +207,10 @@ func DeleteAuthnLogin(c *gin.Context) { return } err = db.RemoveAuthn(user, req.ID) + if err != nil { + common.ErrorResp(c, err, 400) + return + } err = op.DelUserCache(user.Username) if err != nil { common.ErrorResp(c, err, 400) From 48916cdedff1bcff2a0e05c8e672246169e3c4ee Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Wed, 25 Dec 2024 21:17:58 +0800 Subject: [PATCH 396/659] fix(permission): enhance the strictness of permissions (#7705 close #7680) * fix(permission): enhance the strictness of permissions * fix: add initial permissions to admin --- internal/bootstrap/data/user.go | 13 +++++----- internal/model/user.go | 42 +++++++++++++++++---------------- server/webdav.go | 28 ++++++++++++++++------ server/webdav/file.go | 7 ++++++ 4 files changed, 57 insertions(+), 33 deletions(-) diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 3b71e498206..37cba7a5e74 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -26,12 +26,13 @@ func initUser() { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) admin = &model.User{ - Username: "admin", - Salt: salt, - PwdHash: model.TwoHashPwd(adminPassword, salt), - Role: model.ADMIN, - BasePath: "/", - Authn: "[]", + Username: "admin", + Salt: salt, + PwdHash: model.TwoHashPwd(adminPassword, salt), + Role: model.ADMIN, + BasePath: "/", + Authn: "[]", + Permission: 0xFF, // 0(can see hidden) - 7(can remove) } if err := op.CreateUser(admin); err != nil { panic(err) diff --git a/internal/model/user.go b/internal/model/user.go index b4e876a47ab..f75fc6875c5 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -32,16 +32,18 @@ type User struct { Role int `json:"role"` // user's role Disabled bool `json:"disabled"` // Determine permissions by bit - // 0: can see hidden files - // 1: can access without password - // 2: can add offline download tasks - // 3: can mkdir and upload - // 4: can rename - // 5: can move - // 6: can copy - // 7: can remove - // 8: webdav read - // 9: webdav write + // 0: can see hidden files + // 1: can access without password + // 2: can add offline download tasks + // 3: can mkdir and upload + // 4: can rename + // 5: can move + // 6: can copy + // 7: can remove + // 8: webdav read + // 9: webdav write + // 10: ftp/sftp login and read + // 11: ftp/sftp write Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -78,43 +80,43 @@ func (u *User) SetPassword(pwd string) *User { } func (u *User) CanSeeHides() bool { - return u.IsAdmin() || u.Permission&1 == 1 + return u.Permission&1 == 1 } func (u *User) CanAccessWithoutPassword() bool { - return u.IsAdmin() || (u.Permission>>1)&1 == 1 + return (u.Permission>>1)&1 == 1 } func (u *User) CanAddOfflineDownloadTasks() bool { - return u.IsAdmin() || (u.Permission>>2)&1 == 1 + return (u.Permission>>2)&1 == 1 } func (u *User) CanWrite() bool { - return u.IsAdmin() || (u.Permission>>3)&1 == 1 + return (u.Permission>>3)&1 == 1 } func (u *User) CanRename() bool { - return u.IsAdmin() || (u.Permission>>4)&1 == 1 + return (u.Permission>>4)&1 == 1 } func (u *User) CanMove() bool { - return u.IsAdmin() || (u.Permission>>5)&1 == 1 + return (u.Permission>>5)&1 == 1 } func (u *User) CanCopy() bool { - return u.IsAdmin() || (u.Permission>>6)&1 == 1 + return (u.Permission>>6)&1 == 1 } func (u *User) CanRemove() bool { - return u.IsAdmin() || (u.Permission>>7)&1 == 1 + return (u.Permission>>7)&1 == 1 } func (u *User) CanWebdavRead() bool { - return u.IsAdmin() || (u.Permission>>8)&1 == 1 + return (u.Permission>>8)&1 == 1 } func (u *User) CanWebdavManage() bool { - return u.IsAdmin() || (u.Permission>>9)&1 == 1 + return (u.Permission>>9)&1 == 1 } func (u *User) CanFTPAccess() bool { diff --git a/server/webdav.go b/server/webdav.go index 2b5c9618b86..cdfdce7d9d3 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -11,7 +11,6 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/webdav" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -99,12 +98,27 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if !user.CanWebdavManage() && utils.SliceContains([]string{"PUT", "DELETE", "PROPPATCH", "MKCOL", "COPY", "MOVE"}, c.Request.Method) { - if c.Request.Method == "OPTIONS" { - c.Set("user", guest) - c.Next() - return - } + if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!user.CanWebdavManage() || !user.CanWrite()) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "MOVE" && (!user.CanWebdavManage() || (!user.CanMove() && !user.CanRename())) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "COPY" && (!user.CanWebdavManage() || !user.CanCopy()) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "DELETE" && (!user.CanWebdavManage() || !user.CanRemove()) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "PROPPATCH" && !user.CanWebdavManage() { c.Status(http.StatusForbidden) c.Abort() return diff --git a/server/webdav/file.go b/server/webdav/file.go index 01e96f7d223..ac8f5c1cbfb 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -33,6 +33,13 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int dstDir := path.Dir(dst) srcName := path.Base(src) dstName := path.Base(dst) + user := ctx.Value("user").(*model.User) + if srcDir != dstDir && !user.CanMove() { + return http.StatusForbidden, nil + } + if srcName != dstName && !user.CanRename() { + return http.StatusForbidden, nil + } if srcDir == dstDir { err = fs.Rename(ctx, src, dstName) } else { From 42243b1517d32e4ee4fa7bc6cad69b5e9bb2fdfa Mon Sep 17 00:00:00 2001 From: Jealous Date: Wed, 25 Dec 2024 21:23:58 +0800 Subject: [PATCH 397/659] feat(thunder): add offline download tool (#7673) * feat(thunder): add offline download tool * fix(thunder): improve error handling and parse file size in status response --------- Co-authored-by: Andy Hsu --- drivers/thunder/driver.go | 61 +++++++++ drivers/thunder/types.go | 47 +++++++ drivers/thunder/util.go | 1 + internal/offline_download/all.go | 1 + internal/offline_download/thunder/thunder.go | 126 +++++++++++++++++++ internal/offline_download/thunder/util.go | 42 +++++++ internal/offline_download/tool/add.go | 4 + internal/offline_download/tool/download.go | 6 + 8 files changed, 288 insertions(+) create mode 100644 internal/offline_download/thunder/thunder.go create mode 100644 internal/offline_download/thunder/util.go diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 9ba5dd825f7..8403f2617a6 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strconv" "strings" "github.com/alist-org/alist/v3/drivers/base" @@ -522,3 +523,63 @@ func (xc *XunLeiCommon) IsLogin() bool { _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) return err == nil } + +// 离线下载文件 +func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + var resp OfflineDownloadResp + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FILE, + "name": fileName, + "parent_id": parentDir.GetID(), + "upload_type": UPLOAD_TYPE_URL, + "url": base.Json{ + "url": fileUrl, + }, + }) + }, &resp) + + if err != nil { + return nil, err + } + + return &resp.Task, err +} + +/* +获取离线下载任务列表 +*/ +func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) ([]OfflineTask, error) { + res := make([]OfflineTask, 0) + + var resp OfflineListResp + _, err := xc.Request(TASK_API_URL, http.MethodGet, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(map[string]string{ + "type": "offline", + "limit": "10000", + "page_token": nextPageToken, + }) + }, &resp) + + if err != nil { + return nil, fmt.Errorf("failed to get offline list: %w", err) + } + res = append(res, resp.Tasks...) + return res, nil +} + +func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + _, err := xc.Request(TASK_API_URL, http.MethodDelete, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(map[string]string{ + "task_ids": strings.Join(taskIDs, ","), + "delete_files": strconv.FormatBool(deleteFiles), + }) + }, nil) + if err != nil { + return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) + } + return nil +} diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index 7c223673448..b7355b2a6fa 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -204,3 +204,50 @@ type UploadTaskResponse struct { File Files `json:"file"` } + +// 添加离线下载响应 +type OfflineDownloadResp struct { + File *string `json:"file"` + Task OfflineTask `json:"task"` + UploadType string `json:"upload_type"` + URL struct { + Kind string `json:"kind"` + } `json:"url"` +} + +// 离线下载列表 +type OfflineListResp struct { + ExpiresIn int64 `json:"expires_in"` + NextPageToken string `json:"next_page_token"` + Tasks []OfflineTask `json:"tasks"` +} + +// offlineTask +type OfflineTask struct { + Callback string `json:"callback"` + CreatedTime string `json:"created_time"` + FileID string `json:"file_id"` + FileName string `json:"file_name"` + FileSize string `json:"file_size"` + IconLink string `json:"icon_link"` + ID string `json:"id"` + Kind string `json:"kind"` + Message string `json:"message"` + Name string `json:"name"` + Params Params `json:"params"` + Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING + Progress int64 `json:"progress"` + Space string `json:"space"` + StatusSize int64 `json:"status_size"` + Statuses []string `json:"statuses"` + ThirdTaskID string `json:"third_task_id"` + Type string `json:"type"` + UpdatedTime string `json:"updated_time"` + UserID string `json:"user_id"` +} + +type Params struct { + FolderType string `json:"folder_type"` + PredictSpeed string `json:"predict_speed"` + PredictType string `json:"predict_type"` +} diff --git a/drivers/thunder/util.go b/drivers/thunder/util.go index 3ec8db58ffe..f509e6b2fbc 100644 --- a/drivers/thunder/util.go +++ b/drivers/thunder/util.go @@ -17,6 +17,7 @@ import ( const ( API_URL = "https://api-pan.xunlei.com/drive/v1" FILE_API_URL = API_URL + "/files" + TASK_API_URL = API_URL + "/tasks" XLUSER_API_URL = "https://xluser-ssl.xunlei.com/v1" ) diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 6682155dec8..3d0c7c73a0b 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -6,5 +6,6 @@ import ( _ "github.com/alist-org/alist/v3/internal/offline_download/http" _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" + _ "github.com/alist-org/alist/v3/internal/offline_download/thunder" _ "github.com/alist-org/alist/v3/internal/offline_download/transmission" ) diff --git a/internal/offline_download/thunder/thunder.go b/internal/offline_download/thunder/thunder.go new file mode 100644 index 00000000000..3ab8b00212b --- /dev/null +++ b/internal/offline_download/thunder/thunder.go @@ -0,0 +1,126 @@ +package thunder + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/alist-org/alist/v3/drivers/thunder" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" +) + +type Thunder struct { + refreshTaskCache bool +} + +func (t *Thunder) Name() string { + return "thunder" +} + +func (t *Thunder) Items() []model.SettingItem { + return nil +} + +func (t *Thunder) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (t *Thunder) Init() (string, error) { + t.refreshTaskCache = false + return "ok", nil +} + +func (t *Thunder) IsReady() bool { + return true +} + +func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { + // 添加新任务刷新缓存 + t.refreshTaskCache = true + // args.TempDir 已经被修改为了 DstDirPath + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + + ctx := context.Background() + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + + task, err := thunderDriver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + + return task.ID, nil +} + +func (t *Thunder) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + if err != nil { + return err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + ctx := context.Background() + err = thunderDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false) + if err != nil { + return err + } + return nil +} + +func (t *Thunder) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + if err != nil { + return nil, err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + tasks, err := t.GetTasks(thunderDriver) + if err != nil { + return nil, err + } + s := &tool.Status{ + Progress: 0, + NewGID: "", + Completed: false, + Status: "the task has been deleted", + Err: nil, + } + for _, t := range tasks { + if t.ID == task.GID { + s.Progress = float64(t.Progress) + s.Status = t.Message + s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE") + s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) + if err != nil { + s.TotalBytes = 0 + } + if t.Phase == "PHASE_TYPE_ERROR" { + s.Err = errors.New(t.Message) + } + return s, nil + } + } + s.Err = fmt.Errorf("the task has been deleted") + return s, nil +} + +func init() { + tool.Tools.Add(&Thunder{}) +} diff --git a/internal/offline_download/thunder/util.go b/internal/offline_download/thunder/util.go new file mode 100644 index 00000000000..ea400f321d0 --- /dev/null +++ b/internal/offline_download/thunder/util.go @@ -0,0 +1,42 @@ +package thunder + +import ( + "context" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/drivers/thunder" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]thunder.OfflineTask](16)) +var taskG singleflight.Group[[]thunder.OfflineTask] + +func (t *Thunder) GetTasks(thunderDriver *thunder.Thunder) ([]thunder.OfflineTask, error) { + key := op.Key(thunderDriver, "/drive/v1/task") + if !t.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + t.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]thunder.OfflineTask, error) { + ctx := context.Background() + tasks, err := thunderDriver.OfflineList(ctx, "") + if err != nil { + return nil, err + } + // 添加缓存 10s + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]thunder.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 42349e2e397..4158051a8f0 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -77,6 +77,10 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro tempDir = args.DstDirPath // 防止将下载好的文件删除 deletePolicy = DeleteNever + case "thunder": + tempDir = args.DstDirPath + // 防止将下载好的文件删除 + deletePolicy = DeleteNever } taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index a0f1a81b3b5..94bf7dbb660 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -83,6 +83,9 @@ outer: if t.tool.Name() == "pikpak" { return nil } + if t.tool.Name() == "thunder" { + return nil + } if t.tool.Name() == "115 Cloud" { // hack for 115 <-time.After(time.Second * 1) @@ -161,6 +164,9 @@ func (t *DownloadTask) Complete() error { if t.tool.Name() == "pikpak" { return nil } + if t.tool.Name() == "thunder" { + return nil + } if t.tool.Name() == "115 Cloud" { return nil } From 5994c17b4efca630d8ecec88475f2f243781d615 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Mon, 30 Dec 2024 22:48:33 +0800 Subject: [PATCH 398/659] feat(patch): upgrade patch module (#7738) * feat(patch): upgrade patch module * chore(patch): add docs * fix(patch): skip and rewrite invalid last launched version * fix(patch): turn two functions into patches --- cmd/common.go | 1 + internal/bootstrap/config.go | 6 ++ internal/bootstrap/data/user.go | 35 ---------- internal/bootstrap/patch.go | 67 +++++++++++++++++++ internal/bootstrap/patch/all.go | 35 ++++++++++ .../bootstrap/patch/v3_24_0/hash_password.go | 26 +++++++ .../bootstrap/patch/v3_32_0/update_authn.go | 25 +++++++ .../patch/v3_41_0/grant_permission.go | 22 ++++++ internal/conf/config.go | 2 + 9 files changed, 184 insertions(+), 35 deletions(-) create mode 100644 internal/bootstrap/patch.go create mode 100644 internal/bootstrap/patch/all.go create mode 100644 internal/bootstrap/patch/v3_24_0/hash_password.go create mode 100644 internal/bootstrap/patch/v3_32_0/update_authn.go create mode 100644 internal/bootstrap/patch/v3_41_0/grant_permission.go diff --git a/cmd/common.go b/cmd/common.go index fabc3a90f1b..beb558f510f 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -19,6 +19,7 @@ func Init() { bootstrap.InitDB() data.InitData() bootstrap.InitIndex() + bootstrap.InitUpgradePatch() } func Release() { diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 27174c23633..a44c735056b 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -34,6 +34,8 @@ func InitConfig() { log.Fatalf("failed to create config file: %+v", err) } conf.Conf = conf.DefaultConfig() + LastLaunchedVersion = conf.Version + conf.Conf.LastLaunchedVersion = conf.Version if !utils.WriteJsonToFile(configPath, conf.Conf) { log.Fatalf("failed to create default config file") } @@ -47,6 +49,10 @@ func InitConfig() { if err != nil { log.Fatalf("load config error: %+v", err) } + LastLaunchedVersion = conf.Conf.LastLaunchedVersion + if conf.Version != "dev" || LastLaunchedVersion == "" { + conf.Conf.LastLaunchedVersion = conf.Version + } // update config.json struct confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") if err != nil { diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 37cba7a5e74..5b596a85852 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -64,39 +64,4 @@ func initUser() { utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) } } - hashPwdForOldVersion() - updateAuthnForOldVersion() -} - -func hashPwdForOldVersion() { - users, _, err := op.GetUsers(1, -1) - if err != nil { - utils.Log.Fatalf("[hash pwd for old version] failed get users: %v", err) - } - for i := range users { - user := users[i] - if user.PwdHash == "" { - user.SetPassword(user.Password) - user.Password = "" - if err := db.UpdateUser(&user); err != nil { - utils.Log.Fatalf("[hash pwd for old version] failed update user: %v", err) - } - } - } -} - -func updateAuthnForOldVersion() { - users, _, err := op.GetUsers(1, -1) - if err != nil { - utils.Log.Fatalf("[update authn for old version] failed get users: %v", err) - } - for i := range users { - user := users[i] - if user.Authn == "" { - user.Authn = "[]" - if err := db.UpdateUser(&user); err != nil { - utils.Log.Fatalf("[update authn for old version] failed update user: %v", err) - } - } - } } diff --git a/internal/bootstrap/patch.go b/internal/bootstrap/patch.go new file mode 100644 index 00000000000..8dc3ed02745 --- /dev/null +++ b/internal/bootstrap/patch.go @@ -0,0 +1,67 @@ +package bootstrap + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/bootstrap/patch" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var LastLaunchedVersion = "" + +func safeCall(v string, i int, f func()) { + defer func() { + if r := recover(); r != nil { + utils.Log.Errorf("Recovered from patch (version: %s, index: %d) panic: %v", v, i, r) + } + }() + + f() +} + +func getVersion(v string) (major, minor, patchNum int, err error) { + _, err = fmt.Sscanf(v, "v%d.%d.%d", &major, &minor, &patchNum) + return major, minor, patchNum, err +} + +func compareVersion(majorA, minorA, patchNumA, majorB, minorB, patchNumB int) bool { + if majorA != majorB { + return majorA > majorB + } + if minorA != minorB { + return minorA > minorB + } + if patchNumA != patchNumB { + return patchNumA > patchNumB + } + return true +} + +func InitUpgradePatch() { + if conf.Version == "dev" { + return + } + if LastLaunchedVersion == conf.Version { + return + } + if LastLaunchedVersion == "" { + LastLaunchedVersion = "v0.0.0" + } + major, minor, patchNum, err := getVersion(LastLaunchedVersion) + if err != nil { + utils.Log.Warnf("Failed to parse last launched version %s: %v, skipping all patches and rewrite last launched version", LastLaunchedVersion, err) + return + } + for _, vp := range patch.UpgradePatches { + ma, mi, pn, err := getVersion(vp.Version) + if err != nil { + utils.Log.Errorf("Skip invalid version %s patches: %v", vp.Version, err) + continue + } + if compareVersion(ma, mi, pn, major, minor, patchNum) { + for i, p := range vp.Patches { + safeCall(vp.Version, i, p) + } + } + } +} diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go new file mode 100644 index 00000000000..b363d12981d --- /dev/null +++ b/internal/bootstrap/patch/all.go @@ -0,0 +1,35 @@ +package patch + +import ( + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_24_0" + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_32_0" + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_41_0" +) + +type VersionPatches struct { + // Version means if the system is upgraded from Version or an earlier one + // to the current version, all patches in Patches will be executed. + Version string + Patches []func() +} + +var UpgradePatches = []VersionPatches{ + { + Version: "v3.24.0", + Patches: []func(){ + v3_24_0.HashPwdForOldVersion, + }, + }, + { + Version: "v3.32.0", + Patches: []func(){ + v3_32_0.UpdateAuthnForOldVersion, + }, + }, + { + Version: "v3.41.0", + Patches: []func(){ + v3_41_0.GrantAdminPermissions, + }, + }, +} diff --git a/internal/bootstrap/patch/v3_24_0/hash_password.go b/internal/bootstrap/patch/v3_24_0/hash_password.go new file mode 100644 index 00000000000..2adb640d8c9 --- /dev/null +++ b/internal/bootstrap/patch/v3_24_0/hash_password.go @@ -0,0 +1,26 @@ +package v3_24_0 + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// HashPwdForOldVersion encode passwords using SHA256 +// First published: 75acbcc perf: sha256 for user's password (close #3552) by Andy Hsu +func HashPwdForOldVersion() { + users, _, err := op.GetUsers(1, -1) + if err != nil { + utils.Log.Fatalf("[hash pwd for old version] failed get users: %v", err) + } + for i := range users { + user := users[i] + if user.PwdHash == "" { + user.SetPassword(user.Password) + user.Password = "" + if err := db.UpdateUser(&user); err != nil { + utils.Log.Fatalf("[hash pwd for old version] failed update user: %v", err) + } + } + } +} diff --git a/internal/bootstrap/patch/v3_32_0/update_authn.go b/internal/bootstrap/patch/v3_32_0/update_authn.go new file mode 100644 index 00000000000..92a594fda78 --- /dev/null +++ b/internal/bootstrap/patch/v3_32_0/update_authn.go @@ -0,0 +1,25 @@ +package v3_32_0 + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// UpdateAuthnForOldVersion updates users' authn +// First published: bdfc159 fix: webauthn logspam (#6181) by itsHenry +func UpdateAuthnForOldVersion() { + users, _, err := op.GetUsers(1, -1) + if err != nil { + utils.Log.Fatalf("[update authn for old version] failed get users: %v", err) + } + for i := range users { + user := users[i] + if user.Authn == "" { + user.Authn = "[]" + if err := db.UpdateUser(&user); err != nil { + utils.Log.Fatalf("[update authn for old version] failed update user: %v", err) + } + } + } +} diff --git a/internal/bootstrap/patch/v3_41_0/grant_permission.go b/internal/bootstrap/patch/v3_41_0/grant_permission.go new file mode 100644 index 00000000000..d658d184d4d --- /dev/null +++ b/internal/bootstrap/patch/v3_41_0/grant_permission.go @@ -0,0 +1,22 @@ +package v3_41_0 + +import ( + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// GrantAdminPermissions gives admin Permission 0(can see hidden) - 9(webdav manage) +// This patch is written to help users upgrading from older version better adapt to PR AlistGo/alist#7705. +func GrantAdminPermissions() { + admin, err := op.GetAdmin() + if err != nil { + utils.Log.Errorf("Cannot grant permissions to admin: %v", err) + } + if (admin.Permission & 0x3FF) == 0 { + admin.Permission |= 0x3FF + } + err = op.UpdateUser(admin) + if err != nil { + utils.Log.Errorf("Cannot grant permissions to admin: %v", err) + } +} diff --git a/internal/conf/config.go b/internal/conf/config.go index a9b38242e25..d015cda0a91 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -110,6 +110,7 @@ type Config struct { S3 S3 `json:"s3" envPrefix:"S3_"` FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` + LastLaunchedVersion string `json:"last_launched_version"` } func DefaultConfig() *Config { @@ -195,5 +196,6 @@ func DefaultConfig() *Config { Enable: false, Listen: ":5222", }, + LastLaunchedVersion: "", } } From 365fc40dfec03411d3d8231090495761e012c525 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:49:18 +0800 Subject: [PATCH 399/659] fix: static page to limit request method (#7745 close #7667) --- server/static/static.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/static/static.go b/server/static/static.go index ec16014c22b..d5d6ff685cd 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -102,6 +102,10 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { } noRoute(func(c *gin.Context) { + if c.Request.Method != "GET" && c.Request.Method != "POST" { + c.Status(405) + return + } c.Header("Content-Type", "text/html") c.Status(200) if strings.HasPrefix(c.Request.URL.Path, "/@manage") { From 4dce53d72b0d201ce7cf0e88afebbd7b843f6756 Mon Sep 17 00:00:00 2001 From: Mmx <36563672+Mmx233@users.noreply.github.com> Date: Mon, 30 Dec 2024 22:51:05 +0800 Subject: [PATCH 400/659] feat(docker release): improve aria2 image, add aio image (#7750) * build: add argument INSTALL_ARIA2 to dockerfile * feat: run aria2 in main entrypoint * feat(ci): environment matrix for docker release * improve(ci): allow overwrite artifacts in docker release * fix(ci): permission of alist binary in docker; entrypoint logic * improve(aria2): move aria2 data to /opt/aria2; fix permission issues References: https://github.com/AlistGo/with_aria2/pull/13 Co-authored-by: GoodbyeNJN * fix(ci): aio image is not taking effect * fix(build): tar command in aria2 installation process (cherry picked from commit 647285408354807bae64df6a20fefb696ff787de) --------- Co-authored-by: GoodbyeNJN --- .github/workflows/release_docker.yml | 116 ++++++++++++++------------- Dockerfile | 17 +++- Dockerfile.ci | 17 +++- entrypoint.sh | 12 ++- 4 files changed, 100 insertions(+), 62 deletions(-) diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index a2dd2dd72d8..0f559a3f415 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -5,9 +5,16 @@ on: tags: - 'v*' +env: + IMAGE_REGISTRY: 'xhofe/alist' + REGISTRY_USERNAME: 'xhofe' + REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + ARTIFACT_NAME: 'binaries_docker_release' + RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64' + jobs: - release_docker: - name: Release Docker + build_binary: + name: Build Binaries for Docker Release runs-on: ubuntu-latest steps: - name: Checkout @@ -31,11 +38,45 @@ jobs: - name: Build go binary run: bash build.sh release docker-multiplatform - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 + - name: Upload artifacts + uses: actions/upload-artifact@v4 with: - images: xhofe/alist + name: ${{ env.ARTIFACT_NAME }} + overwrite: true + path: | + build/ + !build/*.tgz + !build/musl-libs/** + + release_docker: + needs: build_binary + name: Release Docker image + runs-on: ubuntu-latest + strategy: + matrix: + image: ["latest", "ffmpeg", "aria2", "aio"] + include: + - image: "latest" + build_arg: "" + tag_favor: "" + - image: "ffmpeg" + build_arg: INSTALL_FFMPEG=true + tag_favor: "suffix=-ffmpeg,onlatest=true" + - image: "aria2" + build_arg: INSTALL_ARIA2=true + tag_favor: "suffix=-aria2,onlatest=true" + - image: "aio" + build_arg: | + INSTALL_FFMPEG=true + INSTALL_ARIA2=true + tag_favor: "suffix=-aio,onlatest=true" + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: 'build/' - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -46,63 +87,26 @@ jobs: - name: Login to DockerHub uses: docker/login-action@v3 with: - username: xhofe - password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ env.REGISTRY_USERNAME }} + password: ${{ env.REGISTRY_PASSWORD }} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.ci - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64 - - - name: Docker meta with ffmpeg - id: meta-ffmpeg + - name: Docker meta + id: meta uses: docker/metadata-action@v5 with: - images: xhofe/alist + images: ${{ env.IMAGE_REGISTRY }} flavor: | latest=true - suffix=-ffmpeg,onlatest=true - - - name: Build and push with ffmpeg - id: docker_build_ffmpeg + ${{ matrix.tag_favor }} + + - name: Build and push + id: docker_build uses: docker/build-push-action@v6 with: context: . file: Dockerfile.ci push: true - tags: ${{ steps.meta-ffmpeg.outputs.tags }} - labels: ${{ steps.meta-ffmpeg.outputs.labels }} - build-args: INSTALL_FFMPEG=true - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64 - - release_docker_with_aria2: - needs: release_docker - name: Release docker with aria2 - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - repository: alist-org/with_aria2 - ref: main - persist-credentials: false - fetch-depth: 0 - - - name: Add tag - run: | - git config --local user.email "bot@nn.ci" - git config --local user.name "IlaBot" - git tag -a ${{ github.ref_name }} -m "release ${{ github.ref_name }}" - - - name: Push tags - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.MY_TOKEN }} - branch: main - repository: alist-org/with_aria2 + build-args: ${{ matrix.build_arg }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: ${{ env.RELEASE_PLATFORMS }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 74fa2165482..0e2ee96fb37 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,7 @@ RUN bash build.sh release docker FROM alpine:edge ARG INSTALL_FFMPEG=false +ARG INSTALL_ARIA2=false LABEL MAINTAINER="i@nn.ci" WORKDIR /opt/alist/ @@ -18,13 +19,25 @@ RUN apk update && \ apk upgrade --no-cache && \ apk add --no-cache bash ca-certificates su-exec tzdata; \ [ "$INSTALL_FFMPEG" = "true" ] && apk add --no-cache ffmpeg; \ + [ "$INSTALL_ARIA2" = "true" ] && apk add --no-cache curl aria2 && \ + mkdir -p /opt/aria2/.aria2 && \ + wget https://github.com/P3TERX/aria2.conf/archive/refs/heads/master.tar.gz -O /tmp/aria-conf.tar.gz && \ + tar -zxvf /tmp/aria-conf.tar.gz -C /opt/aria2/.aria2 --strip-components=1 && rm -f /tmp/aria-conf.tar.gz && \ + sed -i 's|rpc-secret|#rpc-secret|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/script.conf && \ + sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/script.conf && \ + touch /opt/aria2/.aria2/aria2.session && \ + /opt/aria2/.aria2/tracker.sh ; \ rm -rf /var/cache/apk/* COPY --from=builder /app/bin/alist ./ COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh && /entrypoint.sh version +RUN chmod +x /opt/alist/alist && \ + chmod +x /entrypoint.sh && /entrypoint.sh version -ENV PUID=0 PGID=0 UMASK=022 +ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/alist/data/ EXPOSE 5244 5245 CMD [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/Dockerfile.ci b/Dockerfile.ci index 3f437f16dfe..25d502a90aa 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -2,6 +2,7 @@ FROM alpine:edge ARG TARGETPLATFORM ARG INSTALL_FFMPEG=false +ARG INSTALL_ARIA2=false LABEL MAINTAINER="i@nn.ci" WORKDIR /opt/alist/ @@ -10,13 +11,25 @@ RUN apk update && \ apk upgrade --no-cache && \ apk add --no-cache bash ca-certificates su-exec tzdata; \ [ "$INSTALL_FFMPEG" = "true" ] && apk add --no-cache ffmpeg; \ + [ "$INSTALL_ARIA2" = "true" ] && apk add --no-cache curl aria2 && \ + mkdir -p /opt/aria2/.aria2 && \ + wget https://github.com/P3TERX/aria2.conf/archive/refs/heads/master.tar.gz -O /tmp/aria-conf.tar.gz && \ + tar -zxvf /tmp/aria-conf.tar.gz -C /opt/aria2/.aria2 --strip-components=1 && rm -f /tmp/aria-conf.tar.gz && \ + sed -i 's|rpc-secret|#rpc-secret|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/script.conf && \ + sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/script.conf && \ + touch /opt/aria2/.aria2/aria2.session && \ + /opt/aria2/.aria2/tracker.sh ; \ rm -rf /var/cache/apk/* COPY /build/${TARGETPLATFORM}/alist ./ COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh && /entrypoint.sh version +RUN chmod +x /opt/alist/alist && \ + chmod +x /entrypoint.sh && /entrypoint.sh version -ENV PUID=0 PGID=0 UMASK=022 +ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/alist/data/ EXPOSE 5244 5245 CMD [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index a0d8083509e..28a18d7d54c 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,11 +1,19 @@ #!/bin/bash -chown -R ${PUID}:${PGID} /opt/alist/ - umask ${UMASK} if [ "$1" = "version" ]; then ./alist version else + if [ "$RUN_ARIA2" = "true" ]; then + chown -R ${PUID}:${PGID} /opt/aria2/ + exec su-exec ${PUID}:${PGID} nohup aria2c \ + --enable-rpc \ + --rpc-allow-origin-all \ + --conf-path=/opt/aria2/.aria2/aria2.conf \ + >/dev/null 2>&1 & + fi + + chown -R ${PUID}:${PGID} /opt/alist/ exec su-exec ${PUID}:${PGID} ./alist server --no-prefix fi \ No newline at end of file From 040dc14ee626491ccbd9db29b06aa2713c0ae758 Mon Sep 17 00:00:00 2001 From: Sakana Date: Mon, 30 Dec 2024 22:51:39 +0800 Subject: [PATCH 401/659] fix(lenovonas_share): stoken expire (#7727) --- drivers/lenovonas_share/driver.go | 44 ++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/drivers/lenovonas_share/driver.go b/drivers/lenovonas_share/driver.go index 12e8514325f..684a2dda647 100644 --- a/drivers/lenovonas_share/driver.go +++ b/drivers/lenovonas_share/driver.go @@ -3,6 +3,7 @@ package LenovoNasShare import ( "context" "net/http" + "time" "github.com/go-resty/resty/v2" @@ -15,7 +16,8 @@ import ( type LenovoNasShare struct { model.Storage Addition - stoken string + stoken string + expireAt int64 } func (d *LenovoNasShare) Config() driver.Config { @@ -27,20 +29,9 @@ func (d *LenovoNasShare) GetAddition() driver.Additional { } func (d *LenovoNasShare) Init(ctx context.Context) error { - if d.Host == "" { - d.Host = "https://siot-share.lenovo.com.cn" - } - query := map[string]string{ - "code": d.ShareId, - "password": d.SharePwd, - } - resp, err := d.request(d.Host+"/oneproxy/api/share/v1/access", http.MethodGet, func(req *resty.Request) { - req.SetQueryParams(query) - }, nil) - if err != nil { + if err := d.getStoken(); err != nil { return err } - d.stoken = utils.Json.Get(resp, "data", "stoken").ToString() return nil } @@ -49,6 +40,7 @@ func (d *LenovoNasShare) Drop(ctx context.Context) error { } func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + d.checkStoken() // 检查stoken是否过期 files := make([]File, 0) var resp Files @@ -71,7 +63,33 @@ func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.Lis }) } +func (d *LenovoNasShare) checkStoken() { // 检查stoken是否过期 + if d.expireAt < time.Now().Unix() { + d.getStoken() + } +} + +func (d *LenovoNasShare) getStoken() error { // 获取stoken + if d.Host == "" { + d.Host = "https://siot-share.lenovo.com.cn" + } + query := map[string]string{ + "code": d.ShareId, + "password": d.SharePwd, + } + resp, err := d.request(d.Host+"/oneproxy/api/share/v1/access", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, nil) + if err != nil { + return err + } + d.stoken = utils.Json.Get(resp, "data", "stoken").ToString() + d.expireAt = utils.Json.Get(resp, "data", "expires_in").ToInt64() + time.Now().Unix() - 60 + return nil +} + func (d *LenovoNasShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + d.checkStoken() // 检查stoken是否过期 query := map[string]string{ "code": d.ShareId, "stoken": d.stoken, From ed149be84b75f03147f8c05e6a40a5daf45570d9 Mon Sep 17 00:00:00 2001 From: Jealous Date: Mon, 30 Dec 2024 22:52:55 +0800 Subject: [PATCH 402/659] feat(index): add `disable index` option for storages (#7730) --- internal/model/storage.go | 5 ++++- internal/op/driver.go | 6 ++++++ internal/search/build.go | 5 +++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/internal/model/storage.go b/internal/model/storage.go index 14bcf45f6e2..e3c7e1f9731 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -1,6 +1,8 @@ package model -import "time" +import ( + "time" +) type Storage struct { ID uint `json:"id" gorm:"primaryKey"` // unique key @@ -13,6 +15,7 @@ type Storage struct { Remark string `json:"remark"` Modified time.Time `json:"modified"` Disabled bool `json:"disabled"` // if disabled + DisableIndex bool `json:"disable_index"` EnableSign bool `json:"enable_sign"` Sort Proxy diff --git a/internal/op/driver.go b/internal/op/driver.go index 4f10e8e23c6..41b6f6d42c7 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -133,6 +133,12 @@ func getMainItems(config driver.Config) []driver.Item { Type: conf.TypeSelect, Options: "front,back", }) + items = append(items, driver.Item{ + Name: "disable_index", + Type: conf.TypeBool, + Default: "false", + Required: true, + }) items = append(items, driver.Item{ Name: "enable_sign", Type: conf.TypeBool, diff --git a/internal/search/build.go b/internal/search/build.go index 9865b2988a1..2888c1f45bb 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -157,6 +157,11 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth return filepath.SkipDir } } + if storage, _, err := op.GetStorageAndActualPath(indexPath); err == nil { + if storage.GetStorage().DisableIndex { + return filepath.SkipDir + } + } // ignore root if indexPath == "/" { return nil From aa1082a56c4d17377667daf9b888ae9270d2c1be Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Mon, 30 Dec 2024 22:54:37 +0800 Subject: [PATCH 403/659] feat(sftp-server): do not generate host key until first enabled (#7734) --- cmd/common.go | 1 - internal/conf/var.go | 3 -- server/sftp.go | 6 ++-- server/{ftp => sftp}/const.go | 2 +- .../ssh.go => server/sftp/hostkey.go | 12 ++++--- server/{ftp => sftp}/sftp.go | 33 ++++++++++--------- 6 files changed, 30 insertions(+), 27 deletions(-) rename server/{ftp => sftp}/const.go (94%) rename internal/bootstrap/ssh.go => server/sftp/hostkey.go (94%) rename server/{ftp => sftp}/sftp.go (64%) diff --git a/cmd/common.go b/cmd/common.go index beb558f510f..47a25f3f266 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -15,7 +15,6 @@ import ( func Init() { bootstrap.InitConfig() bootstrap.Log() - bootstrap.InitHostKey() bootstrap.InitDB() data.InitData() bootstrap.InitIndex() diff --git a/internal/conf/var.go b/internal/conf/var.go index b7277e4112c..0a8eb16fcd1 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -1,7 +1,6 @@ package conf import ( - "golang.org/x/crypto/ssh" "net/url" "regexp" ) @@ -33,5 +32,3 @@ var ( ManageHtml string IndexHtml string ) - -var SSHSigners []ssh.Signer diff --git a/server/sftp.go b/server/sftp.go index d44046a42d7..0455c96230f 100644 --- a/server/sftp.go +++ b/server/sftp.go @@ -9,6 +9,7 @@ import ( "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/ftp" + "github.com/alist-org/alist/v3/server/sftp" "github.com/pkg/errors" "golang.org/x/crypto/ssh" "net/http" @@ -21,6 +22,7 @@ type SftpDriver struct { } func NewSftpDriver() (*SftpDriver, error) { + sftp.InitHostKey() header := &http.Header{} header.Add("User-Agent", setting.GetStr(conf.FTPProxyUserAgent)) return &SftpDriver{ @@ -40,7 +42,7 @@ func (d *SftpDriver) GetConfig() *sftpd.Config { AuthLogCallback: d.AuthLogCallback, BannerCallback: d.GetBanner, } - for _, k := range conf.SSHSigners { + for _, k := range sftp.SSHSigners { serverConfig.AddHostKey(k) } d.config = &sftpd.Config{ @@ -62,7 +64,7 @@ func (d *SftpDriver) GetFileSystem(sc *ssh.ServerConn) (sftpd.FileSystem, error) ctx = context.WithValue(ctx, "meta_pass", "") ctx = context.WithValue(ctx, "client_ip", sc.RemoteAddr().String()) ctx = context.WithValue(ctx, "proxy_header", d.proxyHeader) - return &ftp.SftpDriverAdapter{FtpDriver: ftp.NewAferoAdapter(ctx)}, nil + return &sftp.DriverAdapter{FtpDriver: ftp.NewAferoAdapter(ctx)}, nil } func (d *SftpDriver) Close() { diff --git a/server/ftp/const.go b/server/sftp/const.go similarity index 94% rename from server/ftp/const.go rename to server/sftp/const.go index 1fd14e82d97..58bfe3824ca 100644 --- a/server/ftp/const.go +++ b/server/sftp/const.go @@ -1,4 +1,4 @@ -package ftp +package sftp // From leffss/sftpd const ( diff --git a/internal/bootstrap/ssh.go b/server/sftp/hostkey.go similarity index 94% rename from internal/bootstrap/ssh.go rename to server/sftp/hostkey.go index ec4a07ac6e3..0db103dd6cf 100644 --- a/internal/bootstrap/ssh.go +++ b/server/sftp/hostkey.go @@ -1,4 +1,4 @@ -package bootstrap +package sftp import ( "crypto/rand" @@ -7,14 +7,18 @@ import ( "encoding/pem" "fmt" "github.com/alist-org/alist/v3/cmd/flags" - "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/pkg/utils" "golang.org/x/crypto/ssh" "os" "path/filepath" ) +var SSHSigners []ssh.Signer + func InitHostKey() { + if SSHSigners != nil { + return + } sshPath := filepath.Join(flags.DataDir, "ssh") if !utils.Exists(sshPath) { err := utils.CreateNestedDirectory(sshPath) @@ -23,9 +27,9 @@ func InitHostKey() { return } } - conf.SSHSigners = make([]ssh.Signer, 0, 4) + SSHSigners = make([]ssh.Signer, 0, 4) if rsaKey, ok := LoadOrGenerateRSAHostKey(sshPath); ok { - conf.SSHSigners = append(conf.SSHSigners, rsaKey) + SSHSigners = append(SSHSigners, rsaKey) } // TODO Add keys for other encryption algorithms } diff --git a/server/ftp/sftp.go b/server/sftp/sftp.go similarity index 64% rename from server/ftp/sftp.go rename to server/sftp/sftp.go index 0a11ee1862d..1ceb3f59295 100644 --- a/server/ftp/sftp.go +++ b/server/sftp/sftp.go @@ -1,44 +1,45 @@ -package ftp +package sftp import ( "github.com/KirCute/sftpd-alist" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/ftp" "os" ) -type SftpDriverAdapter struct { - FtpDriver *AferoAdapter +type DriverAdapter struct { + FtpDriver *ftp.AferoAdapter } -func (s *SftpDriverAdapter) OpenFile(_ string, _ uint32, _ *sftpd.Attr) (sftpd.File, error) { +func (s *DriverAdapter) OpenFile(_ string, _ uint32, _ *sftpd.Attr) (sftpd.File, error) { // See also GetHandle return nil, errs.NotImplement } -func (s *SftpDriverAdapter) OpenDir(_ string) (sftpd.Dir, error) { +func (s *DriverAdapter) OpenDir(_ string) (sftpd.Dir, error) { // See also GetHandle return nil, errs.NotImplement } -func (s *SftpDriverAdapter) Remove(name string) error { +func (s *DriverAdapter) Remove(name string) error { return s.FtpDriver.Remove(name) } -func (s *SftpDriverAdapter) Rename(old, new string, _ uint32) error { +func (s *DriverAdapter) Rename(old, new string, _ uint32) error { return s.FtpDriver.Rename(old, new) } -func (s *SftpDriverAdapter) Mkdir(name string, attr *sftpd.Attr) error { +func (s *DriverAdapter) Mkdir(name string, attr *sftpd.Attr) error { return s.FtpDriver.Mkdir(name, attr.Mode) } -func (s *SftpDriverAdapter) Rmdir(name string) error { +func (s *DriverAdapter) Rmdir(name string) error { return s.Remove(name) } -func (s *SftpDriverAdapter) Stat(name string, _ bool) (*sftpd.Attr, error) { +func (s *DriverAdapter) Stat(name string, _ bool) (*sftpd.Attr, error) { stat, err := s.FtpDriver.Stat(name) if err != nil { return nil, err @@ -46,27 +47,27 @@ func (s *SftpDriverAdapter) Stat(name string, _ bool) (*sftpd.Attr, error) { return fileInfoToSftpAttr(stat), nil } -func (s *SftpDriverAdapter) SetStat(_ string, _ *sftpd.Attr) error { +func (s *DriverAdapter) SetStat(_ string, _ *sftpd.Attr) error { return errs.NotSupport } -func (s *SftpDriverAdapter) ReadLink(_ string) (string, error) { +func (s *DriverAdapter) ReadLink(_ string) (string, error) { return "", errs.NotSupport } -func (s *SftpDriverAdapter) CreateLink(_, _ string, _ uint32) error { +func (s *DriverAdapter) CreateLink(_, _ string, _ uint32) error { return errs.NotSupport } -func (s *SftpDriverAdapter) RealPath(path string) (string, error) { +func (s *DriverAdapter) RealPath(path string) (string, error) { return utils.FixAndCleanPath(path), nil } -func (s *SftpDriverAdapter) GetHandle(name string, flags uint32, _ *sftpd.Attr, offset uint64) (sftpd.FileTransfer, error) { +func (s *DriverAdapter) GetHandle(name string, flags uint32, _ *sftpd.Attr, offset uint64) (sftpd.FileTransfer, error) { return s.FtpDriver.GetHandle(name, sftpFlagToOpenMode(flags), int64(offset)) } -func (s *SftpDriverAdapter) ReadDir(name string) ([]sftpd.NamedAttr, error) { +func (s *DriverAdapter) ReadDir(name string) ([]sftpd.NamedAttr, error) { dir, err := s.FtpDriver.ReadDir(name) if err != nil { return nil, err From 6745dcc139dba93eb86fb50ad7849494ef7a1b6b Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Mon, 30 Dec 2024 22:55:09 +0800 Subject: [PATCH 404/659] feat(task): attach creator to `user` of the context (#7729) --- internal/task/base.go | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/internal/task/base.go b/internal/task/base.go index 93f413a7111..22b167417db 100644 --- a/internal/task/base.go +++ b/internal/task/base.go @@ -1,17 +1,21 @@ package task import ( + "context" "github.com/alist-org/alist/v3/internal/model" "github.com/xhofe/tache" + "sync" "time" ) type TaskExtension struct { tache.Base - Creator *model.User - startTime *time.Time - endTime *time.Time - totalBytes int64 + ctx context.Context + ctxInitMutex sync.Mutex + Creator *model.User + startTime *time.Time + endTime *time.Time + totalBytes int64 } func (t *TaskExtension) SetCreator(creator *model.User) { @@ -51,6 +55,17 @@ func (t *TaskExtension) GetTotalBytes() int64 { return t.totalBytes } +func (t *TaskExtension) Ctx() context.Context { + if t.ctx == nil { + t.ctxInitMutex.Lock() + if t.ctx == nil { + t.ctx = context.WithValue(t.Base.Ctx(), "user", t.Creator) + } + t.ctxInitMutex.Unlock() + } + return t.ctx +} + type TaskExtensionInfo interface { tache.TaskWithInfo GetCreator() *model.User From 7fd4ac78515387e5962868aef41acb36ba80064c Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 30 Dec 2024 22:55:47 +0800 Subject: [PATCH 405/659] fix(139): update familyGetFiles pagination logic (#7748 close #7711) --- drivers/139/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/139/util.go b/drivers/139/util.go index ccb6a912f32..d0b4d3b46a7 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -252,7 +252,7 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { } files = append(files, &f) } - if 100*pageNum > resp.Data.TotalCount { + if resp.Data.TotalCount == 0 { break } pageNum++ From e4439e66b9a03d139607584c2290ca2a99f7e184 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Wed, 1 Jan 2025 21:13:34 +0800 Subject: [PATCH 406/659] fix:(baidu_photo): upload erron -6 (#7760 close #7744) * fix:(baidu_photo): upload erron -6 * fix(baidu_photo):api add bdstoken --- drivers/baidu_photo/driver.go | 13 ++++++++++--- drivers/baidu_photo/utils.go | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index d0d69e82222..b584c9a3bb2 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -28,8 +28,9 @@ type BaiduPhoto struct { Addition // AccessToken string - Uk int64 - root model.Obj + Uk int64 + bdstoken string + root model.Obj uploadThread int } @@ -73,6 +74,10 @@ func (d *BaiduPhoto) Init(ctx context.Context) error { if err != nil { return err } + d.bdstoken, err = d.getBDStoken() + if err != nil { + return err + } d.Uk, err = strconv.ParseInt(info.YouaID, 10, 64) return err } @@ -296,6 +301,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil _, err = d.Post(FILE_API_URL_V1+"/precreate", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(params) + r.SetQueryParam("bdstoken", d.bdstoken) }, &precreateResp) if err != nil { return nil, err @@ -324,8 +330,8 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil "path": params["path"], "partseq": fmt.Sprint(partseq), "uploadid": precreateResp.UploadID, + "app_id": "16051585", } - _, err = d.Post("https://c3.pcs.baidu.com/rest/2.0/pcs/superfile2", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(uploadParams) @@ -352,6 +358,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil _, err = d.Post(FILE_API_URL_V1+"/create", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(params) + r.SetQueryParam("bdstoken", d.bdstoken) }, &precreateResp) if err != nil { return nil, err diff --git a/drivers/baidu_photo/utils.go b/drivers/baidu_photo/utils.go index 0b960593bce..6061600ea09 100644 --- a/drivers/baidu_photo/utils.go +++ b/drivers/baidu_photo/utils.go @@ -476,6 +476,21 @@ func (d *BaiduPhoto) uInfo() (*UInfo, error) { return &info, nil } +func (d *BaiduPhoto) getBDStoken() (string, error) { + var info struct { + Result struct { + Bdstoken string `json:"bdstoken"` + Token string `json:"token"` + Uk int64 `json:"uk"` + } `json:"result"` + } + _, err := d.Get("https://pan.baidu.com/api/gettemplatevariable?fields=[%22bdstoken%22,%22token%22,%22uk%22]", nil, &info) + if err != nil { + return "", err + } + return info.Result.Bdstoken, nil +} + func DecryptMd5(encryptMd5 string) string { if _, err := hex.DecodeString(encryptMd5); err == nil { return encryptMd5 From 687124c81d8256ea202689fbca019d6fcffb10af Mon Sep 17 00:00:00 2001 From: Mmx <36563672+Mmx233@users.noreply.github.com> Date: Wed, 1 Jan 2025 21:29:59 +0800 Subject: [PATCH 407/659] ci(build_docker): merge build_docker into release_docker workflow (#7755) * feat(ci): merge build_docker workflow into release_docker * fix(ci): logics of docker meta --- .github/workflows/build_docker.yml | 126 --------------------------- .github/workflows/release_docker.yml | 28 +++++- 2 files changed, 24 insertions(+), 130 deletions(-) delete mode 100644 .github/workflows/build_docker.yml diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml deleted file mode 100644 index 6384c374bf6..00000000000 --- a/.github/workflows/build_docker.yml +++ /dev/null @@ -1,126 +0,0 @@ -name: build_docker - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build_docker: - name: Build Docker - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: xhofe/alist - tags: | - type=schedule - type=ref,event=branch - type=ref,event=tag - type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} - - - name: Docker meta with ffmpeg - id: meta-ffmpeg - uses: docker/metadata-action@v5 - with: - images: xhofe/alist - flavor: | - suffix=-ffmpeg - tags: | - type=schedule - type=ref,event=branch - type=ref,event=tag - type=ref,event=pr - type=raw,value=beta,enable={{is_default_branch}} - - - uses: actions/setup-go@v5 - with: - go-version: 'stable' - - - name: Cache Musl - id: cache-musl - uses: actions/cache@v4 - with: - path: build/musl-libs - key: docker-musl-libs-v2 - - - name: Download Musl Library - if: steps.cache-musl.outputs.cache-hit != 'true' - run: bash build.sh prepare docker-multiplatform - - - name: Build go binary - run: bash build.sh dev docker-multiplatform - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to DockerHub - if: github.event_name == 'push' - uses: docker/login-action@v3 - with: - username: xhofe - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push - id: docker_build - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.ci - push: ${{ github.event_name == 'push' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64 - - - name: Build and push with ffmpeg - id: docker_build_ffmpeg - uses: docker/build-push-action@v6 - with: - context: . - file: Dockerfile.ci - push: ${{ github.event_name == 'push' }} - tags: ${{ steps.meta-ffmpeg.outputs.tags }} - labels: ${{ steps.meta-ffmpeg.outputs.labels }} - build-args: INSTALL_FFMPEG=true - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64 - - build_docker_with_aria2: - needs: build_docker - name: Build docker with aria2 - runs-on: ubuntu-latest - if: github.event_name == 'push' - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - repository: alist-org/with_aria2 - ref: main - persist-credentials: false - fetch-depth: 0 - - - name: Commit - run: | - git config --local user.email "bot@nn.ci" - git config --local user.name "IlaBot" - git commit --allow-empty -m "Trigger build for ${{ github.sha }}" - - - name: Push commit - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.MY_TOKEN }} - branch: main - repository: alist-org/with_aria2 diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 0f559a3f415..f4c79baf2df 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -4,13 +4,30 @@ on: push: tags: - 'v*' + branches: + - main + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true env: - IMAGE_REGISTRY: 'xhofe/alist' + REGISTRY: 'xhofe/alist' REGISTRY_USERNAME: 'xhofe' REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} ARTIFACT_NAME: 'binaries_docker_release' RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64' + IMAGE_PUSH: ${{ github.event_name == 'push' }} + IMAGE_IS_PROD: ${{ github.ref_type == 'tag' }} + IMAGE_TAGS_BETA: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=beta,enable={{is_default_branch}} jobs: build_binary: @@ -85,8 +102,10 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Login to DockerHub + if: env.IMAGE_PUSH == 'true' uses: docker/login-action@v3 with: + logout: true username: ${{ env.REGISTRY_USERNAME }} password: ${{ env.REGISTRY_PASSWORD }} @@ -94,9 +113,10 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.IMAGE_REGISTRY }} + images: ${{ env.REGISTRY }} + tags: ${{ env.IMAGE_IS_PROD == 'true' && '' || env.IMAGE_TAGS_BETA }} flavor: | - latest=true + ${{ env.IMAGE_IS_PROD == 'true' && 'latest=true' || '' }} ${{ matrix.tag_favor }} - name: Build and push @@ -105,7 +125,7 @@ jobs: with: context: . file: Dockerfile.ci - push: true + push: ${{ env.IMAGE_PUSH == 'true' }} build-args: ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From 31a74708653b2567c45b2db0b25db5f68d41c2ef Mon Sep 17 00:00:00 2001 From: Lin Tianchuan <47070449+1024th@users.noreply.github.com> Date: Fri, 10 Jan 2025 20:48:45 +0800 Subject: [PATCH 408/659] feat(local): support both time and percent for video thumbnail (#7802) * feat(local): support percent for video thumbnail The percentage determines the point in the video (as a percentage of the total duration) at which the thumbnail will be generated. * feat(local): support both time and percent for video thumbnail --- drivers/local/driver.go | 22 ++++++++++++++++ drivers/local/meta.go | 1 + drivers/local/util.go | 58 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 2519232e7d6..8a804ef32a6 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -79,6 +79,28 @@ func (d *Local) Init(ctx context.Context) error { } else { d.thumbTokenBucket = NewStaticTokenBucketWithMigration(d.thumbTokenBucket, d.thumbConcurrency) } + // Check the VideoThumbPos value + if d.VideoThumbPos == "" { + d.VideoThumbPos = "20%" + } + if strings.HasSuffix(d.VideoThumbPos, "%") { + percentage := strings.TrimSuffix(d.VideoThumbPos, "%") + val, err := strconv.ParseFloat(percentage, 64) + if err != nil { + return fmt.Errorf("invalid video_thumb_pos value: %s, err: %s", d.VideoThumbPos, err) + } + if val < 0 || val > 100 { + return fmt.Errorf("invalid video_thumb_pos value: %s, the precentage must be a number between 0 and 100", d.VideoThumbPos) + } + } else { + val, err := strconv.ParseFloat(d.VideoThumbPos, 64) + if err != nil { + return fmt.Errorf("invalid video_thumb_pos value: %s, err: %s", d.VideoThumbPos, err) + } + if val < 0 { + return fmt.Errorf("invalid video_thumb_pos value: %s, the time must be a positive number", d.VideoThumbPos) + } + } return nil } diff --git a/drivers/local/meta.go b/drivers/local/meta.go index 5ffac920234..14b0404f784 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -10,6 +10,7 @@ type Addition struct { Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` ThumbCacheFolder string `json:"thumb_cache_folder"` ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` + VideoThumbPos string `json:"video_thumb_pos" default:"20%" required:"false" help:"The position of the video thumbnail. If the value is a number (integer ot floating point), it represents the time in seconds. If the value ends with '%', it represents the percentage of the video duration."` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` MkdirPerm string `json:"mkdir_perm" default:"777"` RecycleBinPath string `json:"recycle_bin_path" default:"delete permanently" help:"path to recycle bin, delete permanently if empty or keep 'delete permanently'"` diff --git a/drivers/local/util.go b/drivers/local/util.go index b994c2056b7..d2fbd097b5a 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -2,11 +2,13 @@ package local import ( "bytes" + "encoding/json" "fmt" "io/fs" "os" "path/filepath" "sort" + "strconv" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -34,10 +36,58 @@ func isSymlinkDir(f fs.FileInfo, path string) bool { return false } -func GetSnapshot(videoPath string, frameNum int) (imgData *bytes.Buffer, err error) { +// Get the snapshot of the video +func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) { + // Run ffprobe to get the video duration + jsonOutput, err := ffmpeg.Probe(videoPath) + if err != nil { + return nil, err + } + // get format.duration from the json string + type probeFormat struct { + Duration string `json:"duration"` + } + type probeData struct { + Format probeFormat `json:"format"` + } + var probe probeData + err = json.Unmarshal([]byte(jsonOutput), &probe) + if err != nil { + return nil, err + } + totalDuration, err := strconv.ParseFloat(probe.Format.Duration, 64) + if err != nil { + return nil, err + } + + var ss string + if strings.HasSuffix(d.VideoThumbPos, "%") { + percentage, err := strconv.ParseFloat(strings.TrimSuffix(d.VideoThumbPos, "%"), 64) + if err != nil { + return nil, err + } + ss = fmt.Sprintf("%f", totalDuration*percentage/100) + } else { + val, err := strconv.ParseFloat(d.VideoThumbPos, 64) + if err != nil { + return nil, err + } + // If the value is greater than the total duration, use the total duration + if val > totalDuration { + ss = fmt.Sprintf("%f", totalDuration) + } else { + ss = d.VideoThumbPos + } + } + + // Run ffmpeg to get the snapshot srcBuf := bytes.NewBuffer(nil) - stream := ffmpeg.Input(videoPath). - Filter("select", ffmpeg.Args{fmt.Sprintf("gte(n,%d)", frameNum)}). + // If the remaining time from the seek point to the end of the video is less + // than the duration of a single frame, ffmpeg cannot extract any frames + // within the specified range and will exit with an error. + // The "noaccurate_seek" option prevents this error and would also speed up + // the seek process. + stream := ffmpeg.Input(videoPath, ffmpeg.KwArgs{"ss": ss, "noaccurate_seek": ""}). Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg"}). GlobalArgs("-loglevel", "error").Silent(true). WithOutput(srcBuf, os.Stdout) @@ -77,7 +127,7 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { } var srcBuf *bytes.Buffer if utils.GetFileType(file.GetName()) == conf.VIDEO { - videoBuf, err := GetSnapshot(fullPath, 10) + videoBuf, err := d.GetSnapshot(fullPath) if err != nil { return nil, nil, err } From 6812ec9a6d7c4a1684f23d23d7d2a06a48bc635d Mon Sep 17 00:00:00 2001 From: Jiang Xiang <869914918@qq.com> Date: Fri, 10 Jan 2025 20:49:50 +0800 Subject: [PATCH 409/659] fix(ilanzou): add accept-encoding request header (#7796 close #7759) --- drivers/ilanzou/util.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index a57e2a4a6be..b8fd5280c77 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -69,9 +69,10 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr req := base.RestyClient.R() req.SetHeaders(map[string]string{ - "Origin": d.conf.site, - "Referer": d.conf.site + "/", - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", + "Origin": d.conf.site, + "Referer": d.conf.site + "/", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", + "Accept-Encoding": "gzip, deflate, br, zstd", }) if callback != nil { From 25b4b55ee108576217259bd850781f5a5f4fc3ce Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Fri, 10 Jan 2025 20:50:20 +0800 Subject: [PATCH 410/659] feat(ftp-server): support resumable downloading (#7792) --- go.mod | 4 ++-- go.sum | 8 ++++---- server/ftp/afero.go | 8 ++++---- server/ftp/fsread.go | 47 ++++++++++++++++++++++++++++++++++++++------ server/ftp/fsup.go | 2 +- 5 files changed, 52 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index 1deaa1d5565..7ca66e155ff 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.4 require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 - github.com/KirCute/sftpd-alist v0.0.11 + github.com/KirCute/sftpd-alist v0.0.12 github.com/SheltonZhu/115driver v1.0.32 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 @@ -62,7 +62,7 @@ require ( github.com/xhofe/tache v0.1.3 github.com/xhofe/wopan-sdk-go v0.1.3 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.30.0 + golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 golang.org/x/net v0.28.0 diff --git a/go.sum b/go.sum index a4e8e12d7b4..101a0bea063 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 h1:ikwCzeqoqN6wvBHOB9OI6dde/jbV7EoTMpUcxtYl5Po= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= -github.com/KirCute/sftpd-alist v0.0.11 h1:BGInXmmLBI+v6S9WZCwvY0DRK1vDprGNcTv/57p2GSo= -github.com/KirCute/sftpd-alist v0.0.11/go.mod h1:pPFzr6GrKqXvFXLr46ZpoqmtSpwH8DKTYloSp/ybzKQ= +github.com/KirCute/sftpd-alist v0.0.12 h1:GNVM5QLbQLAfXP4wGUlXFA2IO6fVek0n0IsGnOuISdg= +github.com/KirCute/sftpd-alist v0.0.12/go.mod h1:2wNK7yyW2XfjyJq10OY6xB4COLac64hOwfV6clDJn6s= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= @@ -574,8 +574,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/server/ftp/afero.go b/server/ftp/afero.go index 448744b1c01..75ae2e433e4 100644 --- a/server/ftp/afero.go +++ b/server/ftp/afero.go @@ -83,9 +83,6 @@ func (a *AferoAdapter) ReadDir(name string) ([]os.FileInfo, error) { func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) { fileSize := a.nextFileSize a.nextFileSize = 0 - if offset != 0 { - return nil, errs.NotSupport - } if (flags & os.O_SYNC) != 0 { return nil, errs.NotSupport } @@ -106,6 +103,9 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve return nil, errors.New("file already exists") } if (flags & os.O_WRONLY) != 0 { + if offset != 0 { + return nil, errs.NotSupport + } trunc := (flags & os.O_TRUNC) != 0 if fileSize > 0 { return OpenUploadWithLength(a.ctx, path, trunc, fileSize) @@ -113,7 +113,7 @@ func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserve return OpenUpload(a.ctx, path, trunc) } } - return OpenDownload(a.ctx, path) + return OpenDownload(a.ctx, path, offset) } func (a *AferoAdapter) SetNextFileSize(size int64) { diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index 74d184b6535..257d2ec838a 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -8,6 +8,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/server/common" "github.com/pkg/errors" "io" @@ -19,10 +20,12 @@ import ( type FileDownloadProxy struct { ftpserver.FileTransfer - reader io.ReadCloser + ss *stream.SeekableStream + reader io.Reader + cur int64 } -func OpenDownload(ctx context.Context, reqPath string) (*FileDownloadProxy, error) { +func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownloadProxy, error) { user := ctx.Value("user").(*model.User) meta, err := op.GetNearestMeta(reqPath) if err != nil { @@ -52,11 +55,22 @@ func OpenDownload(ctx context.Context, reqPath string) (*FileDownloadProxy, erro if err != nil { return nil, err } - return &FileDownloadProxy{reader: ss}, nil + var reader io.Reader + if offset != 0 { + reader, err = ss.RangeRead(http_range.Range{Start: offset, Length: -1}) + if err != nil { + return nil, err + } + } else { + reader = ss + } + return &FileDownloadProxy{ss: ss, reader: reader}, nil } func (f *FileDownloadProxy) Read(p []byte) (n int, err error) { - return f.reader.Read(p) + n, err = f.reader.Read(p) + f.cur += int64(n) + return n, err } func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { @@ -64,11 +78,32 @@ func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { } func (f *FileDownloadProxy) Seek(offset int64, whence int) (int64, error) { - return 0, errs.NotSupport + switch whence { + case io.SeekStart: + break + case io.SeekCurrent: + offset += f.cur + break + case io.SeekEnd: + offset += f.ss.GetSize() + break + default: + return 0, errs.NotSupport + } + if offset < 0 { + return 0, errors.New("Seek: negative position") + } + reader, err := f.ss.RangeRead(http_range.Range{Start: offset, Length: -1}) + if err != nil { + return f.cur, err + } + f.cur = offset + f.reader = reader + return offset, nil } func (f *FileDownloadProxy) Close() error { - return f.reader.Close() + return f.ss.Close() } type OsFileInfoAdapter struct { diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index 96c8468143c..4d626d0efcb 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -63,7 +63,7 @@ func (f *FileUploadProxy) Write(p []byte) (n int, err error) { } func (f *FileUploadProxy) Seek(offset int64, whence int) (int64, error) { - return 0, errs.NotSupport + return f.buffer.Seek(offset, whence) } func (f *FileUploadProxy) Close() error { From 51bcf83511be2bc05ff73d2450784d1a14d9d973 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Fri, 10 Jan 2025 20:50:56 +0800 Subject: [PATCH 411/659] feat(url-tree): support url tree driver writing (#7779 close #5166) * feat: support url tree writing * fix: meta writable * feat: disable writable via addition --- drivers/url_tree/driver.go | 181 +++++++++++++++++++++++++- drivers/url_tree/meta.go | 3 +- drivers/url_tree/types.go | 18 +++ drivers/url_tree/util.go | 46 +++++++ internal/driver/driver.go | 14 ++ internal/offline_download/tool/add.go | 47 +++++-- internal/op/fs.go | 40 ++++++ server/handles/offline_download.go | 4 +- 8 files changed, 338 insertions(+), 15 deletions(-) diff --git a/drivers/url_tree/driver.go b/drivers/url_tree/driver.go index 6a45bb7d4e1..569b3fba5c7 100644 --- a/drivers/url_tree/driver.go +++ b/drivers/url_tree/driver.go @@ -2,7 +2,11 @@ package url_tree import ( "context" + "errors" + "github.com/alist-org/alist/v3/internal/op" stdpath "path" + "strings" + "sync" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -14,7 +18,8 @@ import ( type Urls struct { model.Storage Addition - root *Node + root *Node + mutex sync.RWMutex } func (d *Urls) Config() driver.Config { @@ -40,11 +45,15 @@ func (d *Urls) Drop(ctx context.Context) error { } func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, path) return nodeToObj(node, path) } func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, dir.GetPath()) log.Debugf("path: %s, node: %+v", dir.GetPath(), node) if node == nil { @@ -59,6 +68,8 @@ func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] } func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, file.GetPath()) log.Debugf("path: %s, node: %+v", file.GetPath(), node) if node == nil { @@ -72,6 +83,174 @@ func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* return nil, errs.NotFile } +func (d *Urls) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + node := GetNodeFromRootByPath(d.root, parentDir.GetPath()) + if node == nil { + return nil, errs.ObjectNotFound + } + if node.isFile() { + return nil, errs.NotFolder + } + dir := &Node{ + Name: dirName, + Level: node.Level + 1, + } + node.Children = append(node.Children, dir) + d.updateStorage() + return nodeToObj(dir, stdpath.Join(parentDir.GetPath(), dirName)) +} + +func (d *Urls) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return nil, errors.New("cannot move parent dir to child") + } + d.mutex.Lock() + defer d.mutex.Unlock() + dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dstNode == nil || dstNode.isFile() { + return nil, errs.NotFolder + } + srcDir, srcName := stdpath.Split(srcObj.GetPath()) + srcParentNode := GetNodeFromRootByPath(d.root, srcDir) + if srcParentNode == nil { + return nil, errs.ObjectNotFound + } + newChildren := make([]*Node, 0, len(srcParentNode.Children)) + var srcNode *Node + for _, child := range srcParentNode.Children { + if child.Name == srcName { + srcNode = child + } else { + newChildren = append(newChildren, child) + } + } + if srcNode == nil { + return nil, errs.ObjectNotFound + } + srcParentNode.Children = newChildren + srcNode.setLevel(dstNode.Level + 1) + dstNode.Children = append(dstNode.Children, srcNode) + d.root.calSize() + d.updateStorage() + return nodeToObj(srcNode, stdpath.Join(dstDir.GetPath(), srcName)) +} + +func (d *Urls) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath()) + if srcNode == nil { + return nil, errs.ObjectNotFound + } + srcNode.Name = newName + d.updateStorage() + return nodeToObj(srcNode, stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName)) +} + +func (d *Urls) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return nil, errors.New("cannot copy parent dir to child") + } + d.mutex.Lock() + defer d.mutex.Unlock() + dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dstNode == nil || dstNode.isFile() { + return nil, errs.NotFolder + } + srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath()) + if srcNode == nil { + return nil, errs.ObjectNotFound + } + newNode := srcNode.deepCopy(dstNode.Level + 1) + dstNode.Children = append(dstNode.Children, newNode) + d.root.calSize() + d.updateStorage() + return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), stdpath.Base(srcObj.GetPath()))) +} + +func (d *Urls) Remove(ctx context.Context, obj model.Obj) error { + if !d.Writable { + return errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + objDir, objName := stdpath.Split(obj.GetPath()) + nodeParent := GetNodeFromRootByPath(d.root, objDir) + if nodeParent == nil { + return errs.ObjectNotFound + } + newChildren := make([]*Node, 0, len(nodeParent.Children)) + var deletedObj *Node + for _, child := range nodeParent.Children { + if child.Name != objName { + newChildren = append(newChildren, child) + } else { + deletedObj = child + } + } + if deletedObj == nil { + return errs.ObjectNotFound + } + nodeParent.Children = newChildren + if deletedObj.Size > 0 { + d.root.calSize() + } + d.updateStorage() + return nil +} + +func (d *Urls) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + dirNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dirNode == nil || dirNode.isFile() { + return nil, errs.NotFolder + } + newNode := &Node{ + Name: name, + Level: dirNode.Level + 1, + Url: url, + } + dirNode.Children = append(dirNode.Children, newNode) + if d.HeadSize { + size, err := getSizeFromUrl(url) + if err != nil { + log.Errorf("get size from url error: %s", err) + } else { + newNode.Size = size + d.root.calSize() + } + } + d.updateStorage() + return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), name)) +} + +func (d *Urls) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + return errs.UploadNotSupported +} + +func (d *Urls) updateStorage() { + d.UrlStructure = StringifyTree(d.root) + op.MustSaveDriverStorage(d) +} + //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/url_tree/meta.go b/drivers/url_tree/meta.go index b3ae33dc059..c40414f58c3 100644 --- a/drivers/url_tree/meta.go +++ b/drivers/url_tree/meta.go @@ -12,6 +12,7 @@ type Addition struct { // define other UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://jsd.nn.ci/gh/alist-org/alist/README.md\nhttps://jsd.nn.ci/gh/alist-org/alist/README_cn.md\nfolder:\n CONTRIBUTING.md:1635:https://jsd.nn.ci/gh/alist-org/alist/CONTRIBUTING.md\n CODE_OF_CONDUCT.md:2093:https://jsd.nn.ci/gh/alist-org/alist/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n [FileName:][FileSize:][Modified:]Url"` HeadSize bool `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."` + Writable bool `json:"writable" type:"bool" default:"false"` } var config = driver.Config{ @@ -20,7 +21,7 @@ var config = driver.Config{ OnlyLocal: false, OnlyProxy: false, NoCache: true, - NoUpload: true, + NoUpload: false, NeedMs: false, DefaultRoot: "", CheckStatus: true, diff --git a/drivers/url_tree/types.go b/drivers/url_tree/types.go index 7e8ca3d93ae..cf62d29d65a 100644 --- a/drivers/url_tree/types.go +++ b/drivers/url_tree/types.go @@ -1,5 +1,7 @@ package url_tree +import "github.com/alist-org/alist/v3/pkg/utils" + // Node is a node in the folder tree type Node struct { Url string @@ -44,3 +46,19 @@ func (node *Node) calSize() int64 { node.Size = size return size } + +func (node *Node) setLevel(level int) { + node.Level = level + for _, child := range node.Children { + child.setLevel(level + 1) + } +} + +func (node *Node) deepCopy(level int) *Node { + ret := *node + ret.Level = level + ret.Children, _ = utils.SliceConvert(ret.Children, func(child *Node) (*Node, error) { + return child.deepCopy(level + 1), nil + }) + return &ret +} diff --git a/drivers/url_tree/util.go b/drivers/url_tree/util.go index 4065218fcc1..61a3fde2c1f 100644 --- a/drivers/url_tree/util.go +++ b/drivers/url_tree/util.go @@ -153,6 +153,9 @@ func splitPath(path string) []string { if path == "/" { return []string{"root"} } + if strings.HasSuffix(path, "/") { + path = path[:len(path)-1] + } parts := strings.Split(path, "/") parts[0] = "root" return parts @@ -190,3 +193,46 @@ func getSizeFromUrl(url string) (int64, error) { } return size, nil } + +func StringifyTree(node *Node) string { + sb := strings.Builder{} + if node.Level == -1 { + for i, child := range node.Children { + sb.WriteString(StringifyTree(child)) + if i < len(node.Children)-1 { + sb.WriteString("\n") + } + } + return sb.String() + } + for i := 0; i < node.Level; i++ { + sb.WriteString(" ") + } + if node.Url == "" { + sb.WriteString(node.Name) + sb.WriteString(":") + for _, child := range node.Children { + sb.WriteString("\n") + sb.WriteString(StringifyTree(child)) + } + } else if node.Size == 0 && node.Modified == 0 { + if stdpath.Base(node.Url) == node.Name { + sb.WriteString(node.Url) + } else { + sb.WriteString(fmt.Sprintf("%s:%s", node.Name, node.Url)) + } + } else { + sb.WriteString(node.Name) + sb.WriteString(":") + if node.Size != 0 || node.Modified != 0 { + sb.WriteString(strconv.FormatInt(node.Size, 10)) + sb.WriteString(":") + } + if node.Modified != 0 { + sb.WriteString(strconv.FormatInt(node.Modified, 10)) + sb.WriteString(":") + } + sb.WriteString(node.Url) + } + return sb.String() +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 781e85325ee..6fd5e8d6f90 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -80,6 +80,13 @@ type Put interface { Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error } +type PutURL interface { + // PutURL directly put a URL into the storage + // Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs + // Called when using SimpleHttp for offline downloading, skipping creating a download task + PutURL(ctx context.Context, dstDir model.Obj, name, url string) error +} + //type WriteResult interface { // MkdirResult // MoveResult @@ -109,6 +116,13 @@ type PutResult interface { Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error) } +type PutURLResult interface { + // PutURL directly put a URL into the storage + // Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs + // Called when using SimpleHttp for offline downloading, skipping creating a download task + PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) +} + type UpdateProgress func(percentage float64) type Progress struct { diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 4158051a8f0..405f96cbe2d 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,8 +2,11 @@ package tool import ( "context" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/task" + "net/url" + "path" "path/filepath" "github.com/alist-org/alist/v3/internal/conf" @@ -30,18 +33,6 @@ type AddURLArgs struct { } func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, error) { - // get tool - tool, err := Tools.Get(args.Tool) - if err != nil { - return nil, errors.Wrapf(err, "failed get tool") - } - // check tool is ready - if !tool.IsReady() { - // try to init tool - if _, err := tool.Init(); err != nil { - return nil, errors.Wrapf(err, "failed init tool %s", args.Tool) - } - } // check storage storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) if err != nil { @@ -63,6 +54,23 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro return nil, errors.WithStack(errs.NotFolder) } } + // try putting url + if args.Tool == "SimpleHttp" && tryPutUrl(ctx, storage, dstDirActualPath, args.URL) { + return nil, nil + } + + // get tool + tool, err := Tools.Get(args.Tool) + if err != nil { + return nil, errors.Wrapf(err, "failed get tool") + } + // check tool is ready + if !tool.IsReady() { + // try to init tool + if _, err := tool.Init(); err != nil { + return nil, errors.Wrapf(err, "failed init tool %s", args.Tool) + } + } uid := uuid.NewString() tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) @@ -98,3 +106,18 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro DownloadTaskManager.Add(t) return t, nil } + +func tryPutUrl(ctx context.Context, storage driver.Driver, dstDirActualPath, urlStr string) bool { + _, ok := storage.(driver.PutURL) + _, okResult := storage.(driver.PutURLResult) + if !ok && !okResult { + return false + } + u, err := url.Parse(urlStr) + if err != nil { + return false + } + dstName := path.Base(u.Path) + err = op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) + return err == nil +} diff --git a/internal/op/fs.go b/internal/op/fs.go index e49c941a62f..01727e7598c 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -586,3 +586,43 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } return errors.WithStack(err) } + +func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + dstDirPath = utils.FixAndCleanPath(dstDirPath) + _, err := GetUnwrap(ctx, storage, stdpath.Join(dstDirPath, dstName)) + if err == nil { + return errors.New("obj already exists") + } + err = MakeDir(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessagef(err, "failed to put url") + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessagef(err, "failed to put url") + } + switch s := storage.(type) { + case driver.PutURLResult: + var newObj model.Obj + newObj, err = s.PutURL(ctx, dstDir, dstName, url) + if err == nil { + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.PutURL: + err = s.PutURL(ctx, dstDir, dstName, url) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errs.NotImplement + } + log.Debugf("put url [%s](%s) done", dstName, url) + return errors.WithStack(err) +} diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 9e26030a04d..c7b7af76c10 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -145,7 +145,9 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 500) return } - tasks = append(tasks, t) + if t != nil { + tasks = append(tasks, t) + } } common.SuccessResp(c, gin.H{ "tasks": getTaskInfos(tasks), From e04114d10246734c3ce0a5aa1719e61cee75dc4c Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Fri, 10 Jan 2025 20:59:58 +0800 Subject: [PATCH 412/659] feat(github): add github api driver (#7717) * feat(github): add github api driver * fix: filter submodule operation * feat: rename, copy and move, but with bugs * fix: move and copy returns 422 * fix: change TargetPath in rename msg from parent path to new self path * fix: add non-commit mutex * pref(github): use net/http to put blob * chore: add a help message to `ref` addition --- drivers/all.go | 1 + drivers/github/driver.go | 928 +++++++++++++++++++++++++++++++++++++++ drivers/github/meta.go | 36 ++ drivers/github/types.go | 102 +++++ drivers/github/util.go | 115 +++++ 5 files changed, 1182 insertions(+) create mode 100644 drivers/github/driver.go create mode 100644 drivers/github/meta.go create mode 100644 drivers/github/types.go create mode 100644 drivers/github/util.go diff --git a/drivers/all.go b/drivers/all.go index 4c4ef5c147b..8b253a08558 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -24,6 +24,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" + _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" _ "github.com/alist-org/alist/v3/drivers/halalcloud" diff --git a/drivers/github/driver.go b/drivers/github/driver.go new file mode 100644 index 00000000000..ea8f62762ed --- /dev/null +++ b/drivers/github/driver.go @@ -0,0 +1,928 @@ +package github + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "io" + "net/http" + stdpath "path" + "strings" + "sync" + "text/template" +) + +type Github struct { + model.Storage + Addition + client *resty.Client + mkdirMsgTmpl *template.Template + deleteMsgTmpl *template.Template + putMsgTmpl *template.Template + renameMsgTmpl *template.Template + copyMsgTmpl *template.Template + moveMsgTmpl *template.Template + isOnBranch bool + commitMutex sync.Mutex +} + +func (d *Github) Config() driver.Config { + return config +} + +func (d *Github) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Github) Init(ctx context.Context) error { + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + if d.CommitterName != "" && d.CommitterEmail == "" { + return errors.New("committer email is required") + } + if d.CommitterName == "" && d.CommitterEmail != "" { + return errors.New("committer name is required") + } + if d.AuthorName != "" && d.AuthorEmail == "" { + return errors.New("author email is required") + } + if d.AuthorName == "" && d.AuthorEmail != "" { + return errors.New("author name is required") + } + var err error + d.mkdirMsgTmpl, err = template.New("mkdirCommitMsgTemplate").Parse(d.MkdirCommitMsg) + if err != nil { + return err + } + d.deleteMsgTmpl, err = template.New("deleteCommitMsgTemplate").Parse(d.DeleteCommitMsg) + if err != nil { + return err + } + d.putMsgTmpl, err = template.New("putCommitMsgTemplate").Parse(d.PutCommitMsg) + if err != nil { + return err + } + d.renameMsgTmpl, err = template.New("renameCommitMsgTemplate").Parse(d.RenameCommitMsg) + if err != nil { + return err + } + d.copyMsgTmpl, err = template.New("copyCommitMsgTemplate").Parse(d.CopyCommitMsg) + if err != nil { + return err + } + d.moveMsgTmpl, err = template.New("moveCommitMsgTemplate").Parse(d.MoveCommitMsg) + if err != nil { + return err + } + d.client = base.NewRestyClient(). + SetHeader("Accept", "application/vnd.github.object+json"). + SetHeader("Authorization", "Bearer "+d.Token). + SetHeader("X-GitHub-Api-Version", "2022-11-28"). + SetLogger(log.StandardLogger()). + SetDebug(false) + if d.Ref == "" { + repo, err := d.getRepo() + if err != nil { + return err + } + d.Ref = repo.DefaultBranch + d.isOnBranch = true + } else { + _, err = d.getBranchHead() + d.isOnBranch = err == nil + } + return nil +} + +func (d *Github) Drop(ctx context.Context) error { + return nil +} + +func (d *Github) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + obj, err := d.get(dir.GetPath()) + if err != nil { + return nil, err + } + if obj.Entries == nil { + return nil, errs.NotFolder + } + if len(obj.Entries) >= 1000 { + tree, err := d.getTree(obj.Sha) + if err != nil { + return nil, err + } + if tree.Truncated { + return nil, fmt.Errorf("tree %s is truncated", dir.GetPath()) + } + ret := make([]model.Obj, 0, len(tree.Trees)) + for _, t := range tree.Trees { + if t.Path != ".gitkeep" { + ret = append(ret, t.toModelObj()) + } + } + return ret, nil + } else { + ret := make([]model.Obj, 0, len(obj.Entries)) + for _, entry := range obj.Entries { + if entry.Name != ".gitkeep" { + ret = append(ret, entry.toModelObj()) + } + } + return ret, nil + } +} + +func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, err := d.get(file.GetPath()) + if err != nil { + return nil, err + } + if obj.Type == "submodule" { + return nil, errors.New("cannot download a submodule") + } + return &model.Link{ + URL: obj.DownloadURL, + }, nil +} + +func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + parent, err := d.get(parentDir.GetPath()) + if err != nil { + return err + } + if parent.Entries == nil { + return errs.NotFolder + } + // if parent folder contains .gitkeep only, mark it and delete .gitkeep later + gitKeepSha := "" + if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { + gitKeepSha = parent.Entries[0].Sha + } + + commitMessage, err := getMessage(d.mkdirMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: dirName, + ObjPath: stdpath.Join(parentDir.GetPath(), dirName), + ParentName: parentDir.GetName(), + ParentPath: parentDir.GetPath(), + }, "mkdir") + if err != nil { + return err + } + if err = d.createGitKeep(stdpath.Join(parentDir.GetPath(), dirName), commitMessage); err != nil { + return err + } + if gitKeepSha != "" { + err = d.delete(stdpath.Join(parentDir.GetPath(), ".gitkeep"), gitKeepSha, commitMessage) + } + return err +} + +func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return errors.New("cannot move parent dir to child") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + + var rootSha string + if strings.HasPrefix(dstDir.GetPath(), stdpath.Dir(srcObj.GetPath())) { // /aa/1 -> /aa/bb/ + dstOldSha, dstNewSha, ancestorOldSha, srcParentTree, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + + srcParentPath := stdpath.Dir(srcObj.GetPath()) + dstRest := dstDir.GetPath()[len(srcParentPath):] + if dstRest[0] == '/' { + dstRest = dstRest[1:] + } + dstNextName, _, _ := strings.Cut(dstRest, "/") + dstNextPath := stdpath.Join(srcParentPath, dstNextName) + dstNextTreeSha, err := d.renewParentTrees(dstDir.GetPath(), dstOldSha, dstNewSha, dstNextPath) + if err != nil { + return err + } + var delSrc, dstNextTree *TreeObjReq = nil, nil + for _, t := range srcParentTree.Trees { + if t.Path == dstNextName { + dstNextTree = &t.TreeObjReq + dstNextTree.Sha = dstNextTreeSha + } + if t.Path == srcObj.GetName() { + delSrc = &t.TreeObjReq + delSrc.Sha = nil + } + if delSrc != nil && dstNextTree != nil { + break + } + } + if delSrc == nil || dstNextTree == nil { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, []interface{}{*delSrc, *dstNextTree}) + if err != nil { + return err + } + rootSha, err = d.renewParentTrees(srcParentPath, ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } else if strings.HasPrefix(srcObj.GetPath(), dstDir.GetPath()) { // /aa/bb/1 -> /aa/ + srcParentPath := stdpath.Dir(srcObj.GetPath()) + srcParentTree, srcParentOldSha, err := d.getTreeDirectly(srcParentPath) + if err != nil { + return err + } + var src *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return errors.New("cannot move a submodule") + } + src = &t.TreeObjReq + break + } + } + if src == nil { + return errs.ObjectNotFound + } + + delSrc := *src + delSrc.Sha = nil + delSrcTree := make([]interface{}, 0, 2) + delSrcTree = append(delSrcTree, delSrc) + if len(srcParentTree.Trees) == 1 { + delSrcTree = append(delSrcTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + srcParentNewSha, err := d.newTree(srcParentOldSha, delSrcTree) + if err != nil { + return err + } + srcRest := srcObj.GetPath()[len(dstDir.GetPath()):] + if srcRest[0] == '/' { + srcRest = srcRest[1:] + } + srcNextName, _, ok := strings.Cut(srcRest, "/") + if !ok { // /aa/1 -> /aa/ + return errors.New("cannot move in place") + } + srcNextPath := stdpath.Join(dstDir.GetPath(), srcNextName) + srcNextTreeSha, err := d.renewParentTrees(srcParentPath, srcParentOldSha, srcParentNewSha, srcNextPath) + if err != nil { + return err + } + + ancestorTree, ancestorOldSha, err := d.getTreeDirectly(dstDir.GetPath()) + if err != nil { + return err + } + var srcNextTree *TreeObjReq = nil + for _, t := range ancestorTree.Trees { + if t.Path == srcNextName { + srcNextTree = &t.TreeObjReq + srcNextTree.Sha = srcNextTreeSha + break + } + } + if srcNextTree == nil { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, []interface{}{*srcNextTree, *src}) + if err != nil { + return err + } + rootSha, err = d.renewParentTrees(dstDir.GetPath(), ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } else { // /aa/1 -> /bb/ + // do copy + dstOldSha, dstNewSha, srcParentOldSha, srcParentTree, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + + // delete src object and create new tree + var srcNewTree *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + srcNewTree = &t.TreeObjReq + srcNewTree.Sha = nil + break + } + } + if srcNewTree == nil { + return errs.ObjectNotFound + } + delSrcTree := make([]interface{}, 0, 2) + delSrcTree = append(delSrcTree, *srcNewTree) + if len(srcParentTree.Trees) == 1 { + delSrcTree = append(delSrcTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + srcParentNewSha, err := d.newTree(srcParentOldSha, delSrcTree) + if err != nil { + return err + } + + // renew but the common ancestor of srcPath and dstPath + ancestor, srcChildName, dstChildName, _, _ := getPathCommonAncestor(srcObj.GetPath(), dstDir.GetPath()) + dstNextTreeSha, err := d.renewParentTrees(dstDir.GetPath(), dstOldSha, dstNewSha, stdpath.Join(ancestor, dstChildName)) + if err != nil { + return err + } + srcNextTreeSha, err := d.renewParentTrees(stdpath.Dir(srcObj.GetPath()), srcParentOldSha, srcParentNewSha, stdpath.Join(ancestor, srcChildName)) + if err != nil { + return err + } + + // renew the tree of the last common ancestor + ancestorTree, ancestorOldSha, err := d.getTreeDirectly(ancestor) + if err != nil { + return err + } + newTree := make([]interface{}, 2) + srcBind := false + dstBind := false + for _, t := range ancestorTree.Trees { + if t.Path == srcChildName { + t.Sha = srcNextTreeSha + newTree[0] = t.TreeObjReq + srcBind = true + } + if t.Path == dstChildName { + t.Sha = dstNextTreeSha + newTree[1] = t.TreeObjReq + dstBind = true + } + if srcBind && dstBind { + break + } + } + if !srcBind || !dstBind { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, newTree) + if err != nil { + return err + } + // renew until root + rootSha, err = d.renewParentTrees(ancestor, ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } + + // commit + message, err := getMessage(d.moveMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), + ParentPath: stdpath.Dir(srcObj.GetPath()), + TargetName: stdpath.Base(dstDir.GetPath()), + TargetPath: dstDir.GetPath(), + }, "move") + if err != nil { + return err + } + return d.commit(message, rootSha) +} + +func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + parentDir := stdpath.Dir(srcObj.GetPath()) + tree, _, err := d.getTreeDirectly(parentDir) + if err != nil { + return err + } + newTree := make([]interface{}, 2) + operated := false + for _, t := range tree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return errors.New("cannot rename a submodule") + } + delCopy := t.TreeObjReq + delCopy.Sha = nil + newTree[0] = delCopy + t.Path = newName + newTree[1] = t.TreeObjReq + operated = true + break + } + } + if !operated { + return errs.ObjectNotFound + } + newSha, err := d.newTree(tree.Sha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(parentDir, tree.Sha, newSha, "/") + if err != nil { + return err + } + message, err := getMessage(d.renameMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(parentDir), + ParentPath: parentDir, + TargetName: newName, + TargetPath: stdpath.Join(parentDir, newName), + }, "rename") + if err != nil { + return err + } + return d.commit(message, rootSha) +} + +func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return errors.New("cannot copy parent dir to child") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + + dstSha, newSha, _, _, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(dstDir.GetPath(), dstSha, newSha, "/") + if err != nil { + return err + } + message, err := getMessage(d.copyMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), + ParentPath: stdpath.Dir(srcObj.GetPath()), + TargetName: stdpath.Base(dstDir.GetPath()), + TargetPath: dstDir.GetPath(), + }, "copy") + if err != nil { + return err + } + return d.commit(message, rootSha) +} + +func (d *Github) Remove(ctx context.Context, obj model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + parentDir := stdpath.Dir(obj.GetPath()) + tree, treeSha, err := d.getTreeDirectly(parentDir) + if err != nil { + return err + } + var del *TreeObjReq = nil + for _, t := range tree.Trees { + if t.Path == obj.GetName() { + if t.Type == "commit" { + return errors.New("cannot remove a submodule") + } + del = &t.TreeObjReq + del.Sha = nil + break + } + } + if del == nil { + return errs.ObjectNotFound + } + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, *del) + if len(tree.Trees) == 1 { // completely emptying the repository will get a 404 + newTree = append(newTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + newSha, err := d.newTree(treeSha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(parentDir, treeSha, newSha, "/") + if err != nil { + return err + } + commitMessage, err := getMessage(d.deleteMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: obj.GetName(), + ObjPath: obj.GetPath(), + ParentName: stdpath.Base(parentDir), + ParentPath: parentDir, + }, "remove") + if err != nil { + return err + } + return d.commit(commitMessage, rootSha) +} + +func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + blob, err := d.putBlob(ctx, stream, up) + if err != nil { + return err + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + parent, err := d.get(dstDir.GetPath()) + if err != nil { + return err + } + if parent.Entries == nil { + return errs.NotFolder + } + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, TreeObjReq{ + Path: stream.GetName(), + Mode: "100644", + Type: "blob", + Sha: blob, + }) + if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { + newTree = append(newTree, TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + }) + } + newSha, err := d.newTree(parent.Sha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(dstDir.GetPath(), parent.Sha, newSha, "/") + if err != nil { + return err + } + + commitMessage, err := getMessage(d.putMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: stream.GetName(), + ObjPath: stdpath.Join(dstDir.GetPath(), stream.GetName()), + ParentName: dstDir.GetName(), + ParentPath: dstDir.GetPath(), + }, "upload") + if err != nil { + return err + } + return d.commit(commitMessage, rootSha) +} + +var _ driver.Driver = (*Github)(nil) + +func (d *Github) getContentApiUrl(path string) string { + path = utils.FixAndCleanPath(path) + return fmt.Sprintf("https://api.github.com/repos/%s/%s/contents%s", d.Owner, d.Repo, path) +} + +func (d *Github) get(path string) (*Object, error) { + res, err := d.client.R().SetQueryParam("ref", d.Ref).Get(d.getContentApiUrl(path)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp Object + err = utils.Json.Unmarshal(res.Body(), &resp) + return &resp, err +} + +func (d *Github) createGitKeep(path, message string) error { + body := map[string]interface{}{ + "message": message, + "content": "", + "branch": d.Ref, + } + d.addCommitterAndAuthor(&body) + + res, err := d.client.R().SetBody(body).Put(d.getContentApiUrl(stdpath.Join(path, ".gitkeep"))) + if err != nil { + return err + } + if res.StatusCode() != 200 && res.StatusCode() != 201 { + return toErr(res) + } + return nil +} + +func (d *Github) putBlob(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) (string, error) { + beforeContent := "{\"encoding\":\"base64\",\"content\":\"" + afterContent := "\"}" + length := int64(len(beforeContent)) + calculateBase64Length(stream.GetSize()) + int64(len(afterContent)) + beforeContentReader := strings.NewReader(beforeContent) + contentReader, contentWriter := io.Pipe() + go func() { + encoder := base64.NewEncoder(base64.StdEncoding, contentWriter) + if _, err := io.Copy(encoder, stream); err != nil { + _ = contentWriter.CloseWithError(err) + return + } + _ = encoder.Close() + _ = contentWriter.Close() + }() + afterContentReader := strings.NewReader(afterContent) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("https://api.github.com/repos/%s/%s/git/blobs", d.Owner, d.Repo), + &ReaderWithProgress{ + Reader: io.MultiReader(beforeContentReader, contentReader, afterContentReader), + Length: length, + Progress: up, + }) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("Authorization", "Bearer "+d.Token) + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + req.ContentLength = length + + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + resBody, err := io.ReadAll(res.Body) + if err != nil { + return "", err + } + if res.StatusCode != 201 { + var errMsg ErrResp + if err = utils.Json.Unmarshal(resBody, &errMsg); err != nil { + return "", errors.New(res.Status) + } else { + return "", fmt.Errorf("%s: %s", res.Status, errMsg.Message) + } + } + var resp PutBlobResp + if err = utils.Json.Unmarshal(resBody, &resp); err != nil { + return "", err + } + return resp.Sha, nil +} + +func (d *Github) delete(path, sha, message string) error { + body := map[string]interface{}{ + "message": message, + "sha": sha, + "branch": d.Ref, + } + d.addCommitterAndAuthor(&body) + res, err := d.client.R().SetBody(body).Delete(d.getContentApiUrl(path)) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toErr(res) + } + return nil +} + +func (d *Github) renewParentTrees(path, prevSha, curSha, until string) (string, error) { + for path != until { + path = stdpath.Dir(path) + tree, sha, err := d.getTreeDirectly(path) + if err != nil { + return "", err + } + var newTree *TreeObjReq = nil + for _, t := range tree.Trees { + if t.Sha == prevSha { + newTree = &t.TreeObjReq + newTree.Sha = curSha + break + } + } + if newTree == nil { + return "", errs.ObjectNotFound + } + curSha, err = d.newTree(sha, []interface{}{*newTree}) + if err != nil { + return "", err + } + prevSha = sha + } + return curSha, nil +} + +func (d *Github) getTree(sha string) (*TreeResp, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees/%s", d.Owner, d.Repo, sha)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp TreeResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Github) getTreeDirectly(path string) (*TreeResp, string, error) { + p, err := d.get(path) + if err != nil { + return nil, "", err + } + if p.Entries == nil { + return nil, "", fmt.Errorf("%s is not a folder", path) + } + tree, err := d.getTree(p.Sha) + if err != nil { + return nil, "", err + } + if tree.Truncated { + return nil, "", fmt.Errorf("tree %s is truncated", path) + } + return tree, p.Sha, nil +} + +func (d *Github) newTree(baseSha string, tree []interface{}) (string, error) { + res, err := d.client.R(). + SetBody(&TreeReq{ + BaseTree: baseSha, + Trees: tree, + }). + Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees", d.Owner, d.Repo)) + if err != nil { + return "", err + } + if res.StatusCode() != 201 { + return "", toErr(res) + } + var resp TreeResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + return resp.Sha, nil +} + +func (d *Github) commit(message, treeSha string) error { + oldCommit, err := d.getBranchHead() + body := map[string]interface{}{ + "message": message, + "tree": treeSha, + "parents": []string{oldCommit}, + } + d.addCommitterAndAuthor(&body) + res, err := d.client.R().SetBody(body).Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/commits", d.Owner, d.Repo)) + if err != nil { + return err + } + if res.StatusCode() != 201 { + return toErr(res) + } + var resp CommitResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return err + } + + // update branch head + res, err = d.client.R(). + SetBody(&UpdateRefReq{ + Sha: resp.Sha, + Force: false, + }). + Patch(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/refs/heads/%s", d.Owner, d.Repo, d.Ref)) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toErr(res) + } + return nil +} + +func (d *Github) getBranchHead() (string, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/branches/%s", d.Owner, d.Repo, d.Ref)) + if err != nil { + return "", err + } + if res.StatusCode() != 200 { + return "", toErr(res) + } + var resp BranchResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + return resp.Commit.Sha, nil +} + +func (d *Github) copyWithoutRenewTree(srcObj, dstDir model.Obj) (dstSha, newSha, srcParentSha string, srcParentTree *TreeResp, err error) { + dst, err := d.get(dstDir.GetPath()) + if err != nil { + return "", "", "", nil, err + } + if dst.Entries == nil { + return "", "", "", nil, errs.NotFolder + } + dstSha = dst.Sha + srcParentPath := stdpath.Dir(srcObj.GetPath()) + srcParentTree, srcParentSha, err = d.getTreeDirectly(srcParentPath) + if err != nil { + return "", "", "", nil, err + } + var src *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return "", "", "", nil, errors.New("cannot copy a submodule") + } + src = &t.TreeObjReq + break + } + } + if src == nil { + return "", "", "", nil, errs.ObjectNotFound + } + + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, *src) + if len(dst.Entries) == 1 && dst.Entries[0].Name == ".gitkeep" { + newTree = append(newTree, TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + }) + } + newSha, err = d.newTree(dstSha, newTree) + if err != nil { + return "", "", "", nil, err + } + return dstSha, newSha, srcParentSha, srcParentTree, nil +} + +func (d *Github) getRepo() (*RepoResp, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s", d.Owner, d.Repo)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp RepoResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Github) addCommitterAndAuthor(m *map[string]interface{}) { + if d.CommitterName != "" { + committer := map[string]string{ + "name": d.CommitterName, + "email": d.CommitterEmail, + } + (*m)["committer"] = committer + } + if d.AuthorName != "" { + author := map[string]string{ + "name": d.AuthorName, + "email": d.AuthorEmail, + } + (*m)["author"] = author + } +} diff --git a/drivers/github/meta.go b/drivers/github/meta.go new file mode 100644 index 00000000000..0df4aa60988 --- /dev/null +++ b/drivers/github/meta.go @@ -0,0 +1,36 @@ +package github + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Token string `json:"token" type:"string" required:"true"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` + CommitterName string `json:"committer_name" type:"string"` + CommitterEmail string `json:"committer_email" type:"string"` + AuthorName string `json:"author_name" type:"string"` + AuthorEmail string `json:"author_email" type:"string"` + MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` + DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` + PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` + RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` + CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` + MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` +} + +var config = driver.Config{ + Name: "GitHub API", + LocalSort: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Github{} + }) +} diff --git a/drivers/github/types.go b/drivers/github/types.go new file mode 100644 index 00000000000..425f89795a7 --- /dev/null +++ b/drivers/github/types.go @@ -0,0 +1,102 @@ +package github + +import ( + "github.com/alist-org/alist/v3/internal/model" + "time" +) + +type Links struct { + Git string `json:"git"` + Html string `json:"html"` + Self string `json:"self"` +} + +type Object struct { + Type string `json:"type"` + Encoding string `json:"encoding" required:"false"` + Size int64 `json:"size"` + Name string `json:"name"` + Path string `json:"path"` + Content string `json:"Content" required:"false"` + Sha string `json:"sha"` + URL string `json:"url"` + GitURL string `json:"git_url"` + HtmlURL string `json:"html_url"` + DownloadURL string `json:"download_url"` + Entries []Object `json:"entries" required:"false"` + Links Links `json:"_links"` + SubmoduleGitURL string `json:"submodule_git_url" required:"false"` + Target string `json:"target" required:"false"` +} + +func (o *Object) toModelObj() *model.Object { + return &model.Object{ + Name: o.Name, + Size: o.Size, + Modified: time.Unix(0, 0), + IsFolder: o.Type == "dir", + } +} + +type PutBlobResp struct { + URL string `json:"url"` + Sha string `json:"sha"` +} + +type ErrResp struct { + Message string `json:"message"` + DocumentationURL string `json:"documentation_url"` + Status string `json:"status"` +} + +type TreeObjReq struct { + Path string `json:"path"` + Mode string `json:"mode"` + Type string `json:"type"` + Sha interface{} `json:"sha"` +} + +type TreeObjResp struct { + TreeObjReq + Size int64 `json:"size" required:"false"` + URL string `json:"url"` +} + +func (o *TreeObjResp) toModelObj() *model.Object { + return &model.Object{ + Name: o.Path, + Size: o.Size, + Modified: time.Unix(0, 0), + IsFolder: o.Type == "tree", + } +} + +type TreeResp struct { + Sha string `json:"sha"` + URL string `json:"url"` + Trees []TreeObjResp `json:"tree"` + Truncated bool `json:"truncated"` +} + +type TreeReq struct { + BaseTree string `json:"base_tree"` + Trees []interface{} `json:"tree"` +} + +type CommitResp struct { + Sha string `json:"sha"` +} + +type BranchResp struct { + Name string `json:"name"` + Commit CommitResp `json:"commit"` +} + +type UpdateRefReq struct { + Sha string `json:"sha"` + Force bool `json:"force"` +} + +type RepoResp struct { + DefaultBranch string `json:"default_branch"` +} diff --git a/drivers/github/util.go b/drivers/github/util.go new file mode 100644 index 00000000000..1e7f7fdbf36 --- /dev/null +++ b/drivers/github/util.go @@ -0,0 +1,115 @@ +package github + +import ( + "context" + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "io" + "math" + "strings" + "text/template" +) + +type ReaderWithProgress struct { + Reader io.Reader + Length int64 + Progress func(percentage float64) + offset int64 +} + +func (r *ReaderWithProgress) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + r.offset += int64(n) + r.Progress(math.Min(100.0, float64(r.offset)/float64(r.Length)*100.0)) + return n, err +} + +type MessageTemplateVars struct { + UserName string + ObjName string + ObjPath string + ParentName string + ParentPath string + TargetName string + TargetPath string +} + +func getMessage(tmpl *template.Template, vars *MessageTemplateVars, defaultOpStr string) (string, error) { + sb := strings.Builder{} + if err := tmpl.Execute(&sb, vars); err != nil { + return fmt.Sprintf("%s %s %s", vars.UserName, defaultOpStr, vars.ObjPath), err + } + return sb.String(), nil +} + +func calculateBase64Length(inputLength int64) int64 { + return 4 * ((inputLength + 2) / 3) +} + +func toErr(res *resty.Response) error { + var errMsg ErrResp + if err := utils.Json.Unmarshal(res.Body(), &errMsg); err != nil { + return errors.New(res.Status()) + } else { + return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) + } +} + +// Example input: +// a = /aaa/bbb/ccc +// b = /aaa/b11/ddd/ccc +// +// Output: +// ancestor = /aaa +// aChildName = bbb +// bChildName = b11 +// aRest = bbb/ccc +// bRest = b11/ddd/ccc +func getPathCommonAncestor(a, b string) (ancestor, aChildName, bChildName, aRest, bRest string) { + a = utils.FixAndCleanPath(a) + b = utils.FixAndCleanPath(b) + idx := 1 + for idx < len(a) && idx < len(b) { + if a[idx] != b[idx] { + break + } + idx++ + } + aNextIdx := idx + for aNextIdx < len(a) { + if a[aNextIdx] == '/' { + break + } + aNextIdx++ + } + bNextIdx := idx + for bNextIdx < len(b) { + if b[bNextIdx] == '/' { + break + } + bNextIdx++ + } + for idx > 0 { + if a[idx] == '/' { + break + } + idx-- + } + ancestor = utils.FixAndCleanPath(a[:idx]) + aChildName = a[idx+1 : aNextIdx] + bChildName = b[idx+1 : bNextIdx] + aRest = a[idx+1:] + bRest = b[idx+1:] + return ancestor, aChildName, bChildName, aRest, bRest +} + +func getUsername(ctx context.Context) string { + user, ok := ctx.Value("user").(*model.User) + if !ok { + return "" + } + return user.Username +} From b60da9732f22b22d84f015d6aaabcb2f058871d1 Mon Sep 17 00:00:00 2001 From: Jealous Date: Fri, 10 Jan 2025 21:24:44 +0800 Subject: [PATCH 413/659] feat(offline-download): allow using offline download tools in any storage (#7716) * Feat(offline-download): allow using thunder offline download tool in any storage * Feat(offline-download): allow using 115 offline download tool in any storage * Feat(offline-download): allow using pikpak offline download tool in any storage * style(offline-download): unify offline download tool names * feat(offline-download): show available offline download tools only * Fix(offline-download): update unmodified tool names. --------- Co-authored-by: Andy Hsu --- internal/conf/const.go | 9 + internal/offline_download/115/client.go | 23 +- internal/offline_download/pikpak/pikpak.go | 25 +- internal/offline_download/thunder/thunder.go | 25 +- internal/offline_download/tool/add.go | 33 ++- internal/offline_download/tool/all_test.go | 17 -- internal/offline_download/tool/base.go | 29 -- internal/offline_download/tool/download.go | 55 +--- internal/offline_download/tool/tools.go | 6 +- internal/offline_download/tool/transfer.go | 267 +++++++++++++++--- internal/offline_download/tool/util.go | 41 --- .../offline_download/transmission/client.go | 2 +- server/handles/offline_download.go | 147 +++++++++- server/router.go | 3 + 14 files changed, 478 insertions(+), 204 deletions(-) delete mode 100644 internal/offline_download/tool/all_test.go delete mode 100644 internal/offline_download/tool/util.go diff --git a/internal/conf/const.go b/internal/conf/const.go index 99e8c868931..0e534350de3 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -58,6 +58,15 @@ const ( TransmissionUri = "transmission_uri" TransmissionSeedtime = "transmission_seedtime" + // 115 + Pan115TempDir = "115_temp_dir" + + // pikpak + PikPakTempDir = "pikpak_temp_dir" + + // thunder + ThunderTempDir = "thunder_temp_dir" + // single Token = "token" IndexProgress = "index_progress" diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go index 45f147db06d..3f9d804dabf 100644 --- a/internal/offline_download/115/client.go +++ b/internal/offline_download/115/client.go @@ -3,6 +3,8 @@ package _115 import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/drivers/115" "github.com/alist-org/alist/v3/internal/errs" @@ -33,13 +35,23 @@ func (p *Cloud115) Init() (string, error) { } func (p *Cloud115) IsReady() bool { + tempDir := setting.GetStr(conf.Pan115TempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*_115.Pan115); !ok { + return false + } return true } func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 p.refreshTaskCache = true - // args.TempDir 已经被修改为了 DstDirPath storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err @@ -50,6 +62,11 @@ func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { } ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err @@ -64,7 +81,7 @@ func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { } func (p *Cloud115) Remove(task *tool.DownloadTask) error { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } @@ -81,7 +98,7 @@ func (p *Cloud115) Remove(task *tool.DownloadTask) error { } func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go index f07b3de8aa0..8fdfb3405cf 100644 --- a/internal/offline_download/pikpak/pikpak.go +++ b/internal/offline_download/pikpak/pikpak.go @@ -3,6 +3,8 @@ package pikpak import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" "strconv" "github.com/alist-org/alist/v3/drivers/pikpak" @@ -17,7 +19,7 @@ type PikPak struct { } func (p *PikPak) Name() string { - return "pikpak" + return "PikPak" } func (p *PikPak) Items() []model.SettingItem { @@ -34,13 +36,23 @@ func (p *PikPak) Init() (string, error) { } func (p *PikPak) IsReady() bool { + tempDir := setting.GetStr(conf.PikPakTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*pikpak.PikPak); !ok { + return false + } return true } func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 p.refreshTaskCache = true - // args.TempDir 已经被修改为了 DstDirPath storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err @@ -51,6 +63,11 @@ func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { } ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err @@ -65,7 +82,7 @@ func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { } func (p *PikPak) Remove(task *tool.DownloadTask) error { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } @@ -82,7 +99,7 @@ func (p *PikPak) Remove(task *tool.DownloadTask) error { } func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } diff --git a/internal/offline_download/thunder/thunder.go b/internal/offline_download/thunder/thunder.go index 3ab8b00212b..81b9486184f 100644 --- a/internal/offline_download/thunder/thunder.go +++ b/internal/offline_download/thunder/thunder.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" "strconv" "github.com/alist-org/alist/v3/drivers/thunder" @@ -18,7 +20,7 @@ type Thunder struct { } func (t *Thunder) Name() string { - return "thunder" + return "Thunder" } func (t *Thunder) Items() []model.SettingItem { @@ -35,13 +37,23 @@ func (t *Thunder) Init() (string, error) { } func (t *Thunder) IsReady() bool { + tempDir := setting.GetStr(conf.ThunderTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*thunder.Thunder); !ok { + return false + } return true } func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { // 添加新任务刷新缓存 t.refreshTaskCache = true - // args.TempDir 已经被修改为了 DstDirPath storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) if err != nil { return "", err @@ -52,6 +64,11 @@ func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { } ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) if err != nil { return "", err @@ -66,7 +83,7 @@ func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { } func (t *Thunder) Remove(task *tool.DownloadTask) error { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return err } @@ -83,7 +100,7 @@ func (t *Thunder) Remove(task *tool.DownloadTask) error { } func (t *Thunder) Status(task *tool.DownloadTask) (*tool.Status, error) { - storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + storage, _, err := op.GetStorageAndActualPath(task.TempDir) if err != nil { return nil, err } diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 405f96cbe2d..884e166bab7 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,8 +2,12 @@ package tool import ( "context" + _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/internal/task" "net/url" "path" @@ -76,19 +80,26 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) deletePolicy := args.DeletePolicy + // 如果当前 storage 是对应网盘，则直接下载到目标路径，无需转存 switch args.Tool { case "115 Cloud": - tempDir = args.DstDirPath - // 防止将下载好的文件删除 - deletePolicy = DeleteNever - case "pikpak": - tempDir = args.DstDirPath - // 防止将下载好的文件删除 - deletePolicy = DeleteNever - case "thunder": - tempDir = args.DstDirPath - // 防止将下载好的文件删除 - deletePolicy = DeleteNever + if _, ok := storage.(*_115.Pan115); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.Pan115TempDir), uid) + } + case "PikPak": + if _, ok := storage.(*pikpak.PikPak); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.PikPakTempDir), uid) + } + case "Thunder": + if _, ok := storage.(*thunder.Thunder); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.ThunderTempDir), uid) + } } taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed diff --git a/internal/offline_download/tool/all_test.go b/internal/offline_download/tool/all_test.go deleted file mode 100644 index 27da5e32a89..00000000000 --- a/internal/offline_download/tool/all_test.go +++ /dev/null @@ -1,17 +0,0 @@ -package tool_test - -import ( - "testing" - - "github.com/alist-org/alist/v3/internal/offline_download/tool" -) - -func TestGetFiles(t *testing.T) { - files, err := tool.GetFiles("..") - if err != nil { - t.Fatal(err) - } - for _, file := range files { - t.Log(file.Name, file.Size, file.Path, file.Modified) - } -} diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go index ae9eac2624b..b14169f8a83 100644 --- a/internal/offline_download/tool/base.go +++ b/internal/offline_download/tool/base.go @@ -1,10 +1,6 @@ package tool import ( - "io" - "os" - "time" - "github.com/alist-org/alist/v3/internal/model" ) @@ -40,28 +36,3 @@ type Tool interface { // Run for simple http download Run(task *DownloadTask) error } - -type GetFileser interface { - // GetFiles return the files of the download task, if nil, means walk the temp dir to get the files - GetFiles(task *DownloadTask) []File -} - -type File struct { - // ReadCloser for http client - ReadCloser io.ReadCloser - Name string - Size int64 - Path string - Modified time.Time -} - -func (f *File) GetReadCloser() (io.ReadCloser, error) { - if f.ReadCloser != nil { - return f.ReadCloser, nil - } - file, err := os.Open(f.Path) - if err != nil { - return nil, err - } - return file, nil -} diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 94bf7dbb660..c3b30f1b4cf 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -40,7 +40,7 @@ func (t *DownloadTask) Run() error { } if err := t.tool.Run(t); !errs.IsNotSupportError(err) { if err == nil { - return t.Complete() + return t.Transfer() } return err } @@ -80,10 +80,10 @@ outer: if err != nil { return err } - if t.tool.Name() == "pikpak" { + if t.tool.Name() == "Pikpak" { return nil } - if t.tool.Name() == "thunder" { + if t.tool.Name() == "Thunder" { return nil } if t.tool.Name() == "115 Cloud" { @@ -109,7 +109,7 @@ outer: } } - if t.tool.Name() == "transmission" { + if t.tool.Name() == "Transmission" { // hack for transmission seedTime := setting.GetInt(conf.TransmissionSeedtime, 0) if seedTime >= 0 { @@ -146,7 +146,7 @@ func (t *DownloadTask) Update() (bool, error) { } // if download completed if info.Completed { - err := t.Complete() + err := t.Transfer() return true, errors.WithMessage(err, "failed to transfer file") } // if download failed @@ -156,45 +156,16 @@ func (t *DownloadTask) Update() (bool, error) { return false, nil } -func (t *DownloadTask) Complete() error { - var ( - files []File - err error - ) - if t.tool.Name() == "pikpak" { - return nil - } - if t.tool.Name() == "thunder" { - return nil - } - if t.tool.Name() == "115 Cloud" { - return nil - } - if getFileser, ok := t.tool.(GetFileser); ok { - files = getFileser.GetFiles(t) - } else { - files, err = GetFiles(t.TempDir) - if err != nil { - return errors.Wrapf(err, "failed to get files") - } - } - // upload files - for i := range files { - file := files[i] - tsk := &TransferTask{ - TaskExtension: task.TaskExtension{ - Creator: t.GetCreator(), - }, - file: file, - DstDirPath: t.DstDirPath, - TempDir: t.TempDir, - DeletePolicy: t.DeletePolicy, - FileDir: file.Path, +func (t *DownloadTask) Transfer() error { + toolName := t.tool.Name() + if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" { + // 如果不是直接下载到目标路径，则进行转存 + if t.TempDir != t.DstDirPath { + return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) } - tsk.SetTotalBytes(file.Size) - TransferTaskManager.Add(tsk) + return nil } - return nil + return transferStd(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) } func (t *DownloadTask) GetName() string { diff --git a/internal/offline_download/tool/tools.go b/internal/offline_download/tool/tools.go index 9de7d526ab0..4a31ac7f6b9 100644 --- a/internal/offline_download/tool/tools.go +++ b/internal/offline_download/tool/tools.go @@ -3,6 +3,7 @@ package tool import ( "fmt" "github.com/alist-org/alist/v3/internal/model" + "sort" ) var ( @@ -25,8 +26,11 @@ func (t ToolsManager) Add(tool Tool) { func (t ToolsManager) Names() []string { names := make([]string, 0, len(t)) for name := range t { - names = append(names, name) + if tool, err := t.Get(name); err == nil && tool.IsReady() { + names = append(names, name) + } } + sort.Strings(names) return names } diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index a77c4822f83..8c7ab2448e6 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -1,11 +1,9 @@ package tool import ( + "context" "fmt" - "os" - "path/filepath" - "time" - + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" @@ -14,84 +12,263 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/xhofe/tache" + "net/http" + "os" + stdpath "path" + "path/filepath" + "time" ) type TransferTask struct { task.TaskExtension - FileDir string `json:"file_dir"` - DstDirPath string `json:"dst_dir_path"` - TempDir string `json:"temp_dir"` - DeletePolicy DeletePolicy `json:"delete_policy"` - file File + Status string `json:"-"` //don't save status to save space + SrcObjPath string `json:"src_obj_path"` + DstDirPath string `json:"dst_dir_path"` + SrcStorage driver.Driver `json:"-"` + DstStorage driver.Driver `json:"-"` + SrcStorageMp string `json:"src_storage_mp"` + DstStorageMp string `json:"dst_storage_mp"` + DeletePolicy DeletePolicy `json:"delete_policy"` } func (t *TransferTask) Run() error { t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() - // check dstDir again - var err error - if (t.file == File{}) { - t.file, err = GetFile(t.FileDir) + if t.SrcStorage == nil { + return transferStdPath(t) + } else { + return transferObjPath(t) + } +} + +func (t *TransferTask) GetName() string { + return fmt.Sprintf("transfer [%s](%s) to [%s](%s)", t.SrcStorageMp, t.SrcObjPath, t.DstStorageMp, t.DstDirPath) +} + +func (t *TransferTask) GetStatus() string { + return t.Status +} + +func (t *TransferTask) OnSucceeded() { + if t.DeletePolicy == DeleteOnUploadSucceed || t.DeletePolicy == DeleteAlways { + if t.SrcStorage == nil { + removeStdTemp(t) + } else { + removeObjTemp(t) + } + } +} + +func (t *TransferTask) OnFailed() { + if t.DeletePolicy == DeleteOnUploadFailed || t.DeletePolicy == DeleteAlways { + if t.SrcStorage == nil { + removeStdTemp(t) + } else { + removeObjTemp(t) + } + } +} + +var ( + TransferTaskManager *tache.Manager[*TransferTask] +) + +func transferStd(ctx context.Context, tempDir, dstDirPath string, deletePolicy DeletePolicy) error { + dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get dst storage") + } + entries, err := os.ReadDir(tempDir) + if err != nil { + return err + } + taskCreator, _ := ctx.Value("user").(*model.User) + for _, entry := range entries { + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + SrcObjPath: stdpath.Join(tempDir, entry.Name()), + DstDirPath: dstDirActualPath, + DstStorage: dstStorage, + DstStorageMp: dstStorage.GetStorage().MountPath, + DeletePolicy: deletePolicy, + } + TransferTaskManager.Add(t) + } + return nil +} + +func transferStdPath(t *TransferTask) error { + t.Status = "getting src object" + info, err := os.Stat(t.SrcObjPath) + if err != nil { + return err + } + if info.IsDir() { + t.Status = "src object is dir, listing objs" + entries, err := os.ReadDir(t.SrcObjPath) if err != nil { - return errors.Wrapf(err, "failed to get file %s", t.FileDir) + return err + } + for _, entry := range entries { + srcRawPath := stdpath.Join(t.SrcObjPath, entry.Name()) + dstObjPath := stdpath.Join(t.DstDirPath, info.Name()) + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: t.Creator, + }, + SrcObjPath: srcRawPath, + DstDirPath: dstObjPath, + DstStorage: t.DstStorage, + SrcStorageMp: t.SrcStorageMp, + DstStorageMp: t.DstStorageMp, + DeletePolicy: t.DeletePolicy, + } + TransferTaskManager.Add(t) } + t.Status = "src object is dir, added all transfer tasks of files" + return nil } - storage, dstDirActualPath, err := op.GetStorageAndActualPath(t.DstDirPath) + return transferStdFile(t) +} + +func transferStdFile(t *TransferTask) error { + rc, err := os.Open(t.SrcObjPath) if err != nil { - return errors.WithMessage(err, "failed get storage") + return errors.Wrapf(err, "failed to open file %s", t.SrcObjPath) } - mimetype := utils.GetMimeType(t.file.Path) - rc, err := t.file.GetReadCloser() + info, err := rc.Stat() if err != nil { - return errors.Wrapf(err, "failed to open file %s", t.file.Path) + return errors.Wrapf(err, "failed to get file %s", t.SrcObjPath) } + mimetype := utils.GetMimeType(t.SrcObjPath) s := &stream.FileStream{ Ctx: nil, Obj: &model.Object{ - Name: filepath.Base(t.file.Path), - Size: t.file.Size, - Modified: t.file.Modified, + Name: filepath.Base(t.SrcObjPath), + Size: info.Size(), + Modified: info.ModTime(), IsFolder: false, }, Reader: rc, Mimetype: mimetype, Closers: utils.NewClosers(rc), } - relDir, err := filepath.Rel(t.TempDir, filepath.Dir(t.file.Path)) - if err != nil { - log.Errorf("find relation directory error: %v", err) - } - newDistDir := filepath.Join(dstDirActualPath, relDir) - return op.Put(t.Ctx(), storage, newDistDir, s, t.SetProgress) + t.SetTotalBytes(info.Size()) + return op.Put(t.Ctx(), t.DstStorage, t.DstDirPath, s, t.SetProgress) } -func (t *TransferTask) GetName() string { - return fmt.Sprintf("transfer %s to [%s]", t.file.Path, t.DstDirPath) +func removeStdTemp(t *TransferTask) { + info, err := os.Stat(t.SrcObjPath) + if err != nil || info.IsDir() { + return + } + if err := os.Remove(t.SrcObjPath); err != nil { + log.Errorf("failed to delete temp file %s, error: %s", t.SrcObjPath, err.Error()) + } } -func (t *TransferTask) GetStatus() string { - return "transferring" +func transferObj(ctx context.Context, tempDir, dstDirPath string, deletePolicy DeletePolicy) error { + srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return errors.WithMessage(err, "failed get src storage") + } + dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get dst storage") + } + objs, err := op.List(ctx, srcStorage, srcObjActualPath, model.ListArgs{}) + if err != nil { + return errors.WithMessagef(err, "failed list src [%s] objs", tempDir) + } + taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed + for _, obj := range objs { + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + SrcObjPath: stdpath.Join(srcObjActualPath, obj.GetName()), + DstDirPath: dstDirActualPath, + SrcStorage: srcStorage, + DstStorage: dstStorage, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, + DeletePolicy: deletePolicy, + } + TransferTaskManager.Add(t) + } + return nil } -func (t *TransferTask) OnSucceeded() { - if t.DeletePolicy == DeleteOnUploadSucceed || t.DeletePolicy == DeleteAlways { - err := os.Remove(t.file.Path) +func transferObjPath(t *TransferTask) error { + t.Status = "getting src object" + srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil { + return errors.WithMessagef(err, "failed get src [%s] file", t.SrcObjPath) + } + if srcObj.IsDir() { + t.Status = "src object is dir, listing objs" + objs, err := op.List(t.Ctx(), t.SrcStorage, t.SrcObjPath, model.ListArgs{}) if err != nil { - log.Errorf("failed to delete file %s, error: %s", t.file.Path, err.Error()) + return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcObjPath) } + for _, obj := range objs { + if utils.IsCanceled(t.Ctx()) { + return nil + } + srcObjPath := stdpath.Join(t.SrcObjPath, obj.GetName()) + dstObjPath := stdpath.Join(t.DstDirPath, srcObj.GetName()) + TransferTaskManager.Add(&TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: t.Creator, + }, + SrcObjPath: srcObjPath, + DstDirPath: dstObjPath, + SrcStorage: t.SrcStorage, + DstStorage: t.DstStorage, + SrcStorageMp: t.SrcStorageMp, + DstStorageMp: t.DstStorageMp, + DeletePolicy: t.DeletePolicy, + }) + } + t.Status = "src object is dir, added all transfer tasks of objs" + return nil } + return transferObjFile(t) } -func (t *TransferTask) OnFailed() { - if t.DeletePolicy == DeleteOnUploadFailed || t.DeletePolicy == DeleteAlways { - err := os.Remove(t.file.Path) - if err != nil { - log.Errorf("failed to delete file %s, error: %s", t.file.Path, err.Error()) - } +func transferObjFile(t *TransferTask) error { + srcFile, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil { + return errors.WithMessagef(err, "failed get src [%s] file", t.SrcObjPath) + } + link, _, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcObjPath, model.LinkArgs{ + Header: http.Header{}, + }) + if err != nil { + return errors.WithMessagef(err, "failed get [%s] link", t.SrcObjPath) + } + fs := stream.FileStream{ + Obj: srcFile, + Ctx: t.Ctx(), } + // any link provided is seekable + ss, err := stream.NewSeekableStream(fs, link) + if err != nil { + return errors.WithMessagef(err, "failed get [%s] stream", t.SrcObjPath) + } + t.SetTotalBytes(srcFile.GetSize()) + return op.Put(t.Ctx(), t.DstStorage, t.DstDirPath, ss, t.SetProgress) } -var ( - TransferTaskManager *tache.Manager[*TransferTask] -) +func removeObjTemp(t *TransferTask) { + srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil || srcObj.IsDir() { + return + } + if err := op.Remove(t.Ctx(), t.SrcStorage, t.SrcObjPath); err != nil { + log.Errorf("failed to delete temp obj %s, error: %s", t.SrcObjPath, err.Error()) + } +} diff --git a/internal/offline_download/tool/util.go b/internal/offline_download/tool/util.go deleted file mode 100644 index b2c6ec02bfa..00000000000 --- a/internal/offline_download/tool/util.go +++ /dev/null @@ -1,41 +0,0 @@ -package tool - -import ( - "os" - "path/filepath" -) - -func GetFiles(dir string) ([]File, error) { - var files []File - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - files = append(files, File{ - Name: info.Name(), - Size: info.Size(), - Path: path, - Modified: info.ModTime(), - }) - } - return nil - }) - if err != nil { - return nil, err - } - return files, nil -} - -func GetFile(path string) (File, error) { - info, err := os.Stat(path) - if err != nil { - return File{}, err - } - return File{ - Name: info.Name(), - Size: info.Size(), - Path: path, - Modified: info.ModTime(), - }, nil -} diff --git a/internal/offline_download/transmission/client.go b/internal/offline_download/transmission/client.go index 4131f3e1c53..8049afd6d35 100644 --- a/internal/offline_download/transmission/client.go +++ b/internal/offline_download/transmission/client.go @@ -29,7 +29,7 @@ func (t *Transmission) Run(task *tool.DownloadTask) error { } func (t *Transmission) Name() string { - return "transmission" + return "Transmission" } func (t *Transmission) Items() []model.SettingItem { diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index c7b7af76c10..24ff7a05369 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -1,6 +1,9 @@ package handles import ( + _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" @@ -73,11 +76,6 @@ func SetQbittorrent(c *gin.Context) { common.SuccessResp(c, "ok") } -func OfflineDownloadTools(c *gin.Context) { - tools := tool.Tools.Names() - common.SuccessResp(c, tools) -} - type SetTransmissionReq struct { Uri string `json:"uri" form:"uri"` Seedtime string `json:"seedtime" form:"seedtime"` @@ -97,7 +95,51 @@ func SetTransmission(c *gin.Context) { common.ErrorResp(c, err, 500) return } - _tool, err := tool.Tools.Get("transmission") + _tool, err := tool.Tools.Get("Transmission") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type Set115Req struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func Set115(c *gin.Context) { + var req Set115Req + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*_115.Pan115); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Cloud is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.Pan115TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("115 Cloud") if err != nil { common.ErrorResp(c, err, 500) return @@ -109,6 +151,99 @@ func SetTransmission(c *gin.Context) { common.SuccessResp(c, "ok") } +type SetPikPakReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetPikPak(c *gin.Context) { + var req SetPikPakReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*pikpak.PikPak); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only PikPak is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.PikPakTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("PikPak") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type SetThunderReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetThunder(c *gin.Context) { + var req SetThunderReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*thunder.Thunder); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only Thunder is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.ThunderTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("Thunder") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +func OfflineDownloadTools(c *gin.Context) { + tools := tool.Tools.Names() + common.SuccessResp(c, tools) +} + type AddOfflineDownloadReq struct { Urls []string `json:"urls"` Path string `json:"path"` diff --git a/server/router.go b/server/router.go index 9ff50365a7a..184de51e46a 100644 --- a/server/router.go +++ b/server/router.go @@ -132,6 +132,9 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_aria2", handles.SetAria2) setting.POST("/set_qbit", handles.SetQbittorrent) setting.POST("/set_transmission", handles.SetTransmission) + setting.POST("/set_115", handles.Set115) + setting.POST("/set_pikpak", handles.SetPikPak) + setting.POST("/set_thunder", handles.SetThunder) // retain /admin/task API to ensure compatibility with legacy automation scripts _task(g.Group("/task")) From 880cc7abca72b86877efad275a845de9f8a2a1d0 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sat, 18 Jan 2025 23:24:09 +0800 Subject: [PATCH 414/659] fix(139): use `personal_new` by default (#7836) --- drivers/139/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/139/meta.go b/drivers/139/meta.go index 680e469ded9..d80b8566131 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -9,7 +9,7 @@ type Addition struct { //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` driver.RootID - Type string `json:"type" type:"select" options:"personal,family,personal_new" default:"personal"` + Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` CloudID string `json:"cloud_id"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` } From ab22cf823345f4f6d5473f9aa6fa31cefcefe1b0 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sat, 18 Jan 2025 23:26:58 +0800 Subject: [PATCH 415/659] feat: add `Reference` interface to driver (#7805) * feat: add `Reference` interface to driver * feat(123_share): support reference 123pan --- drivers/123/driver.go | 23 +++--- drivers/123/upload.go | 6 +- drivers/123/util.go | 11 +-- drivers/123_share/driver.go | 15 +++- drivers/123_share/util.go | 3 + drivers/139/driver.go | 109 ++++++++++++++++------------- drivers/139/util.go | 26 +++++-- drivers/189pc/driver.go | 39 +++++++---- drivers/189pc/utils.go | 37 +++++++--- drivers/aliyundrive_open/driver.go | 12 +++- drivers/aliyundrive_open/meta.go | 5 +- drivers/aliyundrive_open/upload.go | 2 +- drivers/aliyundrive_open/util.go | 18 +++-- internal/driver/driver.go | 4 ++ internal/op/storage.go | 24 +++++++ 15 files changed, 230 insertions(+), 104 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 3620431d9b3..3828a59d9f0 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,13 +6,14 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "golang.org/x/time/rate" "io" "net/http" "net/url" "sync" "time" + "golang.org/x/time/rate" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -41,12 +42,12 @@ func (d *Pan123) GetAddition() driver.Additional { } func (d *Pan123) Init(ctx context.Context) error { - _, err := d.request(UserInfo, http.MethodGet, nil, nil) + _, err := d.Request(UserInfo, http.MethodGet, nil, nil) return err } func (d *Pan123) Drop(ctx context.Context) error { - _, _ = d.request(Logout, http.MethodPost, func(req *resty.Request) { + _, _ = d.Request(Logout, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{}) }, nil) return nil @@ -81,8 +82,8 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) "size": f.Size, "type": f.Type, } - resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) { - + resp, err := d.Request(DownloadInfo, http.MethodPost, func(req *resty.Request) { + req.SetBody(data).SetHeaders(headers) }, nil) if err != nil { @@ -135,7 +136,7 @@ func (d *Pan123) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin "size": 0, "type": 1, } - _, err := d.request(Mkdir, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(Mkdir, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err @@ -146,7 +147,7 @@ func (d *Pan123) Move(ctx context.Context, srcObj, dstDir model.Obj) error { "fileIdList": []base.Json{{"FileId": srcObj.GetID()}}, "parentFileId": dstDir.GetID(), } - _, err := d.request(Move, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(Move, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err @@ -158,7 +159,7 @@ func (d *Pan123) Rename(ctx context.Context, srcObj model.Obj, newName string) e "fileId": srcObj.GetID(), "fileName": newName, } - _, err := d.request(Rename, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(Rename, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err @@ -175,7 +176,7 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error { "operation": true, "fileTrashInfoList": []File{f}, } - _, err := d.request(Trash, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(Trash, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err @@ -213,7 +214,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "type": 0, } var resp UploadResp - res, err := d.request(UploadRequest, http.MethodPost, func(req *resty.Request) { + res, err := d.Request(UploadRequest, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &resp) if err != nil { @@ -248,7 +249,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } _, err = uploader.UploadWithContext(ctx, input) } - _, err = d.request(UploadComplete, http.MethodPost, func(req *resty.Request) { + _, err = d.Request(UploadComplete, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileId": resp.Data.FileId, }).SetContext(ctx) diff --git a/drivers/123/upload.go b/drivers/123/upload.go index 6f6221f1148..66627b4cd94 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -25,7 +25,7 @@ func (d *Pan123) getS3PreSignedUrls(ctx context.Context, upReq *UploadResp, star "StorageNode": upReq.Data.StorageNode, } var s3PreSignedUrls S3PreSignedURLs - _, err := d.request(S3PreSignedUrls, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(S3PreSignedUrls, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &s3PreSignedUrls) if err != nil { @@ -44,7 +44,7 @@ func (d *Pan123) getS3Auth(ctx context.Context, upReq *UploadResp, start, end in "uploadId": upReq.Data.UploadId, } var s3PreSignedUrls S3PreSignedURLs - _, err := d.request(S3Auth, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(S3Auth, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &s3PreSignedUrls) if err != nil { @@ -63,7 +63,7 @@ func (d *Pan123) completeS3(ctx context.Context, upReq *UploadResp, file model.F "key": upReq.Data.Key, "uploadId": upReq.Data.UploadId, } - _, err := d.request(UploadCompleteV2, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(UploadCompleteV2, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, nil) return err diff --git a/drivers/123/util.go b/drivers/123/util.go index 6365b1c9a1e..7e5a23970c6 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -194,7 +194,9 @@ func (d *Pan123) login() error { // return &authKey, nil //} -func (d *Pan123) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +func (d *Pan123) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + isRetry := false +do: req := base.RestyClient.R() req.SetHeaders(map[string]string{ "origin": "https://www.123pan.com", @@ -223,12 +225,13 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r body := res.Body() code := utils.Json.Get(body, "code").ToInt() if code != 0 { - if code == 401 { + if !isRetry && code == 401 { err := d.login() if err != nil { return nil, err } - return d.request(url, method, callback, resp) + isRetry = true + goto do } return nil, errors.New(jsoniter.Get(body, "message").ToString()) } @@ -260,7 +263,7 @@ func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([] "operateType": "4", "inDirectSpace": "false", } - _res, err := d.request(FileList, http.MethodGet, func(req *resty.Request) { + _res, err := d.Request(FileList, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { diff --git a/drivers/123_share/driver.go b/drivers/123_share/driver.go index 9c1f3803710..640fb74967e 100644 --- a/drivers/123_share/driver.go +++ b/drivers/123_share/driver.go @@ -4,12 +4,14 @@ import ( "context" "encoding/base64" "fmt" - "golang.org/x/time/rate" "net/http" "net/url" "sync" "time" + "golang.org/x/time/rate" + + _123 "github.com/alist-org/alist/v3/drivers/123" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -23,6 +25,7 @@ type Pan123Share struct { model.Storage Addition apiRateLimit sync.Map + ref *_123.Pan123 } func (d *Pan123Share) Config() driver.Config { @@ -39,7 +42,17 @@ func (d *Pan123Share) Init(ctx context.Context) error { return nil } +func (d *Pan123Share) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*_123.Pan123) + if ok { + d.ref = refStorage + return nil + } + return fmt.Errorf("ref: storage is not 123Pan") +} + func (d *Pan123Share) Drop(ctx context.Context) error { + d.ref = nil return nil } diff --git a/drivers/123_share/util.go b/drivers/123_share/util.go index 80ea8f0ca46..c2140bf604f 100644 --- a/drivers/123_share/util.go +++ b/drivers/123_share/util.go @@ -53,6 +53,9 @@ func GetApi(rawUrl string) string { } func (d *Pan123Share) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if d.ref != nil { + return d.ref.Request(url, method, callback, resp) + } req := base.RestyClient.R() req.SetHeaders(map[string]string{ "origin": "https://www.123pan.com", diff --git a/drivers/139/driver.go b/drivers/139/driver.go index dd154efe42a..ebb30e25d19 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -26,6 +26,7 @@ type Yun139 struct { Addition cron *cron.Cron Account string + ref *Yun139 } func (d *Yun139) Config() driver.Config { @@ -37,61 +38,73 @@ func (d *Yun139) GetAddition() driver.Additional { } func (d *Yun139) Init(ctx context.Context) error { - if d.Authorization == "" { - return fmt.Errorf("authorization is empty") - } - d.cron = cron.NewCron(time.Hour * 24 * 7) - d.cron.Do(func() { - err := d.refreshToken() - if err != nil { - log.Errorf("%+v", err) + if d.ref == nil { + if d.Authorization == "" { + return fmt.Errorf("authorization is empty") } - }) + d.cron = cron.NewCron(time.Hour * 24 * 7) + d.cron.Do(func() { + err := d.refreshToken() + if err != nil { + log.Errorf("%+v", err) + } + }) + } switch d.Addition.Type { case MetaPersonalNew: if len(d.Addition.RootFolderID) == 0 { d.RootFolderID = "/" } - return nil case MetaPersonal: if len(d.Addition.RootFolderID) == 0 { d.RootFolderID = "root" } - fallthrough case MetaGroup: if len(d.Addition.RootFolderID) == 0 { d.RootFolderID = d.CloudID } - fallthrough case MetaFamily: - decode, err := base64.StdEncoding.DecodeString(d.Authorization) - if err != nil { - return err - } - decodeStr := string(decode) - splits := strings.Split(decodeStr, ":") - if len(splits) < 2 { - return fmt.Errorf("authorization is invalid, splits < 2") - } - d.Account = splits[1] - _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ - "qryUserExternInfoReq": base.Json{ - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, - }, - }, nil) - return err default: return errs.NotImplement } + if d.ref != nil { + return nil + } + decode, err := base64.StdEncoding.DecodeString(d.Authorization) + if err != nil { + return err + } + decodeStr := string(decode) + splits := strings.Split(decodeStr, ":") + if len(splits) < 2 { + return fmt.Errorf("authorization is invalid, splits < 2") + } + d.Account = splits[1] + _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ + "qryUserExternInfoReq": base.Json{ + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + }, + }, nil) + return err +} + +func (d *Yun139) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*Yun139) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport } func (d *Yun139) Drop(ctx context.Context) error { if d.cron != nil { d.cron.Stop() } + d.ref = nil return nil } @@ -150,7 +163,7 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin "parentCatalogID": parentDir.GetID(), "newCatalogName": dirName, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, }, @@ -161,7 +174,7 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin data := base.Json{ "cloudID": d.CloudID, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, "docLibName": dirName, @@ -173,7 +186,7 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin data := base.Json{ "catalogName": dirName, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, "groupID": d.CloudID, @@ -219,7 +232,7 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, "contentList": contentList, "catalogList": catalogList, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -247,7 +260,7 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, "newCatalogID": dstDir.GetID(), }, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, }, @@ -282,7 +295,7 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e "catalogID": srcObj.GetID(), "catalogName": newName, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -292,7 +305,7 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e "contentID": srcObj.GetID(), "contentName": newName, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -309,7 +322,7 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e "modifyCatalogName": newName, "path": srcObj.GetPath(), "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -321,7 +334,7 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e "contentName": newName, "path": srcObj.GetPath(), "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -338,7 +351,7 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e // "catalogID": srcObj.GetID(), // "catalogName": newName, // "commonAccountInfo": base.Json{ - // "account": d.Account, + // "account": d.getAccount(), // "accountType": 1, // }, // "path": srcObj.GetPath(), @@ -350,7 +363,7 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e "contentID": srcObj.GetID(), "contentName": newName, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, "path": srcObj.GetPath(), @@ -393,7 +406,7 @@ func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { "newCatalogID": dstDir.GetID(), }, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, }, @@ -430,7 +443,7 @@ func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { "contentList": contentList, "catalogList": catalogList, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -457,7 +470,7 @@ func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { "catalogInfoList": catalogInfoList, }, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, }, @@ -468,7 +481,7 @@ func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { "catalogList": catalogInfoList, "contentList": contentInfoList, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, "sourceCloudID": d.CloudID, @@ -598,7 +611,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "uploadId": resp.Data.UploadId, "partInfos": batchPartInfos, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -735,7 +748,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "parentCatalogID": dstDir.GetID(), "newCatalogName": "", "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } diff --git a/drivers/139/util.go b/drivers/139/util.go index d0b4d3b46a7..2dade2506ad 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -54,6 +54,9 @@ func getTime(t string) time.Time { } func (d *Yun139) refreshToken() error { + if d.ref == nil { + return d.ref.refreshToken() + } url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do" var resp RefreshTokenResp decode, err := base64.StdEncoding.DecodeString(d.Authorization) @@ -99,7 +102,7 @@ func (d *Yun139) request(pathname string, method string, callback base.ReqCallba req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", "CMS-DEVICE": "default", - "Authorization": "Basic " + d.Authorization, + "Authorization": "Basic " + d.getAuthorization(), "mcloud-channel": "1000101", "mcloud-client": "10701", //"mcloud-route": "001", @@ -151,7 +154,7 @@ func (d *Yun139) getFiles(catalogID string) ([]model.Obj, error) { "catalogSortType": 0, "contentSortType": 0, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -199,7 +202,7 @@ func (d *Yun139) newJson(data map[string]interface{}) base.Json { "cloudID": d.CloudID, "cloudType": 1, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -320,7 +323,7 @@ func (d *Yun139) getLink(contentId string) (string, error) { "appName": "", "contentID": contentId, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -383,7 +386,7 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R } req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", - "Authorization": "Basic " + d.Authorization, + "Authorization": "Basic " + d.getAuthorization(), "Caller": "web", "Cms-Device": "default", "Mcloud-Channel": "1000101", @@ -514,3 +517,16 @@ func (d *Yun139) personalGetLink(fileId string) (string, error) { return jsoniter.Get(res, "data", "url").ToString(), nil } } + +func (d *Yun139) getAuthorization() string { + if d.ref != nil { + return d.ref.getAuthorization() + } + return d.Authorization +} +func (d *Yun139) getAccount() string { + if d.ref != nil { + return d.ref.getAccount() + } + return d.Account +} diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 9c01a50fd86..6b502de08b8 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -33,6 +33,7 @@ type Cloud189PC struct { cleanFamilyTransferFile func() storageConfig driver.Config + ref *Cloud189PC } func (y *Cloud189PC) Config() driver.Config { @@ -64,20 +65,22 @@ func (y *Cloud189PC) Init(ctx context.Context) (err error) { y.uploadThread, y.UploadThread = 3, "3" } - // 初始化请求客户端 - if y.client == nil { - y.client = base.NewRestyClient().SetHeaders(map[string]string{ - "Accept": "application/json;charset=UTF-8", - "Referer": WEB_URL, - }) - } + if y.ref == nil { + // 初始化请求客户端 + if y.client == nil { + y.client = base.NewRestyClient().SetHeaders(map[string]string{ + "Accept": "application/json;charset=UTF-8", + "Referer": WEB_URL, + }) + } - // 避免重复登陆 - identity := utils.GetMD5EncodeStr(y.Username + y.Password) - if !y.isLogin() || y.identity != identity { - y.identity = identity - if err = y.login(); err != nil { - return + // 避免重复登陆 + identity := utils.GetMD5EncodeStr(y.Username + y.Password) + if !y.isLogin() || y.identity != identity { + y.identity = identity + if err = y.login(); err != nil { + return + } } } @@ -103,7 +106,17 @@ func (y *Cloud189PC) Init(ctx context.Context) (err error) { return } +func (d *Cloud189PC) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*Cloud189PC) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + func (y *Cloud189PC) Drop(ctx context.Context) error { + y.ref = nil return nil } diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index f5a44455d2e..0c3e54045d0 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -57,11 +57,11 @@ const ( func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { dateOfGmt := getHttpDateStr() - sessionKey := y.tokenInfo.SessionKey - sessionSecret := y.tokenInfo.SessionSecret + sessionKey := y.getTokenInfo().SessionKey + sessionSecret := y.getTokenInfo().SessionSecret if isFamily { - sessionKey = y.tokenInfo.FamilySessionKey - sessionSecret = y.tokenInfo.FamilySessionSecret + sessionKey = y.getTokenInfo().FamilySessionKey + sessionSecret = y.getTokenInfo().FamilySessionSecret } header := map[string]string{ @@ -74,9 +74,9 @@ func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) } func (y *Cloud189PC) EncryptParams(params Params, isFamily bool) string { - sessionSecret := y.tokenInfo.SessionSecret + sessionSecret := y.getTokenInfo().SessionSecret if isFamily { - sessionSecret = y.tokenInfo.FamilySessionSecret + sessionSecret = y.getTokenInfo().FamilySessionSecret } if params != nil { return AesECBEncrypt(params.Encode(), sessionSecret[:16]) @@ -85,7 +85,7 @@ func (y *Cloud189PC) EncryptParams(params Params, isFamily bool) string { } func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, params Params, resp interface{}, isFamily ...bool) ([]byte, error) { - req := y.client.R().SetQueryParams(clientSuffix()) + req := y.getClient().R().SetQueryParams(clientSuffix()) // 设置params paramsData := y.EncryptParams(params, isBool(isFamily...)) @@ -403,6 +403,9 @@ func (y *Cloud189PC) initLoginParam() error { // 刷新会话 func (y *Cloud189PC) refreshSession() (err error) { + if y.ref != nil { + return y.ref.refreshSession() + } var erron RespErr var userSessionResp UserSessionResp _, err = y.client.R(). @@ -620,7 +623,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } // 尝试恢复进度 - uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.tokenInfo.SessionKey, fileMd5Hex) + uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.getTokenInfo().SessionKey, fileMd5Hex) if !ok { //step.2 预上传 params := Params{ @@ -687,7 +690,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { uploadProgress.UploadParts = utils.SliceFilter(uploadProgress.UploadParts, func(s string) bool { return s != "" }) - base.SaveUploadProgress(y, uploadProgress, y.tokenInfo.SessionKey, fileMd5Hex) + base.SaveUploadProgress(y, uploadProgress, y.getTokenInfo().SessionKey, fileMd5Hex) } return nil, err } @@ -1008,7 +1011,7 @@ func (y *Cloud189PC) getFamilyID() (string, error) { return "", fmt.Errorf("cannot get automatically,please input family_id") } for _, info := range infos { - if strings.Contains(y.tokenInfo.LoginName, info.RemarkName) { + if strings.Contains(y.getTokenInfo().LoginName, info.RemarkName) { return fmt.Sprint(info.FamilyID), nil } } @@ -1142,3 +1145,17 @@ func (y *Cloud189PC) WaitBatchTask(aType string, taskID string, t time.Duration) time.Sleep(t) } } + +func (y *Cloud189PC) getTokenInfo() *AppSessionResp { + if y.ref != nil { + return y.ref.getTokenInfo() + } + return y.tokenInfo +} + +func (y *Cloud189PC) getClient() *resty.Client { + if y.ref != nil { + return y.ref.getClient() + } + return y.client +} diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index 4029ad57c37..a65ba05c5af 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -19,12 +19,12 @@ import ( type AliyundriveOpen struct { model.Storage Addition - base string DriveId string limitList func(ctx context.Context, data base.Json) (*Files, error) limitLink func(ctx context.Context, file model.Obj) (*model.Link, error) + ref *AliyundriveOpen } func (d *AliyundriveOpen) Config() driver.Config { @@ -58,7 +58,17 @@ func (d *AliyundriveOpen) Init(ctx context.Context) error { return nil } +func (d *AliyundriveOpen) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*AliyundriveOpen) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + func (d *AliyundriveOpen) Drop(ctx context.Context) error { + d.ref = nil return nil } diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index 5801314396e..03f97f8b795 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -32,11 +32,10 @@ var config = driver.Config{ DefaultRoot: "root", NoOverwriteUpload: true, } +var API_URL = "https://openapi.alipan.com" func init() { op.RegisterDriver(func() driver.Driver { - return &AliyundriveOpen{ - base: "https://openapi.alipan.com", - } + return &AliyundriveOpen{} }) } diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index d152836c075..653a2442346 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -126,7 +126,7 @@ func getProofRange(input string, size int64) (*ProofRange, error) { } func (d *AliyundriveOpen) calProofCode(stream model.FileStreamer) (string, error) { - proofRange, err := getProofRange(d.AccessToken, stream.GetSize()) + proofRange, err := getProofRange(d.getAccessToken(), stream.GetSize()) if err != nil { return "", err } diff --git a/drivers/aliyundrive_open/util.go b/drivers/aliyundrive_open/util.go index 331e6400c97..659d7da7257 100644 --- a/drivers/aliyundrive_open/util.go +++ b/drivers/aliyundrive_open/util.go @@ -19,7 +19,7 @@ import ( // do others that not defined in Driver interface func (d *AliyundriveOpen) _refreshToken() (string, string, error) { - url := d.base + "/oauth/access_token" + url := API_URL + "/oauth/access_token" if d.OauthTokenURL != "" && d.ClientID == "" { url = d.OauthTokenURL } @@ -74,6 +74,9 @@ func getSub(token string) (string, error) { } func (d *AliyundriveOpen) refreshToken() error { + if d.ref != nil { + return d.ref.refreshToken() + } refresh, access, err := d._refreshToken() for i := 0; i < 3; i++ { if err == nil { @@ -100,7 +103,7 @@ func (d *AliyundriveOpen) request(uri, method string, callback base.ReqCallback, func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { req := base.RestyClient.R() // TODO check whether access_token is expired - req.SetHeader("Authorization", "Bearer "+d.AccessToken) + req.SetHeader("Authorization", "Bearer "+d.getAccessToken()) if method == http.MethodPost { req.SetHeader("Content-Type", "application/json") } @@ -109,7 +112,7 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base } var e ErrResp req.SetError(&e) - res, err := req.Execute(method, d.base+uri) + res, err := req.Execute(method, API_URL+uri) if err != nil { if res != nil { log.Errorf("[aliyundrive_open] request error: %s", res.String()) @@ -118,7 +121,7 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base } isRetry := len(retry) > 0 && retry[0] if e.Code != "" { - if !isRetry && (utils.SliceContains([]string{"AccessTokenInvalid", "AccessTokenExpired", "I400JD"}, e.Code) || d.AccessToken == "") { + if !isRetry && (utils.SliceContains([]string{"AccessTokenInvalid", "AccessTokenExpired", "I400JD"}, e.Code) || d.getAccessToken() == "") { err = d.refreshToken() if err != nil { return nil, err, nil @@ -176,3 +179,10 @@ func getNowTime() (time.Time, string) { nowTimeStr := nowTime.Format("2006-01-02T15:04:05.000Z") return nowTime, nowTimeStr } + +func (d *AliyundriveOpen) getAccessToken() string { + if d.ref != nil { + return d.ref.getAccessToken() + } + return d.AccessToken +} diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 6fd5e8d6f90..4571110a8f2 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -144,3 +144,7 @@ func NewProgress(total int64, up UpdateProgress) *Progress { up: up, } } + +type Reference interface { + InitReference(storage Driver) error +} diff --git a/internal/op/storage.go b/internal/op/storage.go index 7d8831f548e..f957f95b596 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -10,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/generic_sync" "github.com/alist-org/alist/v3/pkg/utils" @@ -106,6 +107,29 @@ func initStorage(ctx context.Context, storage model.Storage, storageDriver drive }() // Unmarshal Addition err = utils.Json.UnmarshalFromString(driverStorage.Addition, storageDriver.GetAddition()) + if err == nil { + if ref, ok := storageDriver.(driver.Reference); ok { + if strings.HasPrefix(driverStorage.Remark, "ref:/") { + refMountPath := driverStorage.Remark + i := strings.Index(refMountPath, "\n") + if i > 0 { + refMountPath = refMountPath[4:i] + } else { + refMountPath = refMountPath[4:] + } + var refStorage driver.Driver + refStorage, err = GetStorageByMountPath(refMountPath) + if err != nil { + err = fmt.Errorf("ref: %w", err) + } else { + err = ref.InitReference(refStorage) + if err != nil && errs.IsNotSupportError(err) { + err = fmt.Errorf("ref: storage is not %s", storageDriver.Config().Name) + } + } + } + } + } if err == nil { err = storageDriver.Init(ctx) } From bb40e2e2cdd01b34ba6edfe09448d80e210af177 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Sat, 18 Jan 2025 23:28:12 +0800 Subject: [PATCH 416/659] feat(archive): archive manage (#7817) * feat(archive): archive management * fix(ftp-server): remove duplicate ReadAtSeeker realization * fix(archive): bad seeking of SeekableStream * fix(archive): split internal and driver extraction api * feat(archive): patch * fix(shutdown): clear decompress upload tasks * chore * feat(archive): support .iso format * chore --- cmd/kill.go | 54 +++ cmd/root.go | 1 + cmd/server.go | 2 + cmd/{stop.go => stop_default.go} | 12 +- cmd/stop_windows.go | 34 ++ go.mod | 33 +- go.sum | 244 +++++++++- internal/archive/all.go | 7 + internal/archive/archives/archives.go | 126 ++++++ internal/archive/archives/utils.go | 80 ++++ internal/archive/iso9660/iso9660.go | 96 ++++ internal/archive/iso9660/utils.go | 100 +++++ internal/archive/tool/base.go | 15 + internal/archive/tool/utils.go | 23 + internal/archive/zip/utils.go | 156 +++++++ internal/archive/zip/zip.go | 174 +++++++ internal/bootstrap/data/user.go | 15 +- .../patch/v3_41_0/grant_permission.go | 18 +- internal/bootstrap/task.go | 2 + internal/conf/config.go | 19 +- internal/driver/driver.go | 38 +- internal/errs/errors.go | 4 + internal/fs/archive.go | 395 ++++++++++++++++ internal/fs/fs.go | 41 ++ internal/model/archive.go | 49 ++ internal/model/args.go | 27 ++ internal/model/obj.go | 3 + internal/model/user.go | 10 + internal/op/archive.go | 424 ++++++++++++++++++ internal/stream/stream.go | 218 +++++++++ internal/task/manager.go | 20 + server/ftp/fsread.go | 49 +- server/handles/archive.go | 381 ++++++++++++++++ server/handles/down.go | 82 ++-- server/handles/task.go | 8 +- server/router.go | 11 + 36 files changed, 2849 insertions(+), 122 deletions(-) create mode 100644 cmd/kill.go rename cmd/{stop.go => stop_default.go} (87%) create mode 100644 cmd/stop_windows.go create mode 100644 internal/archive/all.go create mode 100644 internal/archive/archives/archives.go create mode 100644 internal/archive/archives/utils.go create mode 100644 internal/archive/iso9660/iso9660.go create mode 100644 internal/archive/iso9660/utils.go create mode 100644 internal/archive/tool/base.go create mode 100644 internal/archive/tool/utils.go create mode 100644 internal/archive/zip/utils.go create mode 100644 internal/archive/zip/zip.go create mode 100644 internal/fs/archive.go create mode 100644 internal/model/archive.go create mode 100644 internal/op/archive.go create mode 100644 internal/task/manager.go create mode 100644 server/handles/archive.go diff --git a/cmd/kill.go b/cmd/kill.go new file mode 100644 index 00000000000..3378fd70988 --- /dev/null +++ b/cmd/kill.go @@ -0,0 +1,54 @@ +package cmd + +import ( + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "os" +) + +// KillCmd represents the kill command +var KillCmd = &cobra.Command{ + Use: "kill", + Short: "Force kill alist server process by daemon/pid file", + Run: func(cmd *cobra.Command, args []string) { + kill() + }, +} + +func kill() { + initDaemon() + if pid == -1 { + log.Info("Seems not have been started. Try use `alist start` to start server.") + return + } + process, err := os.FindProcess(pid) + if err != nil { + log.Errorf("failed to find process by pid: %d, reason: %v", pid, process) + return + } + err = process.Kill() + if err != nil { + log.Errorf("failed to kill process %d: %v", pid, err) + } else { + log.Info("killed process: ", pid) + } + err = os.Remove(pidFile) + if err != nil { + log.Errorf("failed to remove pid file") + } + pid = -1 +} + +func init() { + RootCmd.AddCommand(KillCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // stopCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // stopCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/root.go b/cmd/root.go index 6bd82b7a4a3..59eb989c3a0 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -6,6 +6,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" _ "github.com/alist-org/alist/v3/drivers" + _ "github.com/alist-org/alist/v3/internal/archive" _ "github.com/alist-org/alist/v3/internal/offline_download" "github.com/spf13/cobra" ) diff --git a/cmd/server.go b/cmd/server.go index 3112a6a9905..d9206cfeb18 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -6,6 +6,7 @@ import ( "fmt" ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" "github.com/KirCute/sftpd-alist" + "github.com/alist-org/alist/v3/internal/fs" "net" "net/http" "os" @@ -159,6 +160,7 @@ the address is defined in config file`, signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit utils.Log.Println("Shutdown server...") + fs.ArchiveContentUploadTaskManager.RemoveAll() Release() ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() diff --git a/cmd/stop.go b/cmd/stop_default.go similarity index 87% rename from cmd/stop.go rename to cmd/stop_default.go index 09fba7b759d..8f133940a29 100644 --- a/cmd/stop.go +++ b/cmd/stop_default.go @@ -1,10 +1,10 @@ -/* -Copyright © 2022 NAME HERE -*/ +//go:build !windows + package cmd import ( "os" + "syscall" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -30,11 +30,11 @@ func stop() { log.Errorf("failed to find process by pid: %d, reason: %v", pid, process) return } - err = process.Kill() + err = process.Signal(syscall.SIGTERM) if err != nil { - log.Errorf("failed to kill process %d: %v", pid, err) + log.Errorf("failed to terminate process %d: %v", pid, err) } else { - log.Info("killed process: ", pid) + log.Info("terminated process: ", pid) } err = os.Remove(pidFile) if err != nil { diff --git a/cmd/stop_windows.go b/cmd/stop_windows.go new file mode 100644 index 00000000000..e086eab1ea8 --- /dev/null +++ b/cmd/stop_windows.go @@ -0,0 +1,34 @@ +//go:build windows + +package cmd + +import ( + "github.com/spf13/cobra" +) + +// StopCmd represents the stop command +var StopCmd = &cobra.Command{ + Use: "stop", + Short: "Same as the kill command", + Run: func(cmd *cobra.Command, args []string) { + stop() + }, +} + +func stop() { + kill() +} + +func init() { + RootCmd.AddCommand(StopCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // stopCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // stopCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/go.mod b/go.mod index 7ca66e155ff..0693dcd32c3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/alist-org/alist/v3 -go 1.22.4 +go 1.23 + +toolchain go1.23.1 require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 @@ -40,17 +42,20 @@ require ( github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 + github.com/kdomanski/iso9660 v0.4.0 github.com/larksuite/oapi-sdk-go/v3 v3.3.1 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.2 + github.com/mholt/archives v0.1.0 github.com/minio/sio v0.4.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.3 - github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 + github.com/orzogc/fake115uploader v0.6.2 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 github.com/pquerna/otp v1.4.0 github.com/rclone/rclone v1.67.0 + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 @@ -61,6 +66,7 @@ require ( github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 github.com/xhofe/tache v0.1.3 github.com/xhofe/wopan-sdk-go v0.1.3 + github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e @@ -77,21 +83,32 @@ require ( ) require ( - github.com/BurntSushi/toml v0.3.1 // indirect + github.com/STARRY-S/zip v0.2.1 // indirect github.com/blevesearch/go-faiss v1.0.20 // indirect github.com/blevesearch/zapx/v16 v16.1.5 // indirect + github.com/bodgit/plumbing v1.3.0 // indirect + github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/windows v1.0.1 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/charmbracelet/x/ansi v0.2.3 // indirect github.com/charmbracelet/x/term v0.2.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fclairamb/go-log v0.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect + github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect + github.com/therootcompany/xz v1.0.1 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 // indirect ) require ( @@ -99,8 +116,8 @@ require ( github.com/RoaringBitmap/roaring v1.9.3 // indirect github.com/abbot/go-http-auth v0.4.0 // indirect github.com/aead/ecdh v0.2.0 // indirect - github.com/andreburgaud/crypt2go v1.2.0 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andreburgaud/crypt2go v1.8.0 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect @@ -161,7 +178,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect - github.com/klauspost/compress v1.17.8 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -196,7 +213,7 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/otiai10/copy v1.14.0 github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect @@ -228,7 +245,7 @@ require ( golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/text v0.21.0 golang.org/x/tools v0.24.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect diff --git a/go.sum b/go.sum index 101a0bea063..9d92a935f4d 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,27 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 h1:ikwCzeqoqN6wvBHOB9OI6dde/jbV7EoTMpUcxtYl5Po= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= github.com/KirCute/sftpd-alist v0.0.12 h1:GNVM5QLbQLAfXP4wGUlXFA2IO6fVek0n0IsGnOuISdg= @@ -12,6 +30,8 @@ github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9 github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= +github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= +github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/SheltonZhu/115driver v1.0.32 h1:Taw1bnfcPJZW0xTdhDvEbBS1tccif7J7DslRp2NkDyQ= github.com/SheltonZhu/115driver v1.0.32/go.mod h1:XXFi23pyhAgzUE8dUEKdGvIdUQKi3wv6zR7C1Do40D8= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= @@ -30,10 +50,11 @@ github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 h1:pIEI87zhv8ZzQcu github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92/go.mod h1:oPJwGY3sLmGgcJamGumz//0A35f4BwQRacyqLNcJTOU= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= -github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/Hcta4l4VrIXrI= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andreburgaud/crypt2go v1.8.0 h1:J73vGTb1P6XL69SSuumbKs0DWn3ulbl9L92ZXBjw6pc= +github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAPJAF5fKOLB9SXg= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -91,6 +112,12 @@ github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= +github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= +github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= +github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= +github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= +github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= +github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= @@ -99,6 +126,7 @@ github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3z github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= @@ -115,8 +143,12 @@ github.com/charmbracelet/x/term v0.2.0 h1:cNB9Ot9q8I711MyZ7myUR5HFWL/lc3OpU8jZ4h github.com/charmbracelet/x/term v0.2.0/go.mod h1:GVxgxAbjUrmpvIINHIQnJJKpMlHiZ4cktEQCN6GWyF0= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -145,8 +177,13 @@ github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1 github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= @@ -175,6 +212,8 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= @@ -220,14 +259,32 @@ github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17w github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -236,6 +293,11 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -243,6 +305,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -257,13 +321,18 @@ github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+l github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI0= github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= @@ -297,18 +366,26 @@ github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFF github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= +github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= +github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= -github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -355,6 +432,8 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/meilisearch/meilisearch-go v0.27.2 h1:3G21dJ5i208shnLPDsIEZ0L0Geg/5oeXABFV7nlK94k= github.com/meilisearch/meilisearch-go v0.27.2/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= +github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= +github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= @@ -400,8 +479,10 @@ github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4 github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= -github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu4h5aYIOzUtLjN08L4Qt4WGaJONMgcaD0ayBJQ= -github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= +github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= +github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/orzogc/fake115uploader v0.6.2 h1:f4LzqeeXpmY7DjOMnzmAnnPTPMA/f/BUclq4ecffTvU= +github.com/orzogc/fake115uploader v0.6.2/go.mod h1:Mqqwv1+gUEjJhUfIQanco3DCTKp+7lSx8DJ3AoRwMoE= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= @@ -410,8 +491,8 @@ github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OI github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -432,6 +513,7 @@ github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= @@ -445,13 +527,17 @@ github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Ny github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= @@ -469,6 +555,8 @@ github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= +github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -501,6 +589,8 @@ github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= @@ -517,6 +607,9 @@ github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6 github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -536,6 +629,10 @@ github.com/xhofe/tache v0.1.3 h1:MipxzlljYX29E1YI/SLC7hVomVF+51iP1OUzlsuq1wE= github.com/xhofe/tache v0.1.3/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= +github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9/go.mod h1:9BnoKCcgJ/+SLhfAXj15352hTOuVmG5Gzo8xNRINfqI= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -545,6 +642,10 @@ github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 h1:X+lH github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22/go.mod h1:1zGRDJd8zlG6P8azG96+uywfh6udYWwhOmUivw+xsuM= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= @@ -555,12 +656,16 @@ go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGX go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= +go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -576,11 +681,35 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -588,8 +717,19 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -600,6 +740,7 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= @@ -607,8 +748,17 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -619,10 +769,20 @@ golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -664,7 +824,9 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -679,6 +841,7 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -686,8 +849,30 @@ golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -700,12 +885,45 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= @@ -744,8 +962,16 @@ gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDa gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/archive/all.go b/internal/archive/all.go new file mode 100644 index 00000000000..18167933b1c --- /dev/null +++ b/internal/archive/all.go @@ -0,0 +1,7 @@ +package archive + +import ( + _ "github.com/alist-org/alist/v3/internal/archive/archives" + _ "github.com/alist-org/alist/v3/internal/archive/iso9660" + _ "github.com/alist-org/alist/v3/internal/archive/zip" +) diff --git a/internal/archive/archives/archives.go b/internal/archive/archives/archives.go new file mode 100644 index 00000000000..b70ba95bc86 --- /dev/null +++ b/internal/archive/archives/archives.go @@ -0,0 +1,126 @@ +package archives + +import ( + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "io" + "io/fs" + "os" + stdpath "path" + "strings" +) + +type Archives struct { +} + +func (_ *Archives) AcceptedExtensions() []string { + return []string{ + ".br", ".bz2", ".gz", ".lz4", ".lz", ".sz", ".s2", ".xz", ".zz", ".zst", ".tar", ".rar", ".7z", + } +} + +func (_ *Archives) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + fsys, err := getFs(ss, args) + if err != nil { + return nil, err + } + _, err = fsys.ReadDir(".") + if err != nil { + return nil, filterPassword(err) + } + return &model.ArchiveMetaInfo{ + Comment: "", + Encrypted: false, + }, nil +} + +func (_ *Archives) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + fsys, err := getFs(ss, args.ArchiveArgs) + if err != nil { + return nil, err + } + innerPath := strings.TrimPrefix(args.InnerPath, "/") + if innerPath == "" { + innerPath = "." + } + obj, err := fsys.ReadDir(innerPath) + if err != nil { + return nil, filterPassword(err) + } + return utils.SliceConvert(obj, func(src os.DirEntry) (model.Obj, error) { + info, err := src.Info() + if err != nil { + return nil, err + } + return toModelObj(info), nil + }) +} + +func (_ *Archives) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + fsys, err := getFs(ss, args.ArchiveArgs) + if err != nil { + return nil, 0, err + } + file, err := fsys.Open(strings.TrimPrefix(args.InnerPath, "/")) + if err != nil { + return nil, 0, filterPassword(err) + } + stat, err := file.Stat() + if err != nil { + return nil, 0, filterPassword(err) + } + return file, stat.Size(), nil +} + +func (_ *Archives) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + fsys, err := getFs(ss, args.ArchiveArgs) + if err != nil { + return err + } + isDir := false + path := strings.TrimPrefix(args.InnerPath, "/") + if path == "" { + isDir = true + path = "." + } else { + stat, err := fsys.Stat(path) + if err != nil { + return filterPassword(err) + } + if stat.IsDir() { + isDir = true + outputPath = stdpath.Join(outputPath, stat.Name()) + err = os.Mkdir(outputPath, 0700) + if err != nil { + return filterPassword(err) + } + } + } + if isDir { + err = fs.WalkDir(fsys, path, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + relPath := strings.TrimPrefix(p, path+"/") + dstPath := stdpath.Join(outputPath, relPath) + if d.IsDir() { + err = os.MkdirAll(dstPath, 0700) + } else { + dir := stdpath.Dir(dstPath) + err = decompress(fsys, p, dir, func(_ float64) {}) + } + return err + }) + } else { + err = decompress(fsys, path, outputPath, up) + } + return filterPassword(err) +} + +var _ tool.Tool = (*Archives)(nil) + +func init() { + tool.RegisterTool(&Archives{}) +} diff --git a/internal/archive/archives/utils.go b/internal/archive/archives/utils.go new file mode 100644 index 00000000000..b72e6bc6a00 --- /dev/null +++ b/internal/archive/archives/utils.go @@ -0,0 +1,80 @@ +package archives + +import ( + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/mholt/archives" + "io" + fs2 "io/fs" + "os" + stdpath "path" + "strings" +) + +func getFs(ss *stream.SeekableStream, args model.ArchiveArgs) (*archives.ArchiveFS, error) { + reader, err := stream.NewReadAtSeeker(ss, 0) + if err != nil { + return nil, err + } + format, _, err := archives.Identify(ss.Ctx, ss.GetName(), reader) + if err != nil { + return nil, errs.UnknownArchiveFormat + } + extractor, ok := format.(archives.Extractor) + if !ok { + return nil, errs.UnknownArchiveFormat + } + switch f := format.(type) { + case archives.SevenZip: + f.Password = args.Password + case archives.Rar: + f.Password = args.Password + } + return &archives.ArchiveFS{ + Stream: io.NewSectionReader(reader, 0, ss.GetSize()), + Format: extractor, + Context: ss.Ctx, + }, nil +} + +func toModelObj(file os.FileInfo) *model.Object { + return &model.Object{ + Name: file.Name(), + Size: file.Size(), + Modified: file.ModTime(), + IsFolder: file.IsDir(), + } +} + +func filterPassword(err error) error { + if err != nil && strings.Contains(err.Error(), "password") { + return errs.WrongArchivePassword + } + return err +} + +func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgress) error { + rc, err := fsys.Open(filePath) + if err != nil { + return err + } + defer rc.Close() + stat, err := rc.Stat() + if err != nil { + return err + } + f, err := os.OpenFile(stdpath.Join(targetPath, stat.Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: rc, + Size: stat.Size(), + }, + UpdateProgress: up, + }) + return err +} diff --git a/internal/archive/iso9660/iso9660.go b/internal/archive/iso9660/iso9660.go new file mode 100644 index 00000000000..e9cb3f538ec --- /dev/null +++ b/internal/archive/iso9660/iso9660.go @@ -0,0 +1,96 @@ +package iso9660 + +import ( + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/kdomanski/iso9660" + "io" + "os" + stdpath "path" +) + +type ISO9660 struct { +} + +func (t *ISO9660) AcceptedExtensions() []string { + return []string{".iso"} +} + +func (t *ISO9660) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return &model.ArchiveMetaInfo{ + Comment: "", + Encrypted: false, + }, nil +} + +func (t *ISO9660) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + img, err := getImage(ss) + if err != nil { + return nil, err + } + dir, err := getObj(img, args.InnerPath) + if err != nil { + return nil, err + } + if !dir.IsDir() { + return nil, errs.NotFolder + } + children, err := dir.GetChildren() + if err != nil { + return nil, err + } + ret := make([]model.Obj, 0, len(children)) + for _, child := range children { + ret = append(ret, toModelObj(child)) + } + return ret, nil +} + +func (t *ISO9660) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + img, err := getImage(ss) + if err != nil { + return nil, 0, err + } + obj, err := getObj(img, args.InnerPath) + if err != nil { + return nil, 0, err + } + if obj.IsDir() { + return nil, 0, errs.NotFile + } + return io.NopCloser(obj.Reader()), obj.Size(), nil +} + +func (t *ISO9660) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + img, err := getImage(ss) + if err != nil { + return err + } + obj, err := getObj(img, args.InnerPath) + if err != nil { + return err + } + if obj.IsDir() { + if args.InnerPath != "/" { + outputPath = stdpath.Join(outputPath, obj.Name()) + if err = os.MkdirAll(outputPath, 0700); err != nil { + return err + } + } + var children []*iso9660.File + if children, err = obj.GetChildren(); err == nil { + err = decompressAll(children, outputPath) + } + } else { + err = decompress(obj, outputPath, up) + } + return err +} + +var _ tool.Tool = (*ISO9660)(nil) + +func init() { + tool.RegisterTool(&ISO9660{}) +} diff --git a/internal/archive/iso9660/utils.go b/internal/archive/iso9660/utils.go new file mode 100644 index 00000000000..12de8e6ea28 --- /dev/null +++ b/internal/archive/iso9660/utils.go @@ -0,0 +1,100 @@ +package iso9660 + +import ( + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/kdomanski/iso9660" + "io" + "os" + stdpath "path" + "strings" +) + +func getImage(ss *stream.SeekableStream) (*iso9660.Image, error) { + reader, err := stream.NewReadAtSeeker(ss, 0) + if err != nil { + return nil, err + } + return iso9660.OpenImage(reader) +} + +func getObj(img *iso9660.Image, path string) (*iso9660.File, error) { + obj, err := img.RootDir() + if err != nil { + return nil, err + } + if path == "/" { + return obj, nil + } + paths := strings.Split(strings.TrimPrefix(path, "/"), "/") + for _, p := range paths { + if !obj.IsDir() { + return nil, errs.ObjectNotFound + } + children, err := obj.GetChildren() + if err != nil { + return nil, err + } + exist := false + for _, child := range children { + if child.Name() == p { + obj = child + exist = true + break + } + } + if !exist { + return nil, errs.ObjectNotFound + } + } + return obj, nil +} + +func toModelObj(file *iso9660.File) model.Obj { + return &model.Object{ + Name: file.Name(), + Size: file.Size(), + Modified: file.ModTime(), + IsFolder: file.IsDir(), + } +} + +func decompress(f *iso9660.File, path string, up model.UpdateProgress) error { + file, err := os.OpenFile(stdpath.Join(path, f.Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(file, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: f.Reader(), + Size: f.Size(), + }, + UpdateProgress: up, + }) + return err +} + +func decompressAll(children []*iso9660.File, path string) error { + for _, child := range children { + if child.IsDir() { + nextChildren, err := child.GetChildren() + if err != nil { + return err + } + nextPath := stdpath.Join(path, child.Name()) + if err = os.MkdirAll(nextPath, 0700); err != nil { + return err + } + if err = decompressAll(nextChildren, nextPath); err != nil { + return err + } + } else { + if err := decompress(child, path, func(_ float64) {}); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/archive/tool/base.go b/internal/archive/tool/base.go new file mode 100644 index 00000000000..08e96614f51 --- /dev/null +++ b/internal/archive/tool/base.go @@ -0,0 +1,15 @@ +package tool + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "io" +) + +type Tool interface { + AcceptedExtensions() []string + GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) + List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) + Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) + Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error +} diff --git a/internal/archive/tool/utils.go b/internal/archive/tool/utils.go new file mode 100644 index 00000000000..822ee894fd1 --- /dev/null +++ b/internal/archive/tool/utils.go @@ -0,0 +1,23 @@ +package tool + +import ( + "github.com/alist-org/alist/v3/internal/errs" +) + +var ( + Tools = make(map[string]Tool) +) + +func RegisterTool(tool Tool) { + for _, ext := range tool.AcceptedExtensions() { + Tools[ext] = tool + } +} + +func GetArchiveTool(ext string) (Tool, error) { + t, ok := Tools[ext] + if !ok { + return nil, errs.UnknownArchiveFormat + } + return t, nil +} diff --git a/internal/archive/zip/utils.go b/internal/archive/zip/utils.go new file mode 100644 index 00000000000..81b47782840 --- /dev/null +++ b/internal/archive/zip/utils.go @@ -0,0 +1,156 @@ +package zip + +import ( + "bytes" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/saintfish/chardet" + "github.com/yeka/zip" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/encoding/unicode/utf32" + "golang.org/x/text/transform" + "io" + "os" + stdpath "path" + "strings" +) + +func toModelObj(file os.FileInfo) *model.Object { + return &model.Object{ + Name: decodeName(file.Name()), + Size: file.Size(), + Modified: file.ModTime(), + IsFolder: file.IsDir(), + } +} + +func decompress(file *zip.File, filePath, outputPath, password string) error { + targetPath := outputPath + dir, base := stdpath.Split(filePath) + if dir != "" { + targetPath = stdpath.Join(targetPath, dir) + err := os.MkdirAll(targetPath, 0700) + if err != nil { + return err + } + } + if base != "" { + err := _decompress(file, targetPath, password, func(_ float64) {}) + if err != nil { + return err + } + } + return nil +} + +func _decompress(file *zip.File, targetPath, password string, up model.UpdateProgress) error { + if file.IsEncrypted() { + file.SetPassword(password) + } + rc, err := file.Open() + if err != nil { + return err + } + defer rc.Close() + f, err := os.OpenFile(stdpath.Join(targetPath, file.FileInfo().Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: rc, + Size: file.FileInfo().Size(), + }, + UpdateProgress: up, + }) + if err != nil { + return err + } + return nil +} + +func filterPassword(err error) error { + if err != nil && strings.Contains(err.Error(), "password") { + return errs.WrongArchivePassword + } + return err +} + +func decodeName(name string) string { + b := []byte(name) + detector := chardet.NewTextDetector() + result, err := detector.DetectBest(b) + if err != nil { + return name + } + enc := getEncoding(result.Charset) + if enc == nil { + return name + } + i := bytes.NewReader(b) + decoder := transform.NewReader(i, enc.NewDecoder()) + content, _ := io.ReadAll(decoder) + return string(content) +} + +func getEncoding(name string) (enc encoding.Encoding) { + switch name { + case "UTF-16BE": + enc = unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM) + case "UTF-16LE": + enc = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) + case "UTF-32BE": + enc = utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM) + case "UTF-32LE": + enc = utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM) + case "ISO-8859-1": + enc = charmap.ISO8859_1 + case "ISO-8859-2": + enc = charmap.ISO8859_2 + case "ISO-8859-3": + enc = charmap.ISO8859_3 + case "ISO-8859-4": + enc = charmap.ISO8859_4 + case "ISO-8859-5": + enc = charmap.ISO8859_5 + case "ISO-8859-6": + enc = charmap.ISO8859_6 + case "ISO-8859-7": + enc = charmap.ISO8859_7 + case "ISO-8859-8": + enc = charmap.ISO8859_8 + case "ISO-8859-8-I": + enc = charmap.ISO8859_8I + case "ISO-8859-9": + enc = charmap.ISO8859_9 + case "windows-1251": + enc = charmap.Windows1251 + case "windows-1256": + enc = charmap.Windows1256 + case "KOI8-R": + enc = charmap.KOI8R + case "Shift_JIS": + enc = japanese.ShiftJIS + case "GB-18030": + enc = simplifiedchinese.GB18030 + case "EUC-JP": + enc = japanese.EUCJP + case "EUC-KR": + enc = korean.EUCKR + case "Big5": + enc = traditionalchinese.Big5 + case "ISO-2022-JP": + enc = japanese.ISO2022JP + default: + enc = nil + } + return +} diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go new file mode 100644 index 00000000000..ccb70e65996 --- /dev/null +++ b/internal/archive/zip/zip.go @@ -0,0 +1,174 @@ +package zip + +import ( + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/yeka/zip" + "io" + "os" + stdpath "path" + "strings" +) + +type Zip struct { +} + +func (_ *Zip) AcceptedExtensions() []string { + return []string{".zip"} +} + +func (_ *Zip) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + reader, err := stream.NewReadAtSeeker(ss, 0) + if err != nil { + return nil, err + } + zipReader, err := zip.NewReader(reader, ss.GetSize()) + if err != nil { + return nil, err + } + encrypted := false + for _, file := range zipReader.File { + if file.IsEncrypted() { + encrypted = true + break + } + } + return &model.ArchiveMetaInfo{ + Comment: zipReader.Comment, + Encrypted: encrypted, + }, nil +} + +func (_ *Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + reader, err := stream.NewReadAtSeeker(ss, 0) + if err != nil { + return nil, err + } + zipReader, err := zip.NewReader(reader, ss.GetSize()) + if err != nil { + return nil, err + } + if args.InnerPath == "/" { + ret := make([]model.Obj, 0) + passVerified := false + for _, file := range zipReader.File { + if !passVerified && file.IsEncrypted() { + file.SetPassword(args.Password) + rc, e := file.Open() + if e != nil { + return nil, filterPassword(e) + } + _ = rc.Close() + passVerified = true + } + name := decodeName(file.Name) + if strings.Contains(strings.TrimSuffix(name, "/"), "/") { + continue + } + ret = append(ret, toModelObj(file.FileInfo())) + } + return ret, nil + } else { + innerPath := strings.TrimPrefix(args.InnerPath, "/") + "/" + ret := make([]model.Obj, 0) + exist := false + for _, file := range zipReader.File { + name := decodeName(file.Name) + if name == innerPath { + exist = true + } + dir := stdpath.Dir(strings.TrimSuffix(name, "/")) + "/" + if dir != innerPath { + continue + } + ret = append(ret, toModelObj(file.FileInfo())) + } + if !exist { + return nil, errs.ObjectNotFound + } + return ret, nil + } +} + +func (_ *Zip) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + reader, err := stream.NewReadAtSeeker(ss, 0) + if err != nil { + return nil, 0, err + } + zipReader, err := zip.NewReader(reader, ss.GetSize()) + if err != nil { + return nil, 0, err + } + innerPath := strings.TrimPrefix(args.InnerPath, "/") + for _, file := range zipReader.File { + if decodeName(file.Name) == innerPath { + if file.IsEncrypted() { + file.SetPassword(args.Password) + } + r, e := file.Open() + if e != nil { + return nil, 0, e + } + return r, file.FileInfo().Size(), nil + } + } + return nil, 0, errs.ObjectNotFound +} + +func (_ *Zip) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + reader, err := stream.NewReadAtSeeker(ss, 0) + if err != nil { + return err + } + zipReader, err := zip.NewReader(reader, ss.GetSize()) + if err != nil { + return err + } + if args.InnerPath == "/" { + for i, file := range zipReader.File { + name := decodeName(file.Name) + err = decompress(file, name, outputPath, args.Password) + if err != nil { + return err + } + up(float64(i+1) * 100.0 / float64(len(zipReader.File))) + } + } else { + innerPath := strings.TrimPrefix(args.InnerPath, "/") + innerBase := stdpath.Base(innerPath) + createdBaseDir := false + for _, file := range zipReader.File { + name := decodeName(file.Name) + if name == innerPath { + err = _decompress(file, outputPath, args.Password, up) + if err != nil { + return err + } + break + } else if strings.HasPrefix(name, innerPath+"/") { + targetPath := stdpath.Join(outputPath, innerBase) + if !createdBaseDir { + err = os.Mkdir(targetPath, 0700) + if err != nil { + return err + } + createdBaseDir = true + } + restPath := strings.TrimPrefix(name, innerPath+"/") + err = decompress(file, restPath, targetPath, args.Password) + if err != nil { + return err + } + } + } + } + return nil +} + +var _ tool.Tool = (*Zip)(nil) + +func init() { + tool.RegisterTool(&Zip{}) +} diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 5b596a85852..9c3f8962ad3 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -26,13 +26,14 @@ func initUser() { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) admin = &model.User{ - Username: "admin", - Salt: salt, - PwdHash: model.TwoHashPwd(adminPassword, salt), - Role: model.ADMIN, - BasePath: "/", - Authn: "[]", - Permission: 0xFF, // 0(can see hidden) - 7(can remove) + Username: "admin", + Salt: salt, + PwdHash: model.TwoHashPwd(adminPassword, salt), + Role: model.ADMIN, + BasePath: "/", + Authn: "[]", + // 0(can see hidden) - 7(can remove) & 12(can read archives) - 13(can decompress archives) + Permission: 0x30FF, } if err := op.CreateUser(admin); err != nil { panic(err) diff --git a/internal/bootstrap/patch/v3_41_0/grant_permission.go b/internal/bootstrap/patch/v3_41_0/grant_permission.go index d658d184d4d..e62d1e8fa90 100644 --- a/internal/bootstrap/patch/v3_41_0/grant_permission.go +++ b/internal/bootstrap/patch/v3_41_0/grant_permission.go @@ -5,18 +5,20 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" ) -// GrantAdminPermissions gives admin Permission 0(can see hidden) - 9(webdav manage) -// This patch is written to help users upgrading from older version better adapt to PR AlistGo/alist#7705. +// GrantAdminPermissions gives admin Permission 0(can see hidden) - 9(webdav manage) and +// 12(can read archives) - 13(can decompress archives) +// This patch is written to help users upgrading from older version better adapt to PR AlistGo/alist#7705 and +// PR AlistGo/alist#7817. func GrantAdminPermissions() { admin, err := op.GetAdmin() if err != nil { utils.Log.Errorf("Cannot grant permissions to admin: %v", err) } - if (admin.Permission & 0x3FF) == 0 { - admin.Permission |= 0x3FF - } - err = op.UpdateUser(admin) - if err != nil { - utils.Log.Errorf("Cannot grant permissions to admin: %v", err) + if (admin.Permission & 0x33FF) == 0 { + admin.Permission |= 0x33FF + err = op.UpdateUser(admin) + if err != nil { + utils.Log.Errorf("Cannot grant permissions to admin: %v", err) + } } } diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go index 3390235320c..9c30c3926b5 100644 --- a/internal/bootstrap/task.go +++ b/internal/bootstrap/task.go @@ -16,4 +16,6 @@ func InitTaskManager() { if len(tool.TransferTaskManager.GetAll()) == 0 { //prevent offline downloaded files from being deleted CleanTempDir() } + fs.ArchiveDownloadTaskManager = tache.NewManager[*fs.ArchiveDownloadTask](tache.WithWorks(conf.Conf.Tasks.Decompress.Workers), tache.WithPersistFunction(db.GetTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant), db.UpdateTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Decompress.MaxRetry)) + fs.ArchiveContentUploadTaskManager.Manager = tache.NewManager[*fs.ArchiveContentUploadTask](tache.WithWorks(conf.Conf.Tasks.DecompressUpload.Workers), tache.WithMaxRetry(conf.Conf.Tasks.DecompressUpload.MaxRetry)) //decompress upload will not support persist } diff --git a/internal/conf/config.go b/internal/conf/config.go index d015cda0a91..4f5c2ae0e35 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -53,10 +53,12 @@ type TaskConfig struct { } type TasksConfig struct { - Download TaskConfig `json:"download" envPrefix:"DOWNLOAD_"` - Transfer TaskConfig `json:"transfer" envPrefix:"TRANSFER_"` - Upload TaskConfig `json:"upload" envPrefix:"UPLOAD_"` - Copy TaskConfig `json:"copy" envPrefix:"COPY_"` + Download TaskConfig `json:"download" envPrefix:"DOWNLOAD_"` + Transfer TaskConfig `json:"transfer" envPrefix:"TRANSFER_"` + Upload TaskConfig `json:"upload" envPrefix:"UPLOAD_"` + Copy TaskConfig `json:"copy" envPrefix:"COPY_"` + Decompress TaskConfig `json:"decompress" envPrefix:"DECOMPRESS_"` + DecompressUpload TaskConfig `json:"decompress_upload" envPrefix:"DECOMPRESS_UPLOAD_"` } type Cors struct { @@ -169,6 +171,15 @@ func DefaultConfig() *Config { MaxRetry: 2, // TaskPersistant: true, }, + Decompress: TaskConfig{ + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, + }, + DecompressUpload: TaskConfig{ + Workers: 5, + MaxRetry: 2, + }, }, Cors: Cors{ AllowOrigins: []string{"*"}, diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 4571110a8f2..09fd42e7658 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -123,7 +123,43 @@ type PutURLResult interface { PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) } -type UpdateProgress func(percentage float64) +type ArchiveReader interface { + // GetArchiveMeta get the meta-info of an archive + // return errs.WrongArchivePassword if the meta-info is also encrypted but provided password is wrong or empty + // return errs.NotImplement to use internal archive tools to get the meta-info, such as the following cases: + // 1. the driver do not support the format of the archive but there may be an internal tool do + // 2. handling archives is a VIP feature, but the driver does not have VIP access + GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) + // ListArchive list the children of model.ArchiveArgs.InnerPath in the archive + // return errs.NotImplement to use internal archive tools to list the children + // return errs.NotSupport if the folder structure should be acquired from model.ArchiveMeta.GetTree + ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) + // Extract get url/filepath/reader of a file in the archive + // return errs.NotImplement to use internal archive tools to extract + Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) +} + +type ArchiveGetter interface { + // ArchiveGet get file by inner path + // return errs.NotImplement to use internal archive tools to get the children + // return errs.NotSupport if the folder structure should be acquired from model.ArchiveMeta.GetTree + ArchiveGet(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (model.Obj, error) +} + +type ArchiveDecompress interface { + ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error +} + +type ArchiveDecompressResult interface { + // ArchiveDecompress decompress an archive + // when args.PutIntoNewDir, the new sub-folder should be named the same to the archive but without the extension + // return each decompressed obj from the root path of the archive when args.PutIntoNewDir is false + // return only the newly created folder when args.PutIntoNewDir is true + // return errs.NotImplement to use internal archive tools to decompress + ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) +} + +type UpdateProgress model.UpdateProgress type Progress struct { Total int64 diff --git a/internal/errs/errors.go b/internal/errs/errors.go index ecfe43e3dc0..2a22dca1e6f 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -19,6 +19,10 @@ var ( StorageNotFound = errors.New("storage not found") StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") StreamPeekFail = errors.New("StreamPeekFail") + + UnknownArchiveFormat = errors.New("unknown archive format") + WrongArchivePassword = errors.New("wrong archive password") + DriverExtractNotSupported = errors.New("driver extraction not supported") ) // NewErr wrap constant error with an extra message diff --git a/internal/fs/archive.go b/internal/fs/archive.go new file mode 100644 index 00000000000..f3e05926e88 --- /dev/null +++ b/internal/fs/archive.go @@ -0,0 +1,395 @@ +package fs + +import ( + "context" + stderrors "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" + "io" + "math/rand" + "mime" + "net/http" + "os" + stdpath "path" + "path/filepath" + "strconv" + "strings" + "time" +) + +type ArchiveDownloadTask struct { + task.TaskExtension + model.ArchiveDecompressArgs + status string + SrcObjPath string + DstDirPath string + srcStorage driver.Driver + dstStorage driver.Driver + SrcStorageMp string + DstStorageMp string + Tool tool.Tool +} + +func (t *ArchiveDownloadTask) GetName() string { + return fmt.Sprintf("decompress [%s](%s)[%s] to [%s](%s) with password <%s>", t.SrcStorageMp, t.SrcObjPath, + t.InnerPath, t.DstStorageMp, t.DstDirPath, t.Password) +} + +func (t *ArchiveDownloadTask) GetStatus() string { + return t.status +} + +func (t *ArchiveDownloadTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + uploadTask, err := t.RunWithoutPushUploadTask() + if err != nil { + return err + } + ArchiveContentUploadTaskManager.Add(uploadTask) + return nil +} + +func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadTask, error) { + var err error + if t.srcStorage == nil { + t.srcStorage, err = op.GetStorageByMountPath(t.SrcStorageMp) + } + l, srcObj, err := op.Link(t.Ctx(), t.srcStorage, t.SrcObjPath, model.LinkArgs{ + Header: http.Header{}, + }) + if err != nil { + return nil, err + } + fs := stream.FileStream{ + Obj: srcObj, + Ctx: t.Ctx(), + } + ss, err := stream.NewSeekableStream(fs, l) + if err != nil { + return nil, err + } + defer func() { + if err := ss.Close(); err != nil { + log.Errorf("failed to close file streamer, %v", err) + } + }() + var decompressUp model.UpdateProgress + if t.CacheFull { + t.SetTotalBytes(srcObj.GetSize()) + t.status = "getting src object" + _, err = ss.CacheFullInTempFileAndUpdateProgress(t.SetProgress) + if err != nil { + return nil, err + } + decompressUp = func(_ float64) {} + } else { + decompressUp = t.SetProgress + } + t.status = "walking and decompressing" + dir, err := os.MkdirTemp(conf.Conf.TempDir, "dir-*") + if err != nil { + return nil, err + } + err = t.Tool.Decompress(ss, dir, t.ArchiveInnerArgs, decompressUp) + if err != nil { + return nil, err + } + baseName := strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName())) + uploadTask := &ArchiveContentUploadTask{ + TaskExtension: task.TaskExtension{ + Creator: t.GetCreator(), + }, + ObjName: baseName, + InPlace: !t.PutIntoNewDir, + FilePath: dir, + DstDirPath: t.DstDirPath, + dstStorage: t.dstStorage, + DstStorageMp: t.DstStorageMp, + } + return uploadTask, nil +} + +var ArchiveDownloadTaskManager *tache.Manager[*ArchiveDownloadTask] + +type ArchiveContentUploadTask struct { + task.TaskExtension + status string + ObjName string + InPlace bool + FilePath string + DstDirPath string + dstStorage driver.Driver + DstStorageMp string + finalized bool +} + +func (t *ArchiveContentUploadTask) GetName() string { + return fmt.Sprintf("upload %s to [%s](%s)", t.ObjName, t.DstStorageMp, t.DstDirPath) +} + +func (t *ArchiveContentUploadTask) GetStatus() string { + return t.status +} + +func (t *ArchiveContentUploadTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + return t.RunWithNextTaskCallback(func(nextTsk *ArchiveContentUploadTask) error { + ArchiveContentUploadTaskManager.Add(nextTsk) + return nil + }) +} + +func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTsk *ArchiveContentUploadTask) error) error { + var err error + if t.dstStorage == nil { + t.dstStorage, err = op.GetStorageByMountPath(t.DstStorageMp) + } + info, err := os.Stat(t.FilePath) + if err != nil { + return err + } + if info.IsDir() { + t.status = "src object is dir, listing objs" + nextDstPath := t.DstDirPath + if !t.InPlace { + nextDstPath = stdpath.Join(nextDstPath, t.ObjName) + err = op.MakeDir(t.Ctx(), t.dstStorage, nextDstPath) + if err != nil { + return err + } + } + entries, err := os.ReadDir(t.FilePath) + if err != nil { + return err + } + var es error + for _, entry := range entries { + var nextFilePath string + if entry.IsDir() { + nextFilePath, err = moveToTempPath(stdpath.Join(t.FilePath, entry.Name()), "dir-") + } else { + nextFilePath, err = moveToTempPath(stdpath.Join(t.FilePath, entry.Name()), "file-") + } + if err != nil { + es = stderrors.Join(es, err) + continue + } + err = f(&ArchiveContentUploadTask{ + TaskExtension: task.TaskExtension{ + Creator: t.GetCreator(), + }, + ObjName: entry.Name(), + InPlace: false, + FilePath: nextFilePath, + DstDirPath: nextDstPath, + dstStorage: t.dstStorage, + DstStorageMp: t.DstStorageMp, + }) + if err != nil { + es = stderrors.Join(es, err) + } + } + if es != nil { + return es + } + } else { + t.SetTotalBytes(info.Size()) + file, err := os.Open(t.FilePath) + if err != nil { + return err + } + fs := &stream.FileStream{ + Obj: &model.Object{ + Name: t.ObjName, + Size: info.Size(), + Modified: time.Now(), + }, + Mimetype: mime.TypeByExtension(filepath.Ext(t.ObjName)), + WebPutAsTask: true, + Reader: file, + } + fs.Closers.Add(file) + t.status = "uploading" + err = op.Put(t.Ctx(), t.dstStorage, t.DstDirPath, fs, t.SetProgress, true) + if err != nil { + return err + } + } + t.deleteSrcFile() + return nil +} + +func (t *ArchiveContentUploadTask) Cancel() { + t.TaskExtension.Cancel() + t.deleteSrcFile() +} + +func (t *ArchiveContentUploadTask) deleteSrcFile() { + if !t.finalized { + _ = os.RemoveAll(t.FilePath) + t.finalized = true + } +} + +func moveToTempPath(path, prefix string) (string, error) { + newPath, err := genTempFileName(prefix) + if err != nil { + return "", err + } + err = os.Rename(path, newPath) + if err != nil { + return "", err + } + return newPath, nil +} + +func genTempFileName(prefix string) (string, error) { + retry := 0 + for retry < 10000 { + newPath := stdpath.Join(conf.Conf.TempDir, prefix+strconv.FormatUint(uint64(rand.Uint32()), 10)) + if _, err := os.Stat(newPath); err != nil { + if os.IsNotExist(err) { + return newPath, nil + } else { + return "", err + } + } + retry++ + } + return "", errors.New("failed to generate temp-file name: too many retries") +} + +type archiveContentUploadTaskManagerType struct { + *tache.Manager[*ArchiveContentUploadTask] +} + +func (m *archiveContentUploadTaskManagerType) Remove(id string) { + if t, ok := m.GetByID(id); ok { + t.deleteSrcFile() + m.Manager.Remove(id) + } +} + +func (m *archiveContentUploadTaskManagerType) RemoveAll() { + tasks := m.GetAll() + for _, t := range tasks { + m.Remove(t.GetID()) + } +} + +func (m *archiveContentUploadTaskManagerType) RemoveByState(state ...tache.State) { + tasks := m.GetByState(state...) + for _, t := range tasks { + m.Remove(t.GetID()) + } +} + +func (m *archiveContentUploadTaskManagerType) RemoveByCondition(condition func(task *ArchiveContentUploadTask) bool) { + tasks := m.GetByCondition(condition) + for _, t := range tasks { + m.Remove(t.GetID()) + } +} + +var ArchiveContentUploadTaskManager = &archiveContentUploadTaskManagerType{ + Manager: nil, +} + +func archiveMeta(ctx context.Context, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { + storage, actualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return nil, errors.WithMessage(err, "failed get storage") + } + return op.GetArchiveMeta(ctx, storage, actualPath, args) +} + +func archiveList(ctx context.Context, path string, args model.ArchiveListArgs) ([]model.Obj, error) { + storage, actualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return nil, errors.WithMessage(err, "failed get storage") + } + return op.ListArchive(ctx, storage, actualPath, args) +} + +func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) (task.TaskExtensionInfo, error) { + srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) + if err != nil { + return nil, errors.WithMessage(err, "failed get src storage") + } + dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return nil, errors.WithMessage(err, "failed get dst storage") + } + if srcStorage.GetStorage() == dstStorage.GetStorage() { + err = op.ArchiveDecompress(ctx, srcStorage, srcObjActualPath, dstDirActualPath, args, lazyCache...) + if !errors.Is(err, errs.NotImplement) { + return nil, err + } + } + ext := stdpath.Ext(srcObjActualPath) + t, err := tool.GetArchiveTool(ext) + if err != nil { + return nil, errors.WithMessagef(err, "failed get [%s] archive tool", ext) + } + taskCreator, _ := ctx.Value("user").(*model.User) + tsk := &ArchiveDownloadTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + ArchiveDecompressArgs: args, + srcStorage: srcStorage, + dstStorage: dstStorage, + SrcObjPath: srcObjActualPath, + DstDirPath: dstDirActualPath, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, + Tool: t, + } + if ctx.Value(conf.NoTaskKey) != nil { + uploadTask, err := tsk.RunWithoutPushUploadTask() + if err != nil { + return nil, errors.WithMessagef(err, "failed download [%s]", srcObjPath) + } + defer uploadTask.deleteSrcFile() + var callback func(t *ArchiveContentUploadTask) error + callback = func(t *ArchiveContentUploadTask) error { + e := t.RunWithNextTaskCallback(callback) + t.deleteSrcFile() + return e + } + return nil, uploadTask.RunWithNextTaskCallback(callback) + } else { + ArchiveDownloadTaskManager.Add(tsk) + return tsk, nil + } +} + +func archiveDriverExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { + storage, actualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed get storage") + } + return op.DriverExtract(ctx, storage, actualPath, args) +} + +func archiveInternalExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + storage, actualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return nil, 0, errors.WithMessage(err, "failed get storage") + } + return op.InternalExtract(ctx, storage, actualPath, args) +} diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 24f1d47fa5e..a873f917301 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -7,6 +7,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/task" log "github.com/sirupsen/logrus" + "io" ) // the param named path of functions in this package is a mount path @@ -109,6 +110,46 @@ func PutAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) return t, err } +func ArchiveMeta(ctx context.Context, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { + meta, err := archiveMeta(ctx, path, args) + if err != nil { + log.Errorf("failed get archive meta %s: %+v", path, err) + } + return meta, err +} + +func ArchiveList(ctx context.Context, path string, args model.ArchiveListArgs) ([]model.Obj, error) { + objs, err := archiveList(ctx, path, args) + if err != nil { + log.Errorf("failed list archive [%s]%s: %+v", path, args.InnerPath, err) + } + return objs, err +} + +func ArchiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) (task.TaskExtensionInfo, error) { + t, err := archiveDecompress(ctx, srcObjPath, dstDirPath, args, lazyCache...) + if err != nil { + log.Errorf("failed decompress [%s]%s: %+v", srcObjPath, args.InnerPath, err) + } + return t, err +} + +func ArchiveDriverExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { + l, obj, err := archiveDriverExtract(ctx, path, args) + if err != nil { + log.Errorf("failed extract [%s]%s: %+v", path, args.InnerPath, err) + } + return l, obj, err +} + +func ArchiveInternalExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + l, obj, err := archiveInternalExtract(ctx, path, args) + if err != nil { + log.Errorf("failed extract [%s]%s: %+v", path, args.InnerPath, err) + } + return l, obj, err +} + type GetStoragesArgs struct { } diff --git a/internal/model/archive.go b/internal/model/archive.go new file mode 100644 index 00000000000..03ac7c360a5 --- /dev/null +++ b/internal/model/archive.go @@ -0,0 +1,49 @@ +package model + +type ObjTree interface { + Obj + GetChildren() []ObjTree +} + +type ObjectTree struct { + Object + Children []ObjTree +} + +func (t *ObjectTree) GetChildren() []ObjTree { + return t.Children +} + +type ArchiveMeta interface { + GetComment() string + // IsEncrypted means if the content of the archive requires a password to access + // GetArchiveMeta should return errs.WrongArchivePassword if the meta-info is also encrypted, + // and the provided password is empty. + IsEncrypted() bool + // GetTree directly returns the full folder structure + // returns nil if the folder structure should be acquired by calling driver.ArchiveReader.ListArchive + GetTree() []ObjTree +} + +type ArchiveMetaInfo struct { + Comment string + Encrypted bool + Tree []ObjTree +} + +func (m *ArchiveMetaInfo) GetComment() string { + return m.Comment +} + +func (m *ArchiveMetaInfo) IsEncrypted() bool { + return m.Encrypted +} + +func (m *ArchiveMetaInfo) GetTree() []ObjTree { + return m.Tree +} + +type ArchiveMetaProvider struct { + ArchiveMeta + DriverProviding bool +} diff --git a/internal/model/args.go b/internal/model/args.go index 613699b95b4..a9feeb206ef 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -48,6 +48,33 @@ type FsOtherArgs struct { Method string `json:"method" form:"method"` Data interface{} `json:"data" form:"data"` } + +type ArchiveArgs struct { + Password string + LinkArgs +} + +type ArchiveInnerArgs struct { + ArchiveArgs + InnerPath string +} + +type ArchiveMetaArgs struct { + ArchiveArgs + Refresh bool +} + +type ArchiveListArgs struct { + ArchiveInnerArgs + Refresh bool +} + +type ArchiveDecompressArgs struct { + ArchiveInnerArgs + CacheFull bool + PutIntoNewDir bool +} + type RangeReadCloserIF interface { RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) utils.ClosersIF diff --git a/internal/model/obj.go b/internal/model/obj.go index 122fb546278..2a72ca9eff4 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -48,8 +48,11 @@ type FileStreamer interface { RangeRead(http_range.Range) (io.Reader, error) //for a non-seekable Stream, if Read is called, this function won't work CacheFullInTempFile() (File, error) + CacheFullInTempFileAndUpdateProgress(up UpdateProgress) (File, error) } +type UpdateProgress func(percentage float64) + type URL interface { URL() string } diff --git a/internal/model/user.go b/internal/model/user.go index f75fc6875c5..eaa0fed9d09 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -44,6 +44,8 @@ type User struct { // 9: webdav write // 10: ftp/sftp login and read // 11: ftp/sftp write + // 12: can read archives + // 13: can decompress archives Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -127,6 +129,14 @@ func (u *User) CanFTPManage() bool { return (u.Permission>>11)&1 == 1 } +func (u *User) CanReadArchives() bool { + return (u.Permission>>12)&1 == 1 +} + +func (u *User) CanDecompress() bool { + return (u.Permission>>13)&1 == 1 +} + func (u *User) JoinPath(reqPath string) (string, error) { return utils.JoinBasePath(u.BasePath, reqPath) } diff --git a/internal/op/archive.go b/internal/op/archive.go new file mode 100644 index 00000000000..6a9fa084778 --- /dev/null +++ b/internal/op/archive.go @@ -0,0 +1,424 @@ +package op + +import ( + "context" + stderrors "errors" + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/stream" + "io" + stdpath "path" + "strings" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/singleflight" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +var archiveMetaCache = cache.NewMemCache(cache.WithShards[*model.ArchiveMetaProvider](64)) +var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider] + +func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + key := Key(storage, path) + if !args.Refresh { + if meta, ok := archiveMetaCache.Get(key); ok { + log.Debugf("use cache when get %s archive meta", path) + return meta, nil + } + } + fn := func() (*model.ArchiveMetaProvider, error) { + _, m, err := getArchiveMeta(ctx, storage, path, args) + if err != nil { + return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err) + } + if !storage.Config().NoCache { + archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } + return m, nil + } + if storage.Config().OnlyLocal { + meta, err := fn() + return meta, err + } + meta, err, _ := archiveMetaG.Do(key, fn) + return meta, err +} + +func getArchiveToolAndStream(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (model.Obj, tool.Tool, *stream.SeekableStream, error) { + l, obj, err := Link(ctx, storage, path, args) + if err != nil { + return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] link", path) + } + ext := stdpath.Ext(obj.GetName()) + t, err := tool.GetArchiveTool(ext) + if err != nil { + return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] archive tool", ext) + } + ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, l) + if err != nil { + return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) + } + return obj, t, ss, nil +} + +func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (model.Obj, *model.ArchiveMetaProvider, error) { + storageAr, ok := storage.(driver.ArchiveReader) + if ok { + obj, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file") + } + if obj.IsDir() { + return nil, nil, errors.WithStack(errs.NotFile) + } + meta, err := storageAr.GetArchiveMeta(ctx, obj, args.ArchiveArgs) + if !errors.Is(err, errs.NotImplement) { + return obj, &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: true}, err + } + } + obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + if err != nil { + return nil, nil, err + } + defer func() { + if err := ss.Close(); err != nil { + log.Errorf("failed to close file streamer, %v", err) + } + }() + meta, err := t.GetMeta(ss, args.ArchiveArgs) + return obj, &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: false}, err +} + +var archiveListCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) +var archiveListG singleflight.Group[[]model.Obj] + +func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + metaKey := Key(storage, path) + key := stdpath.Join(metaKey, args.InnerPath) + if !args.Refresh { + if files, ok := archiveListCache.Get(key); ok { + log.Debugf("use cache when list archive [%s]%s", path, args.InnerPath) + return files, nil + } + if meta, ok := archiveMetaCache.Get(metaKey); ok { + log.Debugf("use meta cache when list archive [%s]%s", path, args.InnerPath) + return getChildrenFromArchiveMeta(meta, args.InnerPath) + } + } + objs, err, _ := archiveListG.Do(key, func() ([]model.Obj, error) { + obj, files, err := listArchive(ctx, storage, path, args) + if err != nil { + return nil, errors.Wrapf(err, "failed to list archive [%s]%s: %+v", path, args.InnerPath, err) + } + // set path + for _, f := range files { + if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && obj.GetPath() != "" { + s.SetPath(stdpath.Join(obj.GetPath(), args.InnerPath, f.GetName())) + } + } + // warp obj name + model.WrapObjsName(files) + // sort objs + if storage.Config().LocalSort { + model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + } + model.ExtractFolder(files, storage.GetStorage().ExtractFolder) + if !storage.Config().NoCache { + if len(files) > 0 { + log.Debugf("set cache: %s => %+v", key, files) + archiveListCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } else { + log.Debugf("del cache: %s", key) + archiveListCache.Del(key) + } + } + return files, nil + }) + return objs, err +} + +func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { + storageAr, ok := storage.(driver.ArchiveReader) + if ok { + obj, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file") + } + if obj.IsDir() { + return nil, nil, errors.WithStack(errs.NotFile) + } + files, err := storageAr.ListArchive(ctx, obj, args.ArchiveInnerArgs) + if !errors.Is(err, errs.NotImplement) { + return obj, files, err + } + } + obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + if err != nil { + return nil, nil, err + } + defer func() { + if err := ss.Close(); err != nil { + log.Errorf("failed to close file streamer, %v", err) + } + }() + files, err := t.List(ss, args.ArchiveInnerArgs) + return obj, files, err +} + +func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { + obj, files, err := _listArchive(ctx, storage, path, args) + if errors.Is(err, errs.NotSupport) { + var meta model.ArchiveMeta + meta, err = GetArchiveMeta(ctx, storage, path, model.ArchiveMetaArgs{ + ArchiveArgs: args.ArchiveArgs, + Refresh: args.Refresh, + }) + if err != nil { + return nil, nil, err + } + files, err = getChildrenFromArchiveMeta(meta, args.InnerPath) + if err != nil { + return nil, nil, err + } + } + if err == nil && obj == nil { + obj, err = GetUnwrap(ctx, storage, path) + } + if err != nil { + return nil, nil, err + } + return obj, files, err +} + +func getChildrenFromArchiveMeta(meta model.ArchiveMeta, innerPath string) ([]model.Obj, error) { + obj := meta.GetTree() + if obj == nil { + return nil, errors.WithStack(errs.NotImplement) + } + dirs := splitPath(innerPath) + for _, dir := range dirs { + var next model.ObjTree + for _, c := range obj { + if c.GetName() == dir { + next = c + break + } + } + if next == nil { + return nil, errors.WithStack(errs.ObjectNotFound) + } + if !next.IsDir() || next.GetChildren() == nil { + return nil, errors.WithStack(errs.NotFolder) + } + obj = next.GetChildren() + } + return utils.SliceConvert(obj, func(src model.ObjTree) (model.Obj, error) { + return src, nil + }) +} + +func splitPath(path string) []string { + var parts []string + for { + dir, file := stdpath.Split(path) + if file == "" { + break + } + parts = append([]string{file}, parts...) + path = strings.TrimSuffix(dir, "/") + } + return parts +} + +func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + af, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file") + } + if af.IsDir() { + return nil, nil, errors.WithStack(errs.NotFile) + } + if g, ok := storage.(driver.ArchiveGetter); ok { + obj, err := g.ArchiveGet(ctx, af, args.ArchiveInnerArgs) + if err == nil { + return af, model.WrapObjName(obj), nil + } + } + + if utils.PathEqual(args.InnerPath, "/") { + return af, &model.ObjWrapName{ + Name: RootName, + Obj: &model.Object{ + Name: af.GetName(), + Path: af.GetPath(), + ID: af.GetID(), + Size: af.GetSize(), + Modified: af.ModTime(), + IsFolder: true, + }, + }, nil + } + + innerDir, name := stdpath.Split(args.InnerPath) + args.InnerPath = strings.TrimSuffix(innerDir, "/") + files, err := ListArchive(ctx, storage, path, args) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed get parent list") + } + for _, f := range files { + if f.GetName() == name { + return af, f, nil + } + } + return nil, nil, errors.WithStack(errs.ObjectNotFound) +} + +type extractLink struct { + Link *model.Link + Obj model.Obj +} + +var extractCache = cache.NewMemCache(cache.WithShards[*extractLink](16)) +var extractG singleflight.Group[*extractLink] + +func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + key := stdpath.Join(Key(storage, path), args.InnerPath) + if link, ok := extractCache.Get(key); ok { + return link.Link, link.Obj, nil + } else if link, ok := extractCache.Get(key + ":" + args.IP); ok { + return link.Link, link.Obj, nil + } + fn := func() (*extractLink, error) { + link, err := driverExtract(ctx, storage, path, args) + if err != nil { + return nil, errors.Wrapf(err, "failed extract archive") + } + if link.Link.Expiration != nil { + if link.Link.IPCacheKey { + key = key + ":" + args.IP + } + extractCache.Set(key, link, cache.WithEx[*extractLink](*link.Link.Expiration)) + } + return link, nil + } + if storage.Config().OnlyLocal { + link, err := fn() + if err != nil { + return nil, nil, err + } + return link.Link, link.Obj, nil + } + link, err, _ := extractG.Do(key, fn) + if err != nil { + return nil, nil, err + } + return link.Link, link.Obj, err +} + +func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*extractLink, error) { + storageAr, ok := storage.(driver.ArchiveReader) + if !ok { + return nil, errs.DriverExtractNotSupported + } + archiveFile, extracted, err := ArchiveGet(ctx, storage, path, model.ArchiveListArgs{ + ArchiveInnerArgs: args, + Refresh: false, + }) + if err != nil { + return nil, errors.WithMessage(err, "failed to get file") + } + if extracted.IsDir() { + return nil, errors.WithStack(errs.NotFile) + } + link, err := storageAr.Extract(ctx, archiveFile, args) + return &extractLink{Link: link, Obj: extracted}, err +} + +type streamWithParent struct { + rc io.ReadCloser + parent *stream.SeekableStream +} + +func (s *streamWithParent) Read(p []byte) (int, error) { + return s.rc.Read(p) +} + +func (s *streamWithParent) Close() error { + err1 := s.rc.Close() + err2 := s.parent.Close() + return stderrors.Join(err1, err2) +} + +func InternalExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + _, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + if err != nil { + return nil, 0, err + } + rc, size, err := t.Extract(ss, args) + if err != nil { + if e := ss.Close(); e != nil { + log.Errorf("failed to close file streamer, %v", e) + } + return nil, 0, err + } + return &streamWithParent{rc: rc, parent: ss}, size, nil +} + +func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + srcPath = utils.FixAndCleanPath(srcPath) + dstDirPath = utils.FixAndCleanPath(dstDirPath) + srcObj, err := GetUnwrap(ctx, storage, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed to get dst dir") + } + + switch s := storage.(type) { + case driver.ArchiveDecompressResult: + var newObjs []model.Obj + newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) + if err == nil { + if newObjs != nil && len(newObjs) > 0 { + for _, newObj := range newObjs { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.ArchiveDecompress: + err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errs.NotImplement + } + return errors.WithStack(err) +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 2c9543c1d94..b19eb07753d 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "math" "os" "github.com/alist-org/alist/v3/internal/errs" @@ -60,6 +61,8 @@ func (f *FileStream) Close() error { err2 = os.RemoveAll(f.tmpFile.Name()) if err2 != nil { err2 = errs.NewErr(err2, "failed to remove tmpFile [%s]", f.tmpFile.Name()) + } else { + f.tmpFile = nil } } @@ -92,6 +95,26 @@ func (f *FileStream) CacheFullInTempFile() (model.File, error) { return f.tmpFile, nil } +func (f *FileStream) CacheFullInTempFileAndUpdateProgress(up model.UpdateProgress) (model.File, error) { + if f.tmpFile != nil { + return f.tmpFile, nil + } + if file, ok := f.Reader.(model.File); ok { + return file, nil + } + tmpF, err := utils.CreateTempFile(&ReaderUpdatingProgress{ + Reader: f, + UpdateProgress: up, + }, f.GetSize()) + if err != nil { + return nil, err + } + f.Add(tmpF) + f.tmpFile = tmpF + f.Reader = tmpF + return f.tmpFile, nil +} + const InMemoryBufMaxSize = 10 // Megabytes const InMemoryBufMaxSizeBytes = InMemoryBufMaxSize * 1024 * 1024 @@ -247,7 +270,202 @@ func (ss *SeekableStream) CacheFullInTempFile() (model.File, error) { return ss.tmpFile, nil } +func (ss *SeekableStream) CacheFullInTempFileAndUpdateProgress(up model.UpdateProgress) (model.File, error) { + if ss.tmpFile != nil { + return ss.tmpFile, nil + } + if ss.mFile != nil { + return ss.mFile, nil + } + tmpF, err := utils.CreateTempFile(&ReaderUpdatingProgress{ + Reader: ss, + UpdateProgress: up, + }, ss.GetSize()) + if err != nil { + return nil, err + } + ss.Add(tmpF) + ss.tmpFile = tmpF + ss.Reader = tmpF + return ss.tmpFile, nil +} + func (f *FileStream) SetTmpFile(r *os.File) { f.Reader = r f.tmpFile = r } + +type ReaderWithSize interface { + io.Reader + GetSize() int64 +} + +type SimpleReaderWithSize struct { + io.Reader + Size int64 +} + +func (r *SimpleReaderWithSize) GetSize() int64 { + return r.Size +} + +type ReaderUpdatingProgress struct { + Reader ReaderWithSize + model.UpdateProgress + offset int +} + +func (r *ReaderUpdatingProgress) Read(p []byte) (n int, err error) { + n, err = r.Reader.Read(p) + r.offset += n + r.UpdateProgress(math.Min(100.0, float64(r.offset)/float64(r.Reader.GetSize())*100.0)) + return n, err +} + +type SStreamReadAtSeeker interface { + model.File + GetRawStream() *SeekableStream +} + +type readerCur struct { + reader io.Reader + cur int64 +} + +type RangeReadReadAtSeeker struct { + ss *SeekableStream + masterOff int64 + readers []*readerCur +} + +type FileReadAtSeeker struct { + ss *SeekableStream +} + +func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (SStreamReadAtSeeker, error) { + if ss.mFile != nil { + _, err := ss.mFile.Seek(offset, io.SeekStart) + if err != nil { + return nil, err + } + return &FileReadAtSeeker{ss: ss}, nil + } + var r io.Reader + var err error + if offset != 0 || utils.IsBool(forceRange...) { + if offset < 0 || offset > ss.GetSize() { + return nil, errors.New("offset out of range") + } + r, err = ss.RangeRead(http_range.Range{Start: offset, Length: -1}) + if err != nil { + return nil, err + } + if rc, ok := r.(io.Closer); ok { + ss.Closers.Add(rc) + } + } else { + r = ss + } + return &RangeReadReadAtSeeker{ + ss: ss, + masterOff: offset, + readers: []*readerCur{{reader: r, cur: offset}}, + }, nil +} + +func (r *RangeReadReadAtSeeker) GetRawStream() *SeekableStream { + return r.ss +} + +func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (*readerCur, error) { + for _, reader := range r.readers { + if reader.cur == off { + return reader, nil + } + } + reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: -1}) + if err != nil { + return nil, err + } + if c, ok := reader.(io.Closer); ok { + r.ss.Closers.Add(c) + } + rc := &readerCur{reader: reader, cur: off} + r.readers = append(r.readers, rc) + return rc, nil +} + +func (r *RangeReadReadAtSeeker) ReadAt(p []byte, off int64) (int, error) { + rc, err := r.getReaderAtOffset(off) + if err != nil { + return 0, err + } + num := 0 + for num < len(p) { + n, err := rc.reader.Read(p[num:]) + rc.cur += int64(n) + num += n + if err != nil { + return num, err + } + } + return num, nil +} + +func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + case io.SeekCurrent: + if offset == 0 { + return r.masterOff, nil + } + offset += r.masterOff + case io.SeekEnd: + offset += r.ss.GetSize() + default: + return 0, errs.NotSupport + } + if offset < 0 { + return r.masterOff, errors.New("invalid seek: negative position") + } + if offset > r.ss.GetSize() { + return r.masterOff, io.EOF + } + r.masterOff = offset + return offset, nil +} + +func (r *RangeReadReadAtSeeker) Read(p []byte) (n int, err error) { + rc, err := r.getReaderAtOffset(r.masterOff) + if err != nil { + return 0, err + } + n, err = rc.reader.Read(p) + rc.cur += int64(n) + r.masterOff += int64(n) + return n, err +} + +func (r *RangeReadReadAtSeeker) Close() error { + return r.ss.Close() +} + +func (f *FileReadAtSeeker) GetRawStream() *SeekableStream { + return f.ss +} + +func (f *FileReadAtSeeker) Read(p []byte) (n int, err error) { + return f.ss.mFile.Read(p) +} + +func (f *FileReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) { + return f.ss.mFile.ReadAt(p, off) +} + +func (f *FileReadAtSeeker) Seek(offset int64, whence int) (int64, error) { + return f.ss.mFile.Seek(offset, whence) +} + +func (f *FileReadAtSeeker) Close() error { + return f.ss.Close() +} diff --git a/internal/task/manager.go b/internal/task/manager.go new file mode 100644 index 00000000000..3caa685a9c9 --- /dev/null +++ b/internal/task/manager.go @@ -0,0 +1,20 @@ +package task + +import "github.com/xhofe/tache" + +type Manager[T tache.Task] interface { + Add(task T) + Cancel(id string) + CancelAll() + CancelByCondition(condition func(task T) bool) + GetAll() []T + GetByID(id string) (T, bool) + GetByState(state ...tache.State) []T + GetByCondition(condition func(task T) bool) []T + Remove(id string) + RemoveAll() + RemoveByState(state ...tache.State) + RemoveByCondition(condition func(task T) bool) + Retry(id string) + RetryAllFailed() +} diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index 257d2ec838a..f7e018e0f5f 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -8,10 +8,8 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/server/common" "github.com/pkg/errors" - "io" fs2 "io/fs" "net/http" "os" @@ -20,9 +18,7 @@ import ( type FileDownloadProxy struct { ftpserver.FileTransfer - ss *stream.SeekableStream - reader io.Reader - cur int64 + reader stream.SStreamReadAtSeeker } func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownloadProxy, error) { @@ -55,22 +51,16 @@ func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownl if err != nil { return nil, err } - var reader io.Reader - if offset != 0 { - reader, err = ss.RangeRead(http_range.Range{Start: offset, Length: -1}) - if err != nil { - return nil, err - } - } else { - reader = ss + reader, err := stream.NewReadAtSeeker(ss, offset) + if err != nil { + _ = ss.Close() + return nil, err } - return &FileDownloadProxy{ss: ss, reader: reader}, nil + return &FileDownloadProxy{reader: reader}, nil } func (f *FileDownloadProxy) Read(p []byte) (n int, err error) { - n, err = f.reader.Read(p) - f.cur += int64(n) - return n, err + return f.reader.Read(p) } func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { @@ -78,32 +68,11 @@ func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { } func (f *FileDownloadProxy) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - break - case io.SeekCurrent: - offset += f.cur - break - case io.SeekEnd: - offset += f.ss.GetSize() - break - default: - return 0, errs.NotSupport - } - if offset < 0 { - return 0, errors.New("Seek: negative position") - } - reader, err := f.ss.RangeRead(http_range.Range{Start: offset, Length: -1}) - if err != nil { - return f.cur, err - } - f.cur = offset - f.reader = reader - return offset, nil + return f.reader.Seek(offset, whence) } func (f *FileDownloadProxy) Close() error { - return f.ss.Close() + return f.reader.Close() } type OsFileInfoAdapter struct { diff --git a/server/handles/archive.go b/server/handles/archive.go new file mode 100644 index 00000000000..29dbf3c2d34 --- /dev/null +++ b/server/handles/archive.go @@ -0,0 +1,381 @@ +package handles + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "mime" + stdpath "path" + "strings" +) + +type ArchiveMetaReq struct { + Path string `json:"path" form:"path"` + Password string `json:"password" form:"password"` + Refresh bool `json:"refresh" form:"refresh"` + ArchivePass string `json:"archive_pass" form:"archive_pass"` +} + +type ArchiveMetaResp struct { + Comment string `json:"comment"` + IsEncrypted bool `json:"encrypted"` + Content []ArchiveContentResp `json:"content"` + RawURL string `json:"raw_url"` + Sign string `json:"sign"` +} + +type ArchiveContentResp struct { + ObjResp + Children []ArchiveContentResp `json:"children,omitempty"` +} + +func toObjsRespWithoutSignAndThumb(obj model.Obj) ObjResp { + return ObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: "", + Thumb: "", + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + } +} + +func toContentResp(objs []model.ObjTree) []ArchiveContentResp { + if objs == nil { + return nil + } + ret, _ := utils.SliceConvert(objs, func(src model.ObjTree) (ArchiveContentResp, error) { + return ArchiveContentResp{ + ObjResp: toObjsRespWithoutSignAndThumb(src), + Children: toContentResp(src.GetChildren()), + }, nil + }) + return ret +} + +func FsArchiveMeta(c *gin.Context) { + var req ArchiveMetaReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if !user.CanReadArchives() { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("meta", meta) + if !common.CanAccess(user, meta, reqPath, req.Password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + archiveArgs := model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: req.ArchivePass, + } + ret, err := fs.ArchiveMeta(c, reqPath, model.ArchiveMetaArgs{ + ArchiveArgs: archiveArgs, + Refresh: req.Refresh, + }) + if err != nil { + if errors.Is(err, errs.WrongArchivePassword) { + common.ErrorResp(c, err, 202) + } else { + common.ErrorResp(c, err, 500) + } + return + } + s := "" + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + s = sign.Sign(reqPath) + } + api := "/ae" + if ret.DriverProviding { + api = "/ad" + } + common.SuccessResp(c, ArchiveMetaResp{ + Comment: ret.GetComment(), + IsEncrypted: ret.IsEncrypted(), + Content: toContentResp(ret.GetTree()), + RawURL: fmt.Sprintf("%s%s%s", common.GetApiUrl(c.Request), api, utils.EncodePath(reqPath, true)), + Sign: s, + }) +} + +type ArchiveListReq struct { + ArchiveMetaReq + model.PageReq + InnerPath string `json:"inner_path" form:"inner_path"` +} + +type ArchiveListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` +} + +func FsArchiveList(c *gin.Context) { + var req ArchiveListReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + user := c.MustGet("user").(*model.User) + if !user.CanReadArchives() { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("meta", meta) + if !common.CanAccess(user, meta, reqPath, req.Password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + objs, err := fs.ArchiveList(c, reqPath, model.ArchiveListArgs{ + ArchiveInnerArgs: model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: req.ArchivePass, + }, + InnerPath: utils.FixAndCleanPath(req.InnerPath), + }, + Refresh: req.Refresh, + }) + if err != nil { + if errors.Is(err, errs.WrongArchivePassword) { + common.ErrorResp(c, err, 202) + } else { + common.ErrorResp(c, err, 500) + } + return + } + total, objs := pagination(objs, &req.PageReq) + ret, _ := utils.SliceConvert(objs, func(src model.Obj) (ObjResp, error) { + return toObjsRespWithoutSignAndThumb(src), nil + }) + common.SuccessResp(c, ArchiveListResp{ + Content: ret, + Total: int64(total), + }) +} + +type ArchiveDecompressReq struct { + SrcDir string `json:"src_dir" form:"src_dir"` + DstDir string `json:"dst_dir" form:"dst_dir"` + Name string `json:"name" form:"name"` + ArchivePass string `json:"archive_pass" form:"archive_pass"` + InnerPath string `json:"inner_path" form:"inner_path"` + CacheFull bool `json:"cache_full" form:"cache_full"` + PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` +} + +func FsArchiveDecompress(c *gin.Context) { + var req ArchiveDecompressReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if !user.CanDecompress() { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, req.Name)) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + dstDir, err := user.JoinPath(req.DstDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + t, err := fs.ArchiveDecompress(c, srcPath, dstDir, model.ArchiveDecompressArgs{ + ArchiveInnerArgs: model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: req.ArchivePass, + }, + InnerPath: utils.FixAndCleanPath(req.InnerPath), + }, + CacheFull: req.CacheFull, + PutIntoNewDir: req.PutIntoNewDir, + }) + if err != nil { + if errors.Is(err, errs.WrongArchivePassword) { + common.ErrorResp(c, err, 202) + } else { + common.ErrorResp(c, err, 500) + } + return + } + common.SuccessResp(c, gin.H{ + "task": getTaskInfo(t), + }) +} + +func ArchiveDown(c *gin.Context) { + archiveRawPath := c.MustGet("path").(string) + innerPath := utils.FixAndCleanPath(c.Query("inner")) + password := c.Query("pass") + filename := stdpath.Base(innerPath) + storage, err := fs.GetStorage(archiveRawPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if common.ShouldProxy(storage, filename) { + ArchiveProxy(c) + return + } else { + link, _, err := fs.ArchiveDriverExtract(c, archiveRawPath, model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + IP: c.ClientIP(), + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: password, + }, + InnerPath: innerPath, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + down(c, link) + } +} + +func ArchiveProxy(c *gin.Context) { + archiveRawPath := c.MustGet("path").(string) + innerPath := utils.FixAndCleanPath(c.Query("inner")) + password := c.Query("pass") + filename := stdpath.Base(innerPath) + storage, err := fs.GetStorage(archiveRawPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if canProxy(storage, filename) { + // TODO: Support external download proxy URL + link, file, err := fs.ArchiveDriverExtract(c, archiveRawPath, model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: password, + }, + InnerPath: innerPath, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + localProxy(c, link, file, storage.GetStorage().ProxyRange) + } else { + common.ErrorStrResp(c, "proxy not allowed", 403) + return + } +} + +func ArchiveInternalExtract(c *gin.Context) { + archiveRawPath := c.MustGet("path").(string) + innerPath := utils.FixAndCleanPath(c.Query("inner")) + password := c.Query("pass") + rc, size, err := fs.ArchiveInternalExtract(c, archiveRawPath, model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: password, + }, + InnerPath: innerPath, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + defer func() { + if err := rc.Close(); err != nil { + log.Errorf("failed to close file streamer, %v", err) + } + }() + headers := map[string]string{ + "Referrer-Policy": "no-referrer", + "Cache-Control": "max-age=0, no-cache, no-store, must-revalidate", + } + if c.Query("attachment") == "true" { + filename := stdpath.Base(innerPath) + headers["Content-Disposition"] = fmt.Sprintf("attachment; filename=\"%s\"", filename) + } + contentType := c.Request.Header.Get("Content-Type") + if contentType == "" { + fileExt := stdpath.Ext(innerPath) + contentType = mime.TypeByExtension(fileExt) + } + c.DataFromReader(200, size, contentType, rc, headers) +} + +func ArchiveExtensions(c *gin.Context) { + var ext []string + for key := range tool.Tools { + ext = append(ext, strings.TrimPrefix(key, ".")) + } + common.SuccessResp(c, ext) +} diff --git a/server/handles/down.go b/server/handles/down.go index 0020ed1453e..f01c9d6683b 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -40,28 +40,7 @@ func Down(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if link.MFile != nil { - defer func(ReadSeekCloser io.ReadCloser) { - err := ReadSeekCloser.Close() - if err != nil { - log.Errorf("close data error: %s", err) - } - }(link.MFile) - } - c.Header("Referrer-Policy", "no-referrer") - c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") - if setting.GetBool(conf.ForwardDirectLinkParams) { - query := c.Request.URL.Query() - for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { - query.Del(v) - } - link.URL, err = utils.InjectQuery(link.URL, query) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - } - c.Redirect(302, link.URL) + down(c, link) } } @@ -95,27 +74,58 @@ func Proxy(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if link.URL != "" && setting.GetBool(conf.ForwardDirectLinkParams) { - query := c.Request.URL.Query() - for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { - query.Del(v) - } - link.URL, err = utils.InjectQuery(link.URL, query) + localProxy(c, link, file, storage.GetStorage().ProxyRange) + } else { + common.ErrorStrResp(c, "proxy not allowed", 403) + return + } +} + +func down(c *gin.Context, link *model.Link) { + var err error + if link.MFile != nil { + defer func(ReadSeekCloser io.ReadCloser) { + err := ReadSeekCloser.Close() if err != nil { - common.ErrorResp(c, err, 500) - return + log.Errorf("close data error: %s", err) } + }(link.MFile) + } + c.Header("Referrer-Policy", "no-referrer") + c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") + if setting.GetBool(conf.ForwardDirectLinkParams) { + query := c.Request.URL.Query() + for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { + query.Del(v) } - if storage.GetStorage().ProxyRange { - common.ProxyRange(link, file.GetSize()) + link.URL, err = utils.InjectQuery(link.URL, query) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + } + c.Redirect(302, link.URL) +} + +func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { + var err error + if link.URL != "" && setting.GetBool(conf.ForwardDirectLinkParams) { + query := c.Request.URL.Query() + for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { + query.Del(v) } - err = common.Proxy(c.Writer, c.Request, link, file) + link.URL, err = utils.InjectQuery(link.URL, query) if err != nil { - common.ErrorResp(c, err, 500, true) + common.ErrorResp(c, err, 500) return } - } else { - common.ErrorStrResp(c, "proxy not allowed", 403) + } + if proxyRange { + common.ProxyRange(link, file.GetSize()) + } + err = common.Proxy(c.Writer, c.Request, link, file) + if err != nil { + common.ErrorResp(c, err, 500, true) return } } diff --git a/server/handles/task.go b/server/handles/task.go index c7d9ef4842d..af7974a9c29 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -75,7 +75,7 @@ func getUserInfo(c *gin.Context) (bool, uint, bool) { } } -func getTargetedHandler[T task.TaskExtensionInfo](manager *tache.Manager[T], callback func(c *gin.Context, task T)) gin.HandlerFunc { +func getTargetedHandler[T task.TaskExtensionInfo](manager task.Manager[T], callback func(c *gin.Context, task T)) gin.HandlerFunc { return func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { @@ -97,7 +97,7 @@ func getTargetedHandler[T task.TaskExtensionInfo](manager *tache.Manager[T], cal } } -func getBatchHandler[T task.TaskExtensionInfo](manager *tache.Manager[T], callback func(task T)) gin.HandlerFunc { +func getBatchHandler[T task.TaskExtensionInfo](manager task.Manager[T], callback func(task T)) gin.HandlerFunc { return func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { @@ -122,7 +122,7 @@ func getBatchHandler[T task.TaskExtensionInfo](manager *tache.Manager[T], callba } } -func taskRoute[T task.TaskExtensionInfo](g *gin.RouterGroup, manager *tache.Manager[T]) { +func taskRoute[T task.TaskExtensionInfo](g *gin.RouterGroup, manager task.Manager[T]) { g.GET("/undone", func(c *gin.Context) { isAdmin, uid, ok := getUserInfo(c) if !ok { @@ -220,4 +220,6 @@ func SetupTaskRoute(g *gin.RouterGroup) { taskRoute(g.Group("/copy"), fs.CopyTaskManager) taskRoute(g.Group("/offline_download"), tool.DownloadTaskManager) taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager) + taskRoute(g.Group("/decompress"), fs.ArchiveDownloadTaskManager) + taskRoute(g.Group("/decompress_upload"), fs.ArchiveContentUploadTaskManager) } diff --git a/server/router.go b/server/router.go index 184de51e46a..63bad60f03f 100644 --- a/server/router.go +++ b/server/router.go @@ -42,6 +42,12 @@ func Init(e *gin.Engine) { g.GET("/p/*path", middlewares.Down, handles.Proxy) g.HEAD("/d/*path", middlewares.Down, handles.Down) g.HEAD("/p/*path", middlewares.Down, handles.Proxy) + g.GET("/ad/*path", middlewares.Down, handles.ArchiveDown) + g.GET("/ap/*path", middlewares.Down, handles.ArchiveProxy) + g.GET("/ae/*path", middlewares.Down, handles.ArchiveInternalExtract) + g.HEAD("/ad/*path", middlewares.Down, handles.ArchiveDown) + g.HEAD("/ap/*path", middlewares.Down, handles.ArchiveProxy) + g.HEAD("/ae/*path", middlewares.Down, handles.ArchiveInternalExtract) api := g.Group("/api") auth := api.Group("", middlewares.Auth) @@ -77,6 +83,7 @@ func Init(e *gin.Engine) { public := api.Group("/public") public.Any("/settings", handles.PublicSettings) public.Any("/offline_download_tools", handles.OfflineDownloadTools) + public.Any("/archive_extensions", handles.ArchiveExtensions) _fs(auth.Group("/fs")) _task(auth.Group("/task", middlewares.AuthNotGuest)) @@ -173,6 +180,10 @@ func _fs(g *gin.RouterGroup) { // g.POST("/add_qbit", handles.AddQbittorrent) // g.POST("/add_transmission", handles.SetTransmission) g.POST("/add_offline_download", handles.AddOfflineDownload) + a := g.Group("/archive") + a.Any("/meta", handles.FsArchiveMeta) + a.Any("/list", handles.FsArchiveList) + a.POST("/decompress", handles.FsArchiveDecompress) } func _task(g *gin.RouterGroup) { From 59e02287b2bbd7f36358305421a20f79a5322f85 Mon Sep 17 00:00:00 2001 From: Jealous Date: Sat, 18 Jan 2025 23:39:07 +0800 Subject: [PATCH 417/659] feat(fs): add `overwrite` option to preventing unintentional overwriting (#7809) --- server/handles/fsmanage.go | 37 ++++++++++++++++++++++++++++++++----- server/handles/fsup.go | 16 ++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 9877b1278d7..9349e7e275d 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -56,9 +56,10 @@ func FsMkdir(c *gin.Context) { } type MoveCopyReq struct { - SrcDir string `json:"src_dir"` - DstDir string `json:"dst_dir"` - Names []string `json:"names"` + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + Names []string `json:"names"` + Overwrite bool `json:"overwrite"` } func FsMove(c *gin.Context) { @@ -86,6 +87,14 @@ func FsMove(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !req.Overwrite { + for _, name := range req.Names { + if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, "file exists", 403) + return + } + } + } for i, name := range req.Names { err := fs.Move(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) if err != nil { @@ -121,6 +130,14 @@ func FsCopy(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !req.Overwrite { + for _, name := range req.Names { + if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, "file exists", 403) + return + } + } + } var addedTasks []task.TaskExtensionInfo for i, name := range req.Names { t, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) @@ -138,8 +155,9 @@ func FsCopy(c *gin.Context) { } type RenameReq struct { - Path string `json:"path"` - Name string `json:"name"` + Path string `json:"path"` + Name string `json:"name"` + Overwrite bool `json:"overwrite"` } func FsRename(c *gin.Context) { @@ -158,6 +176,15 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !req.Overwrite { + dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) + if dstPath != reqPath { + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, "file exists", 403) + return + } + } + } if err := fs.Rename(c, reqPath, req.Name); err != nil { common.ErrorResp(c, err, 500) return diff --git a/server/handles/fsup.go b/server/handles/fsup.go index a17c50f08ee..563afbcd54a 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -34,12 +34,20 @@ func FsStream(c *gin.Context) { return } asTask := c.GetHeader("As-Task") == "true" + overwrite := c.GetHeader("Overwrite") != "false" user := c.MustGet("user").(*model.User) path, err = user.JoinPath(path) if err != nil { common.ErrorResp(c, err, 403) return } + if !overwrite { + if res, _ := fs.Get(c, path, &fs.GetArgs{NoLog: true}); res != nil { + _, _ = io.Copy(io.Discard, c.Request.Body) + common.ErrorStrResp(c, "file exists", 403) + return + } + } dir, name := stdpath.Split(path) sizeStr := c.GetHeader("Content-Length") size, err := strconv.ParseInt(sizeStr, 10, 64) @@ -85,12 +93,20 @@ func FsForm(c *gin.Context) { return } asTask := c.GetHeader("As-Task") == "true" + overwrite := c.GetHeader("Overwrite") != "false" user := c.MustGet("user").(*model.User) path, err = user.JoinPath(path) if err != nil { common.ErrorResp(c, err, 403) return } + if !overwrite { + if res, _ := fs.Get(c, path, &fs.GetArgs{NoLog: true}); res != nil { + _, _ = io.Copy(io.Discard, c.Request.Body) + common.ErrorStrResp(c, "file exists", 403) + return + } + } storage, err := fs.GetStorage(path, &fs.GetStoragesArgs{}) if err != nil { common.ErrorResp(c, err, 400) From 11b6a6012f256facbeaf9314281321b05eeadef3 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Sat, 18 Jan 2025 23:52:02 +0800 Subject: [PATCH 418/659] fix(copy): use Link and Put when the driver does not support copying (#7834) --- internal/fs/copy.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/fs/copy.go b/internal/fs/copy.go index c3fadaab8fa..977f7280db9 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -3,6 +3,7 @@ package fs import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/errs" "net/http" stdpath "path" "time" @@ -69,7 +70,10 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool } // copy if in the same storage, just call driver.Copy if srcStorage.GetStorage() == dstStorage.GetStorage() { - return nil, op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { + return nil, err + } } if ctx.Value(conf.NoTaskKey) != nil { srcObj, err := op.Get(ctx, srcStorage, srcObjActualPath) From c2633dd4436a2337246979a9d866f7e408e6c2ea Mon Sep 17 00:00:00 2001 From: Jealous Date: Thu, 23 Jan 2025 22:49:35 +0800 Subject: [PATCH 419/659] fix(workflow): use the dev version of the web for beta releases (#7862) * fix(workflow): use dev version of the web for beta releases * chore(config): check version string by prefix --- .github/workflows/release_docker.yml | 7 ++++++- build.sh | 11 +++++++++-- internal/bootstrap/config.go | 2 +- internal/bootstrap/patch.go | 3 ++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index f4c79baf2df..7cd05549f18 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -52,7 +52,12 @@ jobs: if: steps.cache-musl.outputs.cache-hit != 'true' run: bash build.sh prepare docker-multiplatform - - name: Build go binary + - name: Build go binary (beta) + if: env.IMAGE_IS_PROD != 'true' + run: bash build.sh beta docker-multiplatform + + - name: Build go binary (release) + if: env.IMAGE_IS_PROD == 'true' run: bash build.sh release docker-multiplatform - name: Upload artifacts diff --git a/build.sh b/build.sh index a87eabf4f81..d6e001c204f 100644 --- a/build.sh +++ b/build.sh @@ -7,6 +7,9 @@ gitCommit=$(git log --pretty=format:"%h" -1) if [ "$1" = "dev" ]; then version="dev" webVersion="dev" +elif [ "$1" = "beta" ]; then + version="beta" + webVersion="dev" else git tag -d beta version=$(git describe --abbrev=0 --tags) @@ -301,8 +304,12 @@ if [ "$1" = "dev" ]; then else BuildDev fi -elif [ "$1" = "release" ]; then - FetchWebRelease +elif [ "$1" = "release" -o "$1" = "beta" ]; then + if [ "$1" = "beta" ]; then + FetchWebDev + else + FetchWebRelease + fi if [ "$2" = "docker" ]; then BuildDocker elif [ "$2" = "docker-multiplatform" ]; then diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index a44c735056b..38b1aa9e269 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -50,7 +50,7 @@ func InitConfig() { log.Fatalf("load config error: %+v", err) } LastLaunchedVersion = conf.Conf.LastLaunchedVersion - if conf.Version != "dev" || LastLaunchedVersion == "" { + if strings.HasPrefix(conf.Version, "v") || LastLaunchedVersion == "" { conf.Conf.LastLaunchedVersion = conf.Version } // update config.json struct diff --git a/internal/bootstrap/patch.go b/internal/bootstrap/patch.go index 8dc3ed02745..2d22d1b6388 100644 --- a/internal/bootstrap/patch.go +++ b/internal/bootstrap/patch.go @@ -5,6 +5,7 @@ import ( "github.com/alist-org/alist/v3/internal/bootstrap/patch" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/pkg/utils" + "strings" ) var LastLaunchedVersion = "" @@ -38,7 +39,7 @@ func compareVersion(majorA, minorA, patchNumA, majorB, minorB, patchNumB int) bo } func InitUpgradePatch() { - if conf.Version == "dev" { + if !strings.HasPrefix(conf.Version, "v") { return } if LastLaunchedVersion == conf.Version { From bdcf450203b70c05d748c605fbf9df9c47c98b2c Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Mon, 27 Jan 2025 20:06:18 +0800 Subject: [PATCH 420/659] fix: resolve concurrent read/write issues in WrapObjName (#7865) --- internal/model/obj.go | 4 ++-- internal/model/object.go | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/internal/model/obj.go b/internal/model/obj.go index 2a72ca9eff4..552b1241e6e 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -115,12 +115,12 @@ func ExtractFolder(objs []Obj, extractFolder string) { } func WrapObjName(objs Obj) Obj { - return &ObjWrapName{Obj: objs} + return &ObjWrapName{Name: utils.MappingName(objs.GetName()), Obj: objs} } func WrapObjsName(objs []Obj) { for i := 0; i < len(objs); i++ { - objs[i] = &ObjWrapName{Obj: objs[i]} + objs[i] = &ObjWrapName{Name: utils.MappingName(objs[i].GetName()), Obj: objs[i]} } } diff --git a/internal/model/object.go b/internal/model/object.go index 93f2c307a03..c8c10bb9d92 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -16,9 +16,6 @@ func (o *ObjWrapName) Unwrap() Obj { } func (o *ObjWrapName) GetName() string { - if o.Name == "" { - o.Name = utils.MappingName(o.Obj.GetName()) - } return o.Name } From 2be0c3d1a088d2c74bb429c9d6072a73bd30fb1b Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Mon, 27 Jan 2025 20:08:39 +0800 Subject: [PATCH 421/659] feat(alias): add `DownloadConcurrency` and `DownloadPartSize` option (#7829) * fix(net): goroutine logic bug (AlistGo/alist#7215) * Fix goroutine logic bug * Fix bug --------- Co-authored-by: hpy hs * perf(net): sequential and dynamic concurrency * fix(net): incorrect error return * feat(alias): add `DownloadConcurrency` and `DownloadPartSize` option * feat(net): add `ConcurrencyLimit` * pref(net): create `chunk` on demand * refactor * refactor * fix(net): `r.Closers.Add` has no effect * refactor --------- Co-authored-by: hpy hs --- drivers/alias/driver.go | 10 + drivers/alias/meta.go | 6 +- drivers/alias/util.go | 10 +- drivers/crypt/driver.go | 7 +- drivers/github/driver.go | 15 +- drivers/halalcloud/driver.go | 16 +- drivers/mega/driver.go | 4 +- drivers/netease_music/types.go | 1 - drivers/netease_music/upload.go | 2 +- drivers/quqi/util.go | 4 +- internal/bootstrap/config.go | 4 + internal/conf/config.go | 2 + internal/model/args.go | 11 +- internal/net/request.go | 364 +++++++++++++----- internal/net/serve.go | 35 +- internal/net/util.go | 3 +- .../offline_download/transmission/client.go | 4 +- internal/stream/stream.go | 5 +- internal/stream/util.go | 44 +-- server/common/proxy.go | 15 +- server/handles/archive.go | 9 +- server/handles/down.go | 9 +- server/s3/backend.go | 52 +-- server/webdav/webdav.go | 2 +- 24 files changed, 396 insertions(+), 238 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 1b439a2c9d9..16215c8e784 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -110,6 +110,16 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( for _, dst := range dsts { link, err := d.link(ctx, dst, sub, args) if err == nil { + if !args.Redirect && len(link.URL) > 0 { + // 正常情况下 多并发 仅支持返回URL的驱动 + // alias套娃alias 可以让crypt、mega等驱动(不返回URL的) 支持并发 + if d.DownloadConcurrency > 0 { + link.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + link.PartSize = d.DownloadPartSize * utils.KB + } + } return link, nil } } diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 45b885753d0..ed657a5d21b 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -9,8 +9,10 @@ type Addition struct { // Usually one of two // driver.RootPath // define other - Paths string `json:"paths" required:"true" type:"text"` - ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"` + Paths string `json:"paths" required:"true" type:"text"` + ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"` + DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` + DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` } var config = driver.Config{ diff --git a/drivers/alias/util.go b/drivers/alias/util.go index c0e9081b0fc..ee17b622e13 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -9,6 +9,7 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -94,10 +95,15 @@ func (d *Alias) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([ func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) (*model.Link, error) { reqPath := stdpath.Join(dst, sub) - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + // 参考 crypt 驱动 + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err } + if _, ok := storage.(*Alias); !ok && !args.Redirect { + link, _, err := op.Link(ctx, storage, reqActualPath, args) + return link, err + } _, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) if err != nil { return nil, err @@ -114,7 +120,7 @@ func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) } return link, nil } - link, _, err := fs.Link(ctx, reqPath, args) + link, _, err := op.Link(ctx, storage, reqActualPath, args) return link, err } diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index b6115896b98..e6f253d187a 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -275,7 +275,6 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( rrc = converted } if rrc != nil { - //remoteRangeReader, err := remoteReader, err := rrc.RangeRead(ctx, http_range.Range{Start: underlyingOffset, Length: length}) remoteClosers.AddClosers(rrc.GetClosers()) if err != nil { @@ -288,10 +287,8 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if err != nil { return nil, err } - //remoteClosers.Add(remoteLink.MFile) - //keep reuse same MFile and close at last. - remoteClosers.Add(remoteLink.MFile) - return io.NopCloser(remoteLink.MFile), nil + // 可以直接返回，读取完也不会调用Close，直到连接断开Close + return remoteLink.MFile, nil } return nil, errs.NotSupport diff --git a/drivers/github/driver.go b/drivers/github/driver.go index ea8f62762ed..eed06882984 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -5,6 +5,13 @@ import ( "encoding/base64" "errors" "fmt" + "io" + "net/http" + stdpath "path" + "strings" + "sync" + "text/template" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -12,12 +19,6 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" - "io" - "net/http" - stdpath "path" - "strings" - "sync" - "text/template" ) type Github struct { @@ -656,7 +657,7 @@ func (d *Github) putBlob(ctx context.Context, stream model.FileStreamer, up driv contentReader, contentWriter := io.Pipe() go func() { encoder := base64.NewEncoder(base64.StdEncoding, contentWriter) - if _, err := io.Copy(encoder, stream); err != nil { + if _, err := utils.CopyWithBuffer(encoder, stream); err != nil { _ = contentWriter.CloseWithError(err) return } diff --git a/drivers/halalcloud/driver.go b/drivers/halalcloud/driver.go index 08bb3808bfd..d3235828201 100644 --- a/drivers/halalcloud/driver.go +++ b/drivers/halalcloud/driver.go @@ -4,12 +4,17 @@ import ( "context" "crypto/sha1" "fmt" + "io" + "net/url" + "path" + "strconv" + "time" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" @@ -19,11 +24,6 @@ import ( pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" "github.com/rclone/rclone/lib/readers" "github.com/zzzhr1990/go-common-entity/userfile" - "io" - "net/url" - "path" - "strconv" - "time" ) type HalalCloud struct { @@ -251,7 +251,6 @@ func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.Lin size := result.FileSize chunks := getChunkSizes(result.Sizes) - var finalClosers utils.Closers resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if httpRange.Length >= 0 && httpRange.Start+httpRange.Length >= size { @@ -269,7 +268,6 @@ func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.Lin sha: result.Sha1, shaTemp: sha1.New(), } - finalClosers.Add(oo) return readers.NewLimitedReadCloser(oo, length), nil } @@ -281,7 +279,7 @@ func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.Lin duration = time.Until(time.Now().Add(time.Hour)) } - resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: finalClosers} + resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader} return &model.Link{ RangeReadCloser: resultRangeReadCloser, Expiration: &duration, diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index 162aeef37e0..198c1f9864c 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -84,7 +84,6 @@ func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* //} size := file.GetSize() - var finalClosers utils.Closers resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if httpRange.Length >= 0 && httpRange.Start+httpRange.Length >= size { @@ -103,11 +102,10 @@ func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* d: down, skip: httpRange.Start, } - finalClosers.Add(oo) return readers.NewLimitedReadCloser(oo, length), nil } - resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: finalClosers} + resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader} resultLink := &model.Link{ RangeReadCloser: resultRangeReadCloser, } diff --git a/drivers/netease_music/types.go b/drivers/netease_music/types.go index edbd40eed59..0e156ad1579 100644 --- a/drivers/netease_music/types.go +++ b/drivers/netease_music/types.go @@ -64,7 +64,6 @@ func (lrc *LyricObj) getLyricLink() *model.Link { sr := io.NewSectionReader(reader, httpRange.Start, httpRange.Length) return io.NopCloser(sr), nil }, - Closers: utils.EmptyClosers(), }, } } diff --git a/drivers/netease_music/upload.go b/drivers/netease_music/upload.go index ece496b36da..7f580bd1744 100644 --- a/drivers/netease_music/upload.go +++ b/drivers/netease_music/upload.go @@ -47,7 +47,7 @@ func (u *uploader) init(stream model.FileStreamer) error { } h := md5.New() - io.Copy(h, stream) + utils.CopyWithBuffer(h, stream) u.md5 = hex.EncodeToString(h.Sum(nil)) _, err := u.file.Seek(0, io.SeekStart) if err != nil { diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go index c025f6ee8af..c57e641bf1e 100644 --- a/drivers/quqi/util.go +++ b/drivers/quqi/util.go @@ -300,9 +300,7 @@ func (d *Quqi) linkFromCDN(id string) (*model.Link, error) { bufferReader := bufio.NewReader(decryptReader) bufferReader.Discard(int(decryptedOffset)) - return utils.NewReadCloser(bufferReader, func() error { - return nil - }), nil + return io.NopCloser(bufferReader), nil } return &model.Link{ diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 38b1aa9e269..db3e20942b6 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -9,6 +9,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/pkg/utils" "github.com/caarlos0/env/v9" log "github.com/sirupsen/logrus" @@ -63,6 +64,9 @@ func InitConfig() { log.Fatalf("update config struct error: %+v", err) } } + if conf.Conf.MaxConcurrency > 0 { + net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: conf.Conf.MaxConcurrency} + } if !conf.Conf.Force { confFromEnv() } diff --git a/internal/conf/config.go b/internal/conf/config.go index 4f5c2ae0e35..39b23227a26 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -106,6 +106,7 @@ type Config struct { Log LogConfig `json:"log"` DelayedStart int `json:"delayed_start" env:"DELAYED_START"` MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` + MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"` TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"` Cors Cors `json:"cors" envPrefix:"CORS_"` @@ -151,6 +152,7 @@ func DefaultConfig() *Config { MaxAge: 28, }, MaxConnections: 0, + MaxConcurrency: 64, TlsInsecureSkipVerify: true, Tasks: TasksConfig{ Download: TaskConfig{ diff --git a/internal/model/args.go b/internal/model/args.go index a9feeb206ef..f29c7e45ee9 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -17,10 +17,11 @@ type ListArgs struct { } type LinkArgs struct { - IP string - Header http.Header - Type string - HttpReq *http.Request + IP string + Header http.Header + Type string + HttpReq *http.Request + Redirect bool } type Link struct { @@ -87,7 +88,7 @@ type RangeReadCloser struct { utils.Closers } -func (r RangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { +func (r *RangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { rc, err := r.RangeReader(ctx, httpRange) r.Closers.Add(rc) return rc, err diff --git a/internal/net/request.go b/internal/net/request.go index 1a7405e4260..d2f3028fac9 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "io" - "math" "net/http" "strconv" "strings" @@ -21,7 +20,7 @@ import ( // DefaultDownloadPartSize is the default range of bytes to get at a time when // using Download(). -const DefaultDownloadPartSize = 1024 * 1024 * 10 +const DefaultDownloadPartSize = utils.MB * 10 // DefaultDownloadConcurrency is the default number of goroutines to spin up // when using Download(). @@ -30,6 +29,8 @@ const DefaultDownloadConcurrency = 2 // DefaultPartBodyMaxRetries is the default number of retries to make when a part fails to download. const DefaultPartBodyMaxRetries = 3 +var DefaultConcurrencyLimit *ConcurrencyLimit + type Downloader struct { PartSize int @@ -44,15 +45,15 @@ type Downloader struct { //RequestParam HttpRequestParams HttpClient HttpRequestFunc + + *ConcurrencyLimit } type HttpRequestFunc func(ctx context.Context, params *HttpRequestParams) (*http.Response, error) func NewDownloader(options ...func(*Downloader)) *Downloader { - d := &Downloader{ - HttpClient: DefaultHttpRequestFunc, - PartSize: DefaultDownloadPartSize, + d := &Downloader{ //允许不设置的选项 PartBodyMaxRetries: DefaultPartBodyMaxRetries, - Concurrency: DefaultDownloadConcurrency, + ConcurrencyLimit: DefaultConcurrencyLimit, } for _, option := range options { option(d) @@ -74,16 +75,16 @@ func (d Downloader) Download(ctx context.Context, p *HttpRequestParams) (readClo impl := downloader{params: &finalP, cfg: d, ctx: ctx} // Ensures we don't need nil checks later on - - impl.partBodyMaxRetries = d.PartBodyMaxRetries - + // 必需的选项 if impl.cfg.Concurrency == 0 { impl.cfg.Concurrency = DefaultDownloadConcurrency } - if impl.cfg.PartSize == 0 { impl.cfg.PartSize = DefaultDownloadPartSize } + if impl.cfg.HttpClient == nil { + impl.cfg.HttpClient = DefaultHttpRequestFunc + } return impl.download() } @@ -91,7 +92,7 @@ func (d Downloader) Download(ctx context.Context, p *HttpRequestParams) (readClo // downloader is the implementation structure used internally by Downloader. type downloader struct { ctx context.Context - cancel context.CancelFunc + cancel context.CancelCauseFunc cfg Downloader params *HttpRequestParams //http request params @@ -101,38 +102,78 @@ type downloader struct { m sync.Mutex nextChunk int //next chunk id - chunks []chunk bufs []*Buf - //totalBytes int64 - written int64 //total bytes of file downloaded from remote - err error + written int64 //total bytes of file downloaded from remote + err error + + concurrency int //剩余的并发数，递减。到0时停止并发 + maxPart int //有多少个分片 + pos int64 + maxPos int64 + m2 sync.Mutex + readingID int // 正在被读取的id +} + +type ConcurrencyLimit struct { + _m sync.Mutex + Limit int // 需要大于0 +} + +var ErrExceedMaxConcurrency = fmt.Errorf("ExceedMaxConcurrency") - partBodyMaxRetries int +func (l *ConcurrencyLimit) sub() error { + l._m.Lock() + defer l._m.Unlock() + if l.Limit-1 < 0 { + return ErrExceedMaxConcurrency + } + l.Limit-- + // log.Debugf("ConcurrencyLimit.sub: %d", l.Limit) + return nil +} +func (l *ConcurrencyLimit) add() { + l._m.Lock() + defer l._m.Unlock() + l.Limit++ + // log.Debugf("ConcurrencyLimit.add: %d", l.Limit) +} + +// 检测是否超过限制 +func (d *downloader) concurrencyCheck() error { + if d.cfg.ConcurrencyLimit != nil { + return d.cfg.ConcurrencyLimit.sub() + } + return nil +} +func (d *downloader) concurrencyFinish() { + if d.cfg.ConcurrencyLimit != nil { + d.cfg.ConcurrencyLimit.add() + } } // download performs the implementation of the object download across ranged GETs. func (d *downloader) download() (io.ReadCloser, error) { - d.ctx, d.cancel = context.WithCancel(d.ctx) + if err := d.concurrencyCheck(); err != nil { + return nil, err + } + d.ctx, d.cancel = context.WithCancelCause(d.ctx) - pos := d.params.Range.Start - maxPos := d.params.Range.Start + d.params.Range.Length - id := 0 - for pos < maxPos { - finalSize := int64(d.cfg.PartSize) - //check boundary - if pos+finalSize > maxPos { - finalSize = maxPos - pos - } - c := chunk{start: pos, size: finalSize, id: id} - d.chunks = append(d.chunks, c) - pos += finalSize - id++ + maxPart := int(d.params.Range.Length / int64(d.cfg.PartSize)) + if d.params.Range.Length%int64(d.cfg.PartSize) > 0 { + maxPart++ } - if len(d.chunks) < d.cfg.Concurrency { - d.cfg.Concurrency = len(d.chunks) + if maxPart < d.cfg.Concurrency { + d.cfg.Concurrency = maxPart } + log.Debugf("cfgConcurrency:%d", d.cfg.Concurrency) if d.cfg.Concurrency == 1 { + if d.cfg.ConcurrencyLimit != nil { + go func() { + <-d.ctx.Done() + d.concurrencyFinish() + }() + } resp, err := d.cfg.HttpClient(d.ctx, d.params) if err != nil { return nil, err @@ -143,61 +184,114 @@ func (d *downloader) download() (io.ReadCloser, error) { // workers d.chunkChannel = make(chan chunk, d.cfg.Concurrency) - for i := 0; i < d.cfg.Concurrency; i++ { - buf := NewBuf(d.ctx, d.cfg.PartSize, i) - d.bufs = append(d.bufs, buf) - go d.downloadPart() - } - // initial tasks - for i := 0; i < d.cfg.Concurrency; i++ { - d.sendChunkTask() - } + d.maxPart = maxPart + d.pos = d.params.Range.Start + d.maxPos = d.params.Range.Start + d.params.Range.Length + d.concurrency = d.cfg.Concurrency + d.sendChunkTask(true) - var rc io.ReadCloser = NewMultiReadCloser(d.chunks[0].buf, d.interrupt, d.finishBuf) + var rc io.ReadCloser = NewMultiReadCloser(d.bufs[0], d.interrupt, d.finishBuf) // Return error return rc, d.err } -func (d *downloader) sendChunkTask() *chunk { - ch := &d.chunks[d.nextChunk] - ch.buf = d.getBuf(d.nextChunk) - ch.buf.Reset(int(ch.size)) - d.chunkChannel <- *ch - d.nextChunk++ - return ch + +func (d *downloader) sendChunkTask(newConcurrency bool) error { + d.m.Lock() + defer d.m.Unlock() + isNewBuf := d.concurrency > 0 + if newConcurrency { + if d.concurrency <= 0 { + return nil + } + if d.nextChunk > 0 { // 第一个不检查，因为已经检查过了 + if err := d.concurrencyCheck(); err != nil { + return err + } + } + d.concurrency-- + go d.downloadPart() + } + + var buf *Buf + if isNewBuf { + buf = NewBuf(d.ctx, d.cfg.PartSize) + d.bufs = append(d.bufs, buf) + } else { + buf = d.getBuf(d.nextChunk) + } + + if d.pos < d.maxPos { + finalSize := int64(d.cfg.PartSize) + switch d.nextChunk { + case 0: + // 最小分片在前面有助视频播放？ + firstSize := d.params.Range.Length % finalSize + if firstSize > 0 { + minSize := finalSize / 2 + if firstSize < minSize { // 最小分片太小就调整到一半 + finalSize = minSize + } else { + finalSize = firstSize + } + } + case 1: + firstSize := d.params.Range.Length % finalSize + minSize := finalSize / 2 + if firstSize > 0 && firstSize < minSize { + finalSize += firstSize - minSize + } + } + buf.Reset(int(finalSize)) + ch := chunk{ + start: d.pos, + size: finalSize, + id: d.nextChunk, + buf: buf, + } + ch.newConcurrency = newConcurrency + d.pos += finalSize + d.nextChunk++ + d.chunkChannel <- ch + return nil + } + return nil } // when the final reader Close, we interrupt func (d *downloader) interrupt() error { - - d.cancel() if d.written != d.params.Range.Length { log.Debugf("Downloader interrupt before finish") if d.getErr() == nil { d.setErr(fmt.Errorf("interrupted")) } } + d.cancel(d.err) defer func() { close(d.chunkChannel) for _, buf := range d.bufs { buf.Close() } + if d.concurrency > 0 { + d.concurrency = -d.concurrency + } + log.Debugf("maxConcurrency:%d", d.cfg.Concurrency+d.concurrency) }() return d.err } func (d *downloader) getBuf(id int) (b *Buf) { - - return d.bufs[id%d.cfg.Concurrency] + return d.bufs[id%len(d.bufs)] } -func (d *downloader) finishBuf(id int) (isLast bool, buf *Buf) { - if id >= len(d.chunks)-1 { +func (d *downloader) finishBuf(id int) (isLast bool, nextBuf *Buf) { + id++ + if id >= d.maxPart { return true, nil } - if d.nextChunk > id+1 { - return false, d.getBuf(id + 1) - } - ch := d.sendChunkTask() - return false, ch.buf + + d.sendChunkTask(false) + + d.readingID = id + return false, d.getBuf(id) } // downloadPart is an individual goroutine worker reading from the ch channel @@ -212,58 +306,119 @@ func (d *downloader) downloadPart() { if d.getErr() != nil { // Drain the channel if there is an error, to prevent deadlocking // of download producer. - continue + break } - log.Debugf("downloadPart tried to get chunk") if err := d.downloadChunk(&c); err != nil { + if err == errCancelConcurrency { + break + } + if err == context.Canceled { + if e := context.Cause(d.ctx); e != nil { + err = e + } + } d.setErr(err) + d.cancel(err) } } + d.concurrencyFinish() } // downloadChunk downloads the chunk func (d *downloader) downloadChunk(ch *chunk) error { - log.Debugf("start new chunk %+v buffer_id =%d", ch, ch.id) + log.Debugf("start chunk_%d, %+v", ch.id, ch) + params := d.getParamsFromChunk(ch) var n int64 var err error - params := d.getParamsFromChunk(ch) - for retry := 0; retry <= d.partBodyMaxRetries; retry++ { + for retry := 0; retry <= d.cfg.PartBodyMaxRetries; retry++ { if d.getErr() != nil { - return d.getErr() + return nil } n, err = d.tryDownloadChunk(params, ch) if err == nil { + d.incrWritten(n) + log.Debugf("chunk_%d downloaded", ch.id) break } - // Check if the returned error is an errReadingBody. - // If err is errReadingBody this indicates that an error - // occurred while copying the http response body. + if d.getErr() != nil { + return nil + } + if utils.IsCanceled(d.ctx) { + return d.ctx.Err() + } + // Check if the returned error is an errNeedRetry. // If this occurs we unwrap the err to set the underlying error // and attempt any remaining retries. - if bodyErr, ok := err.(*errReadingBody); ok { - err = bodyErr.Unwrap() + if e, ok := err.(*errNeedRetry); ok { + err = e.Unwrap() + if n > 0 { + // 测试：下载时 断开 alist向云盘发起的下载连接 + // 校验：下载完后校验文件哈希值 一致 + d.incrWritten(n) + ch.start += n + ch.size -= n + params.Range.Start = ch.start + params.Range.Length = ch.size + } + log.Warnf("err chunk_%d, object part download error %s, retrying attempt %d. %v", + ch.id, params.URL, retry, err) + } else if err == errInfiniteRetry { + retry-- + continue } else { - return err + break } - - //ch.cur = 0 - - log.Debugf("object part body download interrupted %s, err, %v, retrying attempt %d", - params.URL, err, retry) } - d.incrWritten(n) - log.Debugf("down_%d downloaded chunk", ch.id) - //ch.buf.buffer.wg1.Wait() - //log.Debugf("down_%d downloaded chunk,wg wait passed", ch.id) return err } -func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { +var errCancelConcurrency = fmt.Errorf("cancel concurrency") +var errInfiniteRetry = fmt.Errorf("infinite retry") +func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { resp, err := d.cfg.HttpClient(d.ctx, params) if err != nil { - return 0, err + if resp == nil { + return 0, err + } + if ch.id == 0 { //第1个任务 有限的重试，超过重试就会结束请求 + switch resp.StatusCode { + default: + return 0, err + case http.StatusTooManyRequests: + case http.StatusBadGateway: + case http.StatusServiceUnavailable: + case http.StatusGatewayTimeout: + } + <-time.After(time.Millisecond * 200) + return 0, &errNeedRetry{err: fmt.Errorf("http request failure,status: %d", resp.StatusCode)} + } + + // 来到这 说明第1个分片下载 连接成功了 + // 后续分片下载出错都当超载处理 + log.Debugf("err chunk_%d, try downloading:%v", ch.id, err) + + d.m.Lock() + isCancelConcurrency := ch.newConcurrency + if d.concurrency > 0 { // 取消剩余的并发任务 + // 用于计算实际的并发数 + d.concurrency = -d.concurrency + isCancelConcurrency = true + } + if isCancelConcurrency { + d.concurrency-- + d.chunkChannel <- *ch + d.m.Unlock() + return 0, errCancelConcurrency + } + d.m.Unlock() + if ch.id != d.readingID { //正在被读取的优先重试 + d.m2.Lock() + defer d.m2.Unlock() + <-time.After(time.Millisecond * 200) + } + return 0, errInfiniteRetry } defer resp.Body.Close() //only check file size on the first task @@ -273,15 +428,15 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int return 0, err } } - + d.sendChunkTask(true) n, err := utils.CopyWithBuffer(ch.buf, resp.Body) if err != nil { - return n, &errReadingBody{err: err} + return n, &errNeedRetry{err: err} } if n != ch.size { err = fmt.Errorf("chunk download size incorrect, expected=%d, got=%d", ch.size, n) - return n, &errReadingBody{err: err} + return n, &errNeedRetry{err: err} } return n, nil @@ -297,7 +452,7 @@ func (d *downloader) getParamsFromChunk(ch *chunk) *HttpRequestParams { func (d *downloader) checkTotalBytes(resp *http.Response) error { var err error - var totalBytes int64 = math.MinInt64 + totalBytes := int64(-1) contentRange := resp.Header.Get("Content-Range") if len(contentRange) == 0 { // ContentRange is nil when the full file contents is provided, and @@ -329,8 +484,9 @@ func (d *downloader) checkTotalBytes(resp *http.Response) error { err = fmt.Errorf("expect file size=%d unmatch remote report size=%d, need refresh cache", d.params.Size, totalBytes) } if err != nil { - _ = d.interrupt() + // _ = d.interrupt() d.setErr(err) + d.cancel(err) } return err @@ -369,9 +525,7 @@ type chunk struct { buf *Buf id int - // Downloader takes range (start,length), but this chunk is requesting equal/sub range of it. - // To convert the writer to reader eventually, we need to write within the boundary - //boundary http_range.Range + newConcurrency bool } func DefaultHttpRequestFunc(ctx context.Context, params *HttpRequestParams) (*http.Response, error) { @@ -379,7 +533,7 @@ func DefaultHttpRequestFunc(ctx context.Context, params *HttpRequestParams) (*ht res, err := RequestHttp(ctx, "GET", header, params.URL) if err != nil { - return nil, err + return res, err } return res, nil } @@ -392,15 +546,15 @@ type HttpRequestParams struct { //total file size Size int64 } -type errReadingBody struct { +type errNeedRetry struct { err error } -func (e *errReadingBody) Error() string { - return fmt.Sprintf("failed to read part body: %v", e.err) +func (e *errNeedRetry) Error() string { + return e.err.Error() } -func (e *errReadingBody) Unwrap() error { +func (e *errNeedRetry) Unwrap() error { return e.err } @@ -438,9 +592,13 @@ func (mr MultiReadCloser) Read(p []byte) (n int, err error) { } mr.cfg.curBuf = next mr.cfg.rPos++ - //current.Close() return n, nil } + if err == context.Canceled { + if e := context.Cause(mr.cfg.curBuf.ctx); e != nil { + err = e + } + } return n, err } func (mr MultiReadCloser) Close() error { @@ -453,18 +611,16 @@ type Buf struct { ctx context.Context off int rw sync.Mutex - //notify chan struct{} } // NewBuf is a buffer that can have 1 read & 1 write at the same time. // when read is faster write, immediately feed data to read after written -func NewBuf(ctx context.Context, maxSize int, id int) *Buf { +func NewBuf(ctx context.Context, maxSize int) *Buf { d := make([]byte, 0, maxSize) return &Buf{ ctx: ctx, buffer: bytes.NewBuffer(d), size: maxSize, - //notify: make(chan struct{}), } } func (br *Buf) Reset(size int) { @@ -502,8 +658,6 @@ func (br *Buf) Read(p []byte) (n int, err error) { select { case <-br.ctx.Done(): return 0, br.ctx.Err() - //case <-br.notify: - // return 0, nil case <-time.After(time.Millisecond * 200): return 0, nil } @@ -516,13 +670,9 @@ func (br *Buf) Write(p []byte) (n int, err error) { br.rw.Lock() defer br.rw.Unlock() n, err = br.buffer.Write(p) - select { - //case br.notify <- struct{}{}: - default: - } return } func (br *Buf) Close() { - //close(br.notify) + br.buffer.Reset() } diff --git a/internal/net/serve.go b/internal/net/serve.go index e85f61a8950..6216cd217e9 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -52,7 +52,8 @@ import ( // // If the caller has set w's ETag header formatted per RFC 7232, section 2.3, // ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range. -func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReaderFunc model.RangeReaderFunc) { +func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReadCloser model.RangeReadCloserIF) { + defer RangeReadCloser.Close() setLastModified(w, modTime) done, rangeReq := checkPreconditions(w, r, modTime) if done { @@ -110,11 +111,19 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time // or unknown file size, ignore the range request. ranges = nil } + + // 使用请求的Context + // 不然从sendContent读不到数据，即使请求断开CopyBuffer也会一直堵塞 + ctx := r.Context() switch { case len(ranges) == 0: - reader, err := RangeReaderFunc(context.Background(), http_range.Range{Length: -1}) + reader, err := RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + code = http.StatusRequestedRangeNotSatisfiable + if err == ErrExceedMaxConcurrency { + code = http.StatusTooManyRequests + } + http.Error(w, err.Error(), code) return } sendContent = reader @@ -131,9 +140,13 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time // does not request multiple parts might not support // multipart responses." ra := ranges[0] - sendContent, err = RangeReaderFunc(context.Background(), ra) + sendContent, err = RangeReadCloser.RangeRead(ctx, ra) if err != nil { - http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) + code = http.StatusRequestedRangeNotSatisfiable + if err == ErrExceedMaxConcurrency { + code = http.StatusTooManyRequests + } + http.Error(w, err.Error(), code) return } sendSize = ra.Length @@ -158,7 +171,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time pw.CloseWithError(err) return } - reader, err := RangeReaderFunc(context.Background(), ra) + reader, err := RangeReadCloser.RangeRead(ctx, ra) if err != nil { pw.CloseWithError(err) return @@ -167,14 +180,12 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time pw.CloseWithError(err) return } - //defer reader.Close() } mw.Close() pw.Close() }() } - //defer sendContent.Close() w.Header().Set("Accept-Ranges", "bytes") if w.Header().Get("Content-Encoding") == "" { @@ -190,7 +201,11 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time if written != sendSize { log.Warnf("Maybe size incorrect or reader not giving correct/full data, or connection closed before finish. written bytes: %d ,sendSize:%d, ", written, sendSize) } - http.Error(w, err.Error(), http.StatusInternalServerError) + code = http.StatusInternalServerError + if err == ErrExceedMaxConcurrency { + code = http.StatusTooManyRequests + } + http.Error(w, err.Error(), code) } } } @@ -239,7 +254,7 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea _ = res.Body.Close() msg := string(all) log.Debugln(msg) - return nil, fmt.Errorf("http request [%s] failure,status: %d response:%s", URL, res.StatusCode, msg) + return res, fmt.Errorf("http request [%s] failure,status: %d response:%s", URL, res.StatusCode, msg) } return res, nil } diff --git a/internal/net/util.go b/internal/net/util.go index 44201859487..45301dde9f2 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -2,7 +2,6 @@ package net import ( "fmt" - "github.com/alist-org/alist/v3/pkg/utils" "io" "math" "mime/multipart" @@ -11,6 +10,8 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/http_range" log "github.com/sirupsen/logrus" ) diff --git a/internal/offline_download/transmission/client.go b/internal/offline_download/transmission/client.go index 8049afd6d35..ae136009875 100644 --- a/internal/offline_download/transmission/client.go +++ b/internal/offline_download/transmission/client.go @@ -5,7 +5,6 @@ import ( "context" "encoding/base64" "fmt" - "io" "net/http" "net/url" "strconv" @@ -15,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/hekmon/transmissionrpc/v3" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -92,7 +92,7 @@ func (t *Transmission) AddURL(args *tool.AddUrlArgs) (string, error) { buffer := new(bytes.Buffer) encoder := base64.NewEncoder(base64.StdEncoding, buffer) // Stream file to the encoder - if _, err = io.Copy(encoder, resp.Body); err != nil { + if _, err = utils.CopyWithBuffer(encoder, resp.Body); err != nil { return "", errors.Wrap(err, "can't copy file content into the base64 encoder") } // Flush last bytes diff --git a/internal/stream/stream.go b/internal/stream/stream.go index b19eb07753d..0915ee6ba1e 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -122,7 +122,8 @@ const InMemoryBufMaxSizeBytes = InMemoryBufMaxSize * 1024 * 1024 // also support a peeking RangeRead at very start, but won't buffer more than 10MB data in memory func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if httpRange.Length == -1 { - httpRange.Length = f.GetSize() + // 参考 internal/net/request.go + httpRange.Length = f.GetSize() - httpRange.Start } if f.peekBuff != nil && httpRange.Start < int64(f.peekBuff.Len()) && httpRange.Start+httpRange.Length-1 < int64(f.peekBuff.Len()) { return io.NewSectionReader(f.peekBuff, httpRange.Start, httpRange.Length), nil @@ -210,7 +211,7 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) // RangeRead is not thread-safe, pls use it in single thread only. func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if httpRange.Length == -1 { - httpRange.Length = ss.GetSize() + httpRange.Length = ss.GetSize() - httpRange.Start } if ss.mFile != nil { return io.NewSectionReader(ss.mFile, httpRange.Start, httpRange.Length), nil diff --git a/internal/stream/util.go b/internal/stream/util.go index 7d2b7ef7509..16854c38c80 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -6,7 +6,6 @@ import ( "io" "net/http" - "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/pkg/http_range" @@ -17,7 +16,6 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl if len(link.URL) == 0 { return nil, fmt.Errorf("can't create RangeReadCloser since URL is empty in link") } - //remoteClosers := utils.EmptyClosers() rangeReaderFunc := func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { if link.Concurrency != 0 || link.PartSize != 0 { header := net.ProcessHeader(http.Header{}, link.Header) @@ -32,37 +30,29 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl HeaderRef: header, } rc, err := down.Download(ctx, req) - if err != nil { - return nil, errs.NewErr(err, "GetReadCloserFromLink failed") - } - return rc, nil + return rc, err } - if len(link.URL) > 0 { - response, err := RequestRangedHttp(ctx, link, r.Start, r.Length) - if err != nil { - if response == nil { - return nil, fmt.Errorf("http request failure, err:%s", err) - } - return nil, fmt.Errorf("http request failure,status: %d err:%s", response.StatusCode, err) - } - if r.Start == 0 && (r.Length == -1 || r.Length == size) || response.StatusCode == http.StatusPartialContent || - checkContentRange(&response.Header, r.Start) { - return response.Body, nil - } else if response.StatusCode == http.StatusOK { - log.Warnf("remote http server not supporting range request, expect low perfromace!") - readCloser, err := net.GetRangedHttpReader(response.Body, r.Start, r.Length) - if err != nil { - return nil, err - } - return readCloser, nil - + response, err := RequestRangedHttp(ctx, link, r.Start, r.Length) + if err != nil { + if response == nil { + return nil, fmt.Errorf("http request failure, err:%s", err) } - + return nil, err + } + if r.Start == 0 && (r.Length == -1 || r.Length == size) || response.StatusCode == http.StatusPartialContent || + checkContentRange(&response.Header, r.Start) { return response.Body, nil + } else if response.StatusCode == http.StatusOK { + log.Warnf("remote http server not supporting range request, expect low perfromace!") + readCloser, err := net.GetRangedHttpReader(response.Body, r.Start, r.Length) + if err != nil { + return nil, err + } + return readCloser, nil } - return nil, errs.NotSupport + return response.Body, nil } resultRangeReadCloser := model.RangeReadCloser{RangeReader: rangeReaderFunc} return &resultRangeReadCloser, nil diff --git a/server/common/proxy.go b/server/common/proxy.go index 10923613ede..2d828efdfcc 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -27,16 +27,11 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. return nil } else if link.RangeReadCloser != nil { attachFileName(w, file) - net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), link.RangeReadCloser.RangeRead) - defer func() { - _ = link.RangeReadCloser.Close() - }() + net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), link.RangeReadCloser) return nil } else if link.Concurrency != 0 || link.PartSize != 0 { attachFileName(w, file) size := file.GetSize() - //var finalClosers model.Closers - finalClosers := utils.EmptyClosers() header := net.ProcessHeader(r.Header, link.Header) rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { down := net.NewDownloader(func(d *net.Downloader) { @@ -50,16 +45,14 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. HeaderRef: header, } rc, err := down.Download(ctx, req) - finalClosers.Add(rc) return rc, err } - net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), rangeReader) - defer finalClosers.Close() + net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &model.RangeReadCloser{RangeReader: rangeReader}) return nil } else { //transparent proxy header := net.ProcessHeader(r.Header, link.Header) - res, err := net.RequestHttp(context.Background(), r.Method, header, link.URL) + res, err := net.RequestHttp(r.Context(), r.Method, header, link.URL) if err != nil { return err } @@ -72,7 +65,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. if r.Method == http.MethodHead { return nil } - _, err = io.Copy(w, res.Body) + _, err = utils.CopyWithBuffer(w, res.Body) if err != nil { return err } diff --git a/server/handles/archive.go b/server/handles/archive.go index 29dbf3c2d34..bad99bace83 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -281,10 +281,11 @@ func ArchiveDown(c *gin.Context) { link, _, err := fs.ArchiveDriverExtract(c, archiveRawPath, model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ LinkArgs: model.LinkArgs{ - IP: c.ClientIP(), - Header: c.Request.Header, - Type: c.Query("type"), - HttpReq: c.Request, + IP: c.ClientIP(), + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + Redirect: true, }, Password: password, }, diff --git a/server/handles/down.go b/server/handles/down.go index f01c9d6683b..b2f9a21bc63 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -31,10 +31,11 @@ func Down(c *gin.Context) { return } else { link, _, err := fs.Link(c, rawPath, model.LinkArgs{ - IP: c.ClientIP(), - Header: c.Request.Header, - Type: c.Query("type"), - HttpReq: c.Request, + IP: c.ClientIP(), + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + Redirect: true, }) if err != nil { common.ErrorResp(c, err, 500) diff --git a/server/s3/backend.go b/server/s3/backend.go index e0cfd9676b0..bca45008cde 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -6,13 +6,14 @@ import ( "context" "encoding/hex" "fmt" - "github.com/pkg/errors" "io" "path" "strings" "sync" "time" + "github.com/pkg/errors" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" @@ -173,15 +174,27 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string if link.RangeReadCloser == nil && link.MFile == nil && len(link.URL) == 0 { return nil, fmt.Errorf("the remote storage driver need to be enhanced to support s3") } - remoteFileSize := file.GetSize() - remoteClosers := utils.EmptyClosers() - rangeReaderFunc := func(ctx context.Context, start, length int64) (io.ReadCloser, error) { + + var rdr io.ReadCloser + length := int64(-1) + start := int64(0) + if rnge != nil { + start, length = rnge.Start, rnge.Length + } + // 参考 server/common/proxy.go + if link.MFile != nil { + _, err := link.MFile.Seek(start, io.SeekStart) + if err != nil { + return nil, err + } + rdr = link.MFile + } else { + remoteFileSize := file.GetSize() if length >= 0 && start+length >= remoteFileSize { length = -1 } rrc := link.RangeReadCloser if len(link.URL) > 0 { - rangedRemoteLink := &model.Link{ URL: link.URL, Header: link.Header, @@ -194,35 +207,12 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string } if rrc != nil { remoteReader, err := rrc.RangeRead(ctx, http_range.Range{Start: start, Length: length}) - remoteClosers.AddClosers(rrc.GetClosers()) - if err != nil { - return nil, err - } - return remoteReader, nil - } - if link.MFile != nil { - _, err := link.MFile.Seek(start, io.SeekStart) if err != nil { return nil, err } - //remoteClosers.Add(remoteLink.MFile) - //keep reuse same MFile and close at last. - remoteClosers.Add(link.MFile) - return io.NopCloser(link.MFile), nil - } - return nil, errs.NotSupport - } - - var rdr io.ReadCloser - if rnge != nil { - rdr, err = rangeReaderFunc(ctx, rnge.Start, rnge.Length) - if err != nil { - return nil, err - } - } else { - rdr, err = rangeReaderFunc(ctx, 0, -1) - if err != nil { - return nil, err + rdr = utils.ReadCloser{Reader: remoteReader, Closer: rrc} + } else { + return nil, errs.NotSupport } } diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index b84e65b06b7..6585056bbde 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -263,7 +263,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") http.Redirect(w, r, u, http.StatusFound) } else { - link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{IP: utils.ClientIP(r), Header: r.Header, HttpReq: r}) + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{IP: utils.ClientIP(r), Header: r.Header, HttpReq: r, Redirect: true}) if err != nil { return http.StatusInternalServerError, err } From 5c5d8378e5650fe6b81807956d450f700ba7acae Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Mon, 27 Jan 2025 20:08:56 +0800 Subject: [PATCH 422/659] fix(archive): unable to preview (#7843) * fix(archive): unrecognition zip * feat(archive): add tree for zip meta * fix bug * refactor(archive): meta cache time use Link Expiration first * feat(archive): return sort policy in meta (#2) * refactor * perf(archive): reduce new network requests --------- Co-authored-by: KirCute_ECT <951206789@qq.com> --- internal/archive/archives/archives.go | 31 ++++-- internal/archive/archives/utils.go | 12 ++- internal/archive/zip/zip.go | 102 +++++++++++++++--- internal/model/archive.go | 4 + internal/op/archive.go | 43 ++++++-- internal/stream/stream.go | 144 +++++++++++++++++++++----- pkg/utils/io.go | 6 ++ server/handles/archive.go | 18 ++-- 8 files changed, 287 insertions(+), 73 deletions(-) diff --git a/internal/archive/archives/archives.go b/internal/archive/archives/archives.go index b70ba95bc86..6d48624fa2e 100644 --- a/internal/archive/archives/archives.go +++ b/internal/archive/archives/archives.go @@ -1,42 +1,53 @@ package archives import ( - "github.com/alist-org/alist/v3/internal/archive/tool" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/pkg/utils" "io" "io/fs" "os" stdpath "path" "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" ) type Archives struct { } -func (_ *Archives) AcceptedExtensions() []string { +func (*Archives) AcceptedExtensions() []string { return []string{ ".br", ".bz2", ".gz", ".lz4", ".lz", ".sz", ".s2", ".xz", ".zz", ".zst", ".tar", ".rar", ".7z", } } -func (_ *Archives) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { +func (*Archives) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { fsys, err := getFs(ss, args) if err != nil { return nil, err } - _, err = fsys.ReadDir(".") + files, err := fsys.ReadDir(".") if err != nil { return nil, filterPassword(err) } + + tree := make([]model.ObjTree, 0, len(files)) + for _, file := range files { + info, err := file.Info() + if err != nil { + continue + } + tree = append(tree, &model.ObjectTree{Object: *toModelObj(info)}) + } return &model.ArchiveMetaInfo{ Comment: "", Encrypted: false, + Tree: tree, }, nil } -func (_ *Archives) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { +func (*Archives) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { fsys, err := getFs(ss, args.ArchiveArgs) if err != nil { return nil, err @@ -58,7 +69,7 @@ func (_ *Archives) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) }) } -func (_ *Archives) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { +func (*Archives) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { fsys, err := getFs(ss, args.ArchiveArgs) if err != nil { return nil, 0, err @@ -74,7 +85,7 @@ func (_ *Archives) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArg return file, stat.Size(), nil } -func (_ *Archives) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { +func (*Archives) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { fsys, err := getFs(ss, args.ArchiveArgs) if err != nil { return err diff --git a/internal/archive/archives/utils.go b/internal/archive/archives/utils.go index b72e6bc6a00..fdae10091f6 100644 --- a/internal/archive/archives/utils.go +++ b/internal/archive/archives/utils.go @@ -1,15 +1,16 @@ package archives import ( - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/mholt/archives" "io" fs2 "io/fs" "os" stdpath "path" "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/mholt/archives" ) func getFs(ss *stream.SeekableStream, args model.ArchiveArgs) (*archives.ArchiveFS, error) { @@ -17,6 +18,9 @@ func getFs(ss *stream.SeekableStream, args model.ArchiveArgs) (*archives.Archive if err != nil { return nil, err } + if r, ok := reader.(*stream.RangeReadReadAtSeeker); ok { + r.InitHeadCache() + } format, _, err := archives.Identify(ss.Ctx, ss.GetName(), reader) if err != nil { return nil, errs.UnknownArchiveFormat diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go index ccb70e65996..e5285518b05 100644 --- a/internal/archive/zip/zip.go +++ b/internal/archive/zip/zip.go @@ -1,25 +1,26 @@ package zip import ( + "io" + "os" + stdpath "path" + "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" "github.com/yeka/zip" - "io" - "os" - stdpath "path" - "strings" ) type Zip struct { } -func (_ *Zip) AcceptedExtensions() []string { +func (*Zip) AcceptedExtensions() []string { return []string{".zip"} } -func (_ *Zip) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { +func (*Zip) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { reader, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return nil, err @@ -29,19 +30,81 @@ func (_ *Zip) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model. return nil, err } encrypted := false + dirMap := make(map[string]*model.ObjectTree) + dirMap["."] = &model.ObjectTree{} for _, file := range zipReader.File { if file.IsEncrypted() { encrypted = true break } + + name := strings.TrimPrefix(decodeName(file.Name), "/") + var dir string + var dirObj *model.ObjectTree + isNewFolder := false + if !file.FileInfo().IsDir() { + // 先将 文件 添加到 所在的文件夹 + dir = stdpath.Dir(name) + dirObj = dirMap[dir] + if dirObj == nil { + isNewFolder = true + dirObj = &model.ObjectTree{} + dirObj.IsFolder = true + dirObj.Name = stdpath.Base(dir) + dirObj.Modified = file.ModTime() + dirMap[dir] = dirObj + } + dirObj.Children = append( + dirObj.Children, &model.ObjectTree{ + Object: *toModelObj(file.FileInfo()), + }, + ) + } else { + dir = strings.TrimSuffix(name, "/") + dirObj = dirMap[dir] + if dirObj == nil { + isNewFolder = true + dirObj = &model.ObjectTree{} + dirMap[dir] = dirObj + } + dirObj.IsFolder = true + dirObj.Name = stdpath.Base(dir) + dirObj.Modified = file.ModTime() + } + if isNewFolder { + // 将 文件夹 添加到 父文件夹 + dir = stdpath.Dir(dir) + pDirObj := dirMap[dir] + if pDirObj != nil { + pDirObj.Children = append(pDirObj.Children, dirObj) + continue + } + + for { + // 考虑压缩包仅记录文件的路径，不记录文件夹 + pDirObj = &model.ObjectTree{} + pDirObj.IsFolder = true + pDirObj.Name = stdpath.Base(dir) + pDirObj.Modified = file.ModTime() + dirMap[dir] = pDirObj + pDirObj.Children = append(pDirObj.Children, dirObj) + dir = stdpath.Dir(dir) + if dirMap[dir] != nil { + break + } + dirObj = pDirObj + } + } } + return &model.ArchiveMetaInfo{ Comment: zipReader.Comment, Encrypted: encrypted, + Tree: dirMap["."].GetChildren(), }, nil } -func (_ *Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { +func (*Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { reader, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return nil, err @@ -53,6 +116,7 @@ func (_ *Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]mo if args.InnerPath == "/" { ret := make([]model.Obj, 0) passVerified := false + var dir *model.Object for _, file := range zipReader.File { if !passVerified && file.IsEncrypted() { file.SetPassword(args.Password) @@ -63,12 +127,24 @@ func (_ *Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]mo _ = rc.Close() passVerified = true } - name := decodeName(file.Name) - if strings.Contains(strings.TrimSuffix(name, "/"), "/") { + name := strings.TrimSuffix(decodeName(file.Name), "/") + if strings.Contains(name, "/") { + // 有些压缩包不压缩第一个文件夹 + strs := strings.Split(name, "/") + if dir == nil && len(strs) == 2 { + dir = &model.Object{ + Name: strs[0], + Modified: ss.ModTime(), + IsFolder: true, + } + } continue } ret = append(ret, toModelObj(file.FileInfo())) } + if len(ret) == 0 && dir != nil { + ret = append(ret, dir) + } return ret, nil } else { innerPath := strings.TrimPrefix(args.InnerPath, "/") + "/" @@ -76,13 +152,11 @@ func (_ *Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]mo exist := false for _, file := range zipReader.File { name := decodeName(file.Name) - if name == innerPath { - exist = true - } dir := stdpath.Dir(strings.TrimSuffix(name, "/")) + "/" if dir != innerPath { continue } + exist = true ret = append(ret, toModelObj(file.FileInfo())) } if !exist { @@ -92,7 +166,7 @@ func (_ *Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]mo } } -func (_ *Zip) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { +func (*Zip) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { reader, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return nil, 0, err @@ -117,7 +191,7 @@ func (_ *Zip) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (i return nil, 0, errs.ObjectNotFound } -func (_ *Zip) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { +func (*Zip) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { reader, err := stream.NewReadAtSeeker(ss, 0) if err != nil { return err diff --git a/internal/model/archive.go b/internal/model/archive.go index 03ac7c360a5..01b83691e3f 100644 --- a/internal/model/archive.go +++ b/internal/model/archive.go @@ -1,5 +1,7 @@ package model +import "time" + type ObjTree interface { Obj GetChildren() []ObjTree @@ -45,5 +47,7 @@ func (m *ArchiveMetaInfo) GetTree() []ObjTree { type ArchiveMetaProvider struct { ArchiveMeta + *Sort DriverProviding bool + Expiration *time.Duration } diff --git a/internal/op/archive.go b/internal/op/archive.go index 6a9fa084778..a241838c9c4 100644 --- a/internal/op/archive.go +++ b/internal/op/archive.go @@ -3,13 +3,14 @@ package op import ( "context" stderrors "errors" - "github.com/alist-org/alist/v3/internal/archive/tool" - "github.com/alist-org/alist/v3/internal/stream" "io" stdpath "path" "strings" "time" + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/Xhofe/go-cache" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -40,8 +41,8 @@ func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg if err != nil { return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err) } - if !storage.Config().NoCache { - archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + if m.Expiration != nil { + archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](*m.Expiration)) } return m, nil } @@ -82,7 +83,15 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg } meta, err := storageAr.GetArchiveMeta(ctx, obj, args.ArchiveArgs) if !errors.Is(err, errs.NotImplement) { - return obj, &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: true}, err + archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: true} + if meta.GetTree() != nil { + archiveMetaProvider.Sort = &storage.GetStorage().Sort + } + if !storage.Config().NoCache { + Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) + archiveMetaProvider.Expiration = &Expiration + } + return obj, archiveMetaProvider, err } } obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) @@ -95,7 +104,21 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg } }() meta, err := t.GetMeta(ss, args.ArchiveArgs) - return obj, &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: false}, err + if err != nil { + return nil, nil, err + } + archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: false} + if meta.GetTree() != nil { + archiveMetaProvider.Sort = &storage.GetStorage().Sort + } + if !storage.Config().NoCache { + Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) + archiveMetaProvider.Expiration = &Expiration + } else if ss.Link.MFile == nil { + // alias、crypt 驱动 + archiveMetaProvider.Expiration = ss.Link.Expiration + } + return obj, archiveMetaProvider, err } var archiveListCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) @@ -113,10 +136,10 @@ func ListArchive(ctx context.Context, storage driver.Driver, path string, args m log.Debugf("use cache when list archive [%s]%s", path, args.InnerPath) return files, nil } - if meta, ok := archiveMetaCache.Get(metaKey); ok { - log.Debugf("use meta cache when list archive [%s]%s", path, args.InnerPath) - return getChildrenFromArchiveMeta(meta, args.InnerPath) - } + // if meta, ok := archiveMetaCache.Get(metaKey); ok { + // log.Debugf("use meta cache when list archive [%s]%s", path, args.InnerPath) + // return getChildrenFromArchiveMeta(meta, args.InnerPath) + // } } objs, err, _ := archiveListG.Do(key, func() ([]model.Obj, error) { obj, files, err := listArchive(ctx, storage, path, args) diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 0915ee6ba1e..1962fb46745 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -13,6 +13,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/sirupsen/logrus" ) type FileStream struct { @@ -189,6 +190,7 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) if ss.Link.RangeReadCloser != nil { ss.rangeReadCloser = ss.Link.RangeReadCloser + ss.Add(ss.rangeReadCloser) return &ss, nil } if len(ss.Link.URL) > 0 { @@ -197,6 +199,7 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) return nil, err } ss.rangeReadCloser = rrc + ss.Add(rrc) return &ss, nil } } @@ -248,8 +251,6 @@ func (ss *SeekableStream) Read(p []byte) (n int, err error) { return 0, nil } ss.Reader = io.NopCloser(rc) - ss.Closers.Add(rc) - } return ss.Reader.Read(p) } @@ -337,10 +338,62 @@ type RangeReadReadAtSeeker struct { ss *SeekableStream masterOff int64 readers []*readerCur + *headCache +} + +type headCache struct { + *readerCur + bufs [][]byte +} + +func (c *headCache) read(p []byte) (n int, err error) { + pL := len(p) + logrus.Debugf("headCache read_%d", pL) + if c.cur < int64(pL) { + bufL := int64(pL) - c.cur + buf := make([]byte, bufL) + lr := io.LimitReader(c.reader, bufL) + off := 0 + for c.cur < int64(pL) { + n, err = lr.Read(buf[off:]) + off += n + c.cur += int64(n) + if err == io.EOF && n == int(bufL) { + err = nil + } + if err != nil { + break + } + } + c.bufs = append(c.bufs, buf) + } + n = 0 + if c.cur >= int64(pL) { + for i := 0; n < pL; i++ { + buf := c.bufs[i] + r := len(buf) + if n+r > pL { + r = pL - n + } + n += copy(p[n:], buf[:r]) + } + } + return +} +func (r *headCache) close() error { + for i := range r.bufs { + r.bufs[i] = nil + } + r.bufs = nil + return nil } -type FileReadAtSeeker struct { - ss *SeekableStream +func (r *RangeReadReadAtSeeker) InitHeadCache() { + if r.ss.Link.MFile == nil && r.masterOff == 0 { + reader := r.readers[0] + r.readers = r.readers[1:] + r.headCache = &headCache{readerCur: reader} + } } func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (SStreamReadAtSeeker, error) { @@ -351,27 +404,23 @@ func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (SStr } return &FileReadAtSeeker{ss: ss}, nil } - var r io.Reader - var err error + r := &RangeReadReadAtSeeker{ + ss: ss, + masterOff: offset, + } if offset != 0 || utils.IsBool(forceRange...) { if offset < 0 || offset > ss.GetSize() { return nil, errors.New("offset out of range") } - r, err = ss.RangeRead(http_range.Range{Start: offset, Length: -1}) + _, err := r.getReaderAtOffset(offset) if err != nil { return nil, err } - if rc, ok := r.(io.Closer); ok { - ss.Closers.Add(rc) - } } else { - r = ss + rc := &readerCur{reader: ss, cur: offset} + r.readers = append(r.readers, rc) } - return &RangeReadReadAtSeeker{ - ss: ss, - masterOff: offset, - readers: []*readerCur{{reader: r, cur: offset}}, - }, nil + return r, nil } func (r *RangeReadReadAtSeeker) GetRawStream() *SeekableStream { @@ -379,38 +428,71 @@ func (r *RangeReadReadAtSeeker) GetRawStream() *SeekableStream { } func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (*readerCur, error) { + var rc *readerCur for _, reader := range r.readers { + if reader.cur == -1 { + continue + } if reader.cur == off { + // logrus.Debugf("getReaderAtOffset match_%d", off) return reader, nil } + if reader.cur > 0 && off >= reader.cur && (rc == nil || reader.cur < rc.cur) { + rc = reader + } + } + if rc != nil && off-rc.cur <= utils.MB { + n, err := utils.CopyWithBufferN(utils.NullWriter{}, rc.reader, off-rc.cur) + rc.cur += n + if err == io.EOF && rc.cur == off { + err = nil + } + if err == nil { + logrus.Debugf("getReaderAtOffset old_%d", off) + return rc, nil + } + rc.cur = -1 } - reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: -1}) + logrus.Debugf("getReaderAtOffset new_%d", off) + + // Range请求不能超过文件大小，有些云盘处理不了就会返回整个文件 + reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: r.ss.GetSize() - off}) if err != nil { return nil, err } - if c, ok := reader.(io.Closer); ok { - r.ss.Closers.Add(c) - } - rc := &readerCur{reader: reader, cur: off} + rc = &readerCur{reader: reader, cur: off} r.readers = append(r.readers, rc) return rc, nil } func (r *RangeReadReadAtSeeker) ReadAt(p []byte, off int64) (int, error) { + if off == 0 && r.headCache != nil { + return r.headCache.read(p) + } rc, err := r.getReaderAtOffset(off) if err != nil { return 0, err } - num := 0 + n, num := 0, 0 for num < len(p) { - n, err := rc.reader.Read(p[num:]) + n, err = rc.reader.Read(p[num:]) rc.cur += int64(n) num += n - if err != nil { - return num, err + if err == nil { + continue + } + if err == io.EOF { + // io.EOF是reader读取完了 + rc.cur = -1 + // yeka/zip包 没有处理EOF，我们要兼容 + // https://github.com/yeka/zip/blob/03d6312748a9d6e0bc0c9a7275385c09f06d9c14/reader.go#L433 + if num == len(p) { + err = nil + } } + break } - return num, nil + return num, err } func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) { @@ -437,6 +519,9 @@ func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) { } func (r *RangeReadReadAtSeeker) Read(p []byte) (n int, err error) { + if r.masterOff == 0 && r.headCache != nil { + return r.headCache.read(p) + } rc, err := r.getReaderAtOffset(r.masterOff) if err != nil { return 0, err @@ -448,9 +533,16 @@ func (r *RangeReadReadAtSeeker) Read(p []byte) (n int, err error) { } func (r *RangeReadReadAtSeeker) Close() error { + if r.headCache != nil { + r.headCache.close() + } return r.ss.Close() } +type FileReadAtSeeker struct { + ss *SeekableStream +} + func (f *FileReadAtSeeker) GetRawStream() *SeekableStream { return f.ss } diff --git a/pkg/utils/io.go b/pkg/utils/io.go index e06fb235b8b..c314307d51d 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -233,3 +233,9 @@ func CopyWithBufferN(dst io.Writer, src io.Reader, n int64) (written int64, err } return } + +type NullWriter struct{} + +func (NullWriter) Write(p []byte) (n int, err error) { + return len(p), nil +} diff --git a/server/handles/archive.go b/server/handles/archive.go index bad99bace83..6ff13641b44 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -2,6 +2,10 @@ package handles import ( "fmt" + "net/url" + stdpath "path" + "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/errs" @@ -15,9 +19,6 @@ import ( "github.com/gin-gonic/gin" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - "mime" - stdpath "path" - "strings" ) type ArchiveMetaReq struct { @@ -31,6 +32,7 @@ type ArchiveMetaResp struct { Comment string `json:"comment"` IsEncrypted bool `json:"encrypted"` Content []ArchiveContentResp `json:"content"` + Sort *model.Sort `json:"sort,omitempty"` RawURL string `json:"raw_url"` Sign string `json:"sign"` } @@ -128,6 +130,7 @@ func FsArchiveMeta(c *gin.Context) { Comment: ret.GetComment(), IsEncrypted: ret.IsEncrypted(), Content: toContentResp(ret.GetTree()), + Sort: ret.Sort, RawURL: fmt.Sprintf("%s%s%s", common.GetApiUrl(c.Request), api, utils.EncodePath(reqPath, true)), Sign: s, }) @@ -361,14 +364,11 @@ func ArchiveInternalExtract(c *gin.Context) { "Referrer-Policy": "no-referrer", "Cache-Control": "max-age=0, no-cache, no-store, must-revalidate", } - if c.Query("attachment") == "true" { - filename := stdpath.Base(innerPath) - headers["Content-Disposition"] = fmt.Sprintf("attachment; filename=\"%s\"", filename) - } + filename := stdpath.Base(innerPath) + headers["Content-Disposition"] = fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, filename, url.PathEscape(filename)) contentType := c.Request.Header.Get("Content-Type") if contentType == "" { - fileExt := stdpath.Ext(innerPath) - contentType = mime.TypeByExtension(fileExt) + contentType = utils.GetMimeType(filename) } c.DataFromReader(200, size, contentType, rc, headers) } From 0d4c63e9ff6a4d542c5cee1d5ca56cf9f6102276 Mon Sep 17 00:00:00 2001 From: Jealous Date: Mon, 27 Jan 2025 20:09:17 +0800 Subject: [PATCH 423/659] feat(fs): display the existing filename in error message (#7877) --- server/handles/fsmanage.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 9349e7e275d..c527464e2e4 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -90,7 +90,7 @@ func FsMove(c *gin.Context) { if !req.Overwrite { for _, name := range req.Names { if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { - common.ErrorStrResp(c, "file exists", 403) + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } } @@ -133,7 +133,7 @@ func FsCopy(c *gin.Context) { if !req.Overwrite { for _, name := range req.Names { if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { - common.ErrorStrResp(c, "file exists", 403) + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } } @@ -180,7 +180,7 @@ func FsRename(c *gin.Context) { dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) if dstPath != reqPath { if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { - common.ErrorStrResp(c, "file exists", 403) + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", req.Name), 403) return } } From cafdb4d407c9d23663c25c39e424090eeffa5fa9 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Mon, 27 Jan 2025 20:11:21 +0800 Subject: [PATCH 424/659] fix(139): correct path handling in groupGetFiles (#7850 closes #7848,#7603) * fix(139): correct path handling in groupGetFiles * perf(139): reduce the number of requests in groupGetFiles * refactor(139): check authorization expiration (#10) * refactor(139): check authorization expiration * fix bug * chore(139): update api version to 7.14.0 --------- Co-authored-by: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> --- drivers/139/driver.go | 53 ++++++++++++++++++++------------------ drivers/139/util.go | 60 +++++++++++++++++++++++++++++-------------- 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index ebb30e25d19..cf64a8fdb7d 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -2,13 +2,11 @@ package _139 import ( "context" - "encoding/base64" "fmt" "io" "net/http" "path" "strconv" - "strings" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -42,7 +40,11 @@ func (d *Yun139) Init(ctx context.Context) error { if d.Authorization == "" { return fmt.Errorf("authorization is empty") } - d.cron = cron.NewCron(time.Hour * 24 * 7) + err := d.refreshToken() + if err != nil { + return err + } + d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err := d.refreshToken() if err != nil { @@ -67,28 +69,29 @@ func (d *Yun139) Init(ctx context.Context) error { default: return errs.NotImplement } - if d.ref != nil { - return nil - } - decode, err := base64.StdEncoding.DecodeString(d.Authorization) - if err != nil { - return err - } - decodeStr := string(decode) - splits := strings.Split(decodeStr, ":") - if len(splits) < 2 { - return fmt.Errorf("authorization is invalid, splits < 2") - } - d.Account = splits[1] - _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ - "qryUserExternInfoReq": base.Json{ - "commonAccountInfo": base.Json{ - "account": d.getAccount(), - "accountType": 1, - }, - }, - }, nil) - return err + // if d.ref != nil { + // return nil + // } + // decode, err := base64.StdEncoding.DecodeString(d.Authorization) + // if err != nil { + // return err + // } + // decodeStr := string(decode) + // splits := strings.Split(decodeStr, ":") + // if len(splits) < 2 { + // return fmt.Errorf("authorization is invalid, splits < 2") + // } + // d.Account = splits[1] + // _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ + // "qryUserExternInfoReq": base.Json{ + // "commonAccountInfo": base.Json{ + // "account": d.getAccount(), + // "accountType": 1, + // }, + // }, + // }, nil) + // return err + return nil } func (d *Yun139) InitReference(storage driver.Driver) error { diff --git a/drivers/139/util.go b/drivers/139/util.go index 2dade2506ad..3e1a61edc81 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/url" + "path" "sort" "strconv" "strings" @@ -54,17 +55,37 @@ func getTime(t string) time.Time { } func (d *Yun139) refreshToken() error { - if d.ref == nil { + if d.ref != nil { return d.ref.refreshToken() } - url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do" - var resp RefreshTokenResp decode, err := base64.StdEncoding.DecodeString(d.Authorization) if err != nil { - return err + return fmt.Errorf("authorization decode failed: %s", err) } decodeStr := string(decode) splits := strings.Split(decodeStr, ":") + if len(splits) < 3 { + return fmt.Errorf("authorization is invalid, splits < 3") + } + strs := strings.Split(splits[2], "|") + if len(strs) < 4 { + return fmt.Errorf("authorization is invalid, strs < 4") + } + expiration, err := strconv.ParseInt(strs[3], 10, 64) + if err != nil { + return fmt.Errorf("authorization is invalid") + } + expiration -= time.Now().UnixMilli() + if expiration > 1000*60*60*24*15 { + // Authorization有效期大于15天无需刷新 + return nil + } + if expiration < 0 { + return fmt.Errorf("authorization has expired") + } + + url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do" + var resp RefreshTokenResp reqBody := "" + splits[2] + "" + splits[1] + "656" _, err = base.RestyClient.R(). ForceContentType("application/xml"). @@ -108,15 +129,16 @@ func (d *Yun139) request(pathname string, method string, callback base.ReqCallba //"mcloud-route": "001", "mcloud-sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), //"mcloud-skey":"", - "mcloud-version": "6.6.0", - "Origin": "https://yun.139.com", - "Referer": "https://yun.139.com/w/", - "x-DeviceInfo": "||9|6.6.0|chrome|95.0.4638.69|uwIy75obnsRPIwlJSd7D9GhUvFwG96ce||macos 10.15.2||zh-CN|||", - "x-huawei-channelSrc": "10000034", - "x-inner-ntwk": "2", - "x-m4c-caller": "PC", - "x-m4c-src": "10002", - "x-SvcType": svcType, + "mcloud-version": "7.14.0", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/w/", + "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", + "x-huawei-channelSrc": "10000034", + "x-inner-ntwk": "2", + "x-m4c-caller": "PC", + "x-m4c-src": "10002", + "x-SvcType": svcType, + "Inner-Hcy-Router-Https": "1", }) var e BaseResp @@ -269,12 +291,12 @@ func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) { for { data := d.newJson(base.Json{ "groupID": d.CloudID, - "catalogID": catalogID, + "catalogID": path.Base(catalogID), "contentSortType": 0, "sortDirection": 1, "startNumber": pageNum, "endNumber": pageNum + 99, - "path": catalogID, + "path": path.Join(d.RootFolderID, catalogID), }) var resp QueryGroupContentListResp @@ -310,7 +332,7 @@ func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) { } files = append(files, &f) } - if pageNum > resp.Data.GetGroupContentResult.NodeCount { + if (pageNum + 99) > resp.Data.GetGroupContentResult.NodeCount { break } pageNum = pageNum + 100 @@ -393,10 +415,10 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R "Mcloud-Client": "10701", "Mcloud-Route": "001", "Mcloud-Sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), - "Mcloud-Version": "7.13.0", + "Mcloud-Version": "7.14.0", "Origin": "https://yun.139.com", "Referer": "https://yun.139.com/w/", - "x-DeviceInfo": "||9|7.13.0|chrome|120.0.0.0|||windows 10||zh-CN|||", + "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", "x-huawei-channelSrc": "10000034", "x-inner-ntwk": "2", "x-m4c-caller": "PC", @@ -405,7 +427,7 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R "X-Yun-Api-Version": "v1", "X-Yun-App-Channel": "10000034", "X-Yun-Channel-Source": "10000034", - "X-Yun-Client-Info": "||9|7.13.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", + "X-Yun-Client-Info": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", "X-Yun-Module-Type": "100", "X-Yun-Svc-Type": "1", }) From 23f3178f39981a6bdfcf90871cb8bfc3aed05117 Mon Sep 17 00:00:00 2001 From: LaoShui <79132480+laoshuikaixue@users.noreply.github.com> Date: Mon, 27 Jan 2025 20:13:35 +0800 Subject: [PATCH 425/659] chore(README): formatting spacing in README links (#7879) [skip ci] --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8140f325a9b..d1189188c41 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ --- -English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing](./CONTRIBUTING.md) | [CODE_OF_CONDUCT](./CODE_OF_CONDUCT.md) +English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing](./CONTRIBUTING.md) | [CODE_OF_CONDUCT](./CODE_OF_CONDUCT.md) ## Features From d5ec998699dd592e3ee7f54cf5bcce7dc697c173 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Mon, 27 Jan 2025 20:18:10 +0800 Subject: [PATCH 426/659] feat(task): allow retry canceled (#7852) --- internal/conf/config.go | 14 ++++++++------ internal/fs/archive.go | 6 +++++- internal/fs/copy.go | 1 + internal/offline_download/tool/download.go | 1 + internal/offline_download/tool/transfer.go | 1 + internal/task/base.go | 15 +++++++++++++++ 6 files changed, 31 insertions(+), 7 deletions(-) diff --git a/internal/conf/config.go b/internal/conf/config.go index 39b23227a26..1766ae8406f 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -53,12 +53,13 @@ type TaskConfig struct { } type TasksConfig struct { - Download TaskConfig `json:"download" envPrefix:"DOWNLOAD_"` - Transfer TaskConfig `json:"transfer" envPrefix:"TRANSFER_"` - Upload TaskConfig `json:"upload" envPrefix:"UPLOAD_"` - Copy TaskConfig `json:"copy" envPrefix:"COPY_"` - Decompress TaskConfig `json:"decompress" envPrefix:"DECOMPRESS_"` - DecompressUpload TaskConfig `json:"decompress_upload" envPrefix:"DECOMPRESS_UPLOAD_"` + Download TaskConfig `json:"download" envPrefix:"DOWNLOAD_"` + Transfer TaskConfig `json:"transfer" envPrefix:"TRANSFER_"` + Upload TaskConfig `json:"upload" envPrefix:"UPLOAD_"` + Copy TaskConfig `json:"copy" envPrefix:"COPY_"` + Decompress TaskConfig `json:"decompress" envPrefix:"DECOMPRESS_"` + DecompressUpload TaskConfig `json:"decompress_upload" envPrefix:"DECOMPRESS_UPLOAD_"` + AllowRetryCanceled bool `json:"allow_retry_canceled" env:"ALLOW_RETRY_CANCELED"` } type Cors struct { @@ -182,6 +183,7 @@ func DefaultConfig() *Config { Workers: 5, MaxRetry: 2, }, + AllowRetryCanceled: false, }, Cors: Cors{ AllowOrigins: []string{"*"}, diff --git a/internal/fs/archive.go b/internal/fs/archive.go index f3e05926e88..3913182702c 100644 --- a/internal/fs/archive.go +++ b/internal/fs/archive.go @@ -50,6 +50,7 @@ func (t *ArchiveDownloadTask) GetStatus() string { } func (t *ArchiveDownloadTask) Run() error { + t.ReinitCtx() t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() @@ -144,6 +145,7 @@ func (t *ArchiveContentUploadTask) GetStatus() string { } func (t *ArchiveContentUploadTask) Run() error { + t.ReinitCtx() t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() @@ -235,7 +237,9 @@ func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTsk *Archi func (t *ArchiveContentUploadTask) Cancel() { t.TaskExtension.Cancel() - t.deleteSrcFile() + if !conf.Conf.Tasks.AllowRetryCanceled { + t.deleteSrcFile() + } } func (t *ArchiveContentUploadTask) deleteSrcFile() { diff --git a/internal/fs/copy.go b/internal/fs/copy.go index 977f7280db9..155e3cf7a87 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -39,6 +39,7 @@ func (t *CopyTask) GetStatus() string { } func (t *CopyTask) Run() error { + t.ReinitCtx() t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index c3b30f1b4cf..42b2dbfb2cb 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -28,6 +28,7 @@ type DownloadTask struct { } func (t *DownloadTask) Run() error { + t.ReinitCtx() t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go index 8c7ab2448e6..1d5ece612ce 100644 --- a/internal/offline_download/tool/transfer.go +++ b/internal/offline_download/tool/transfer.go @@ -32,6 +32,7 @@ type TransferTask struct { } func (t *TransferTask) Run() error { + t.ReinitCtx() t.ClearEndTime() t.SetStartTime(time.Now()) defer func() { t.SetEndTime(time.Now()) }() diff --git a/internal/task/base.go b/internal/task/base.go index 22b167417db..c3703bd161f 100644 --- a/internal/task/base.go +++ b/internal/task/base.go @@ -2,6 +2,7 @@ package task import ( "context" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/xhofe/tache" "sync" @@ -66,6 +67,20 @@ func (t *TaskExtension) Ctx() context.Context { return t.ctx } +func (t *TaskExtension) ReinitCtx() { + if !conf.Conf.Tasks.AllowRetryCanceled { + return + } + select { + case <-t.Base.Ctx().Done(): + ctx, cancel := context.WithCancel(context.Background()) + t.SetCtx(ctx) + t.SetCancelFunc(cancel) + t.ctx = nil + default: + } +} + type TaskExtensionInfo interface { tache.TaskWithInfo GetCreator() *model.User From 5eff8cc7bffdbe5a20a37d1a9964bb391d42baaa Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Mon, 27 Jan 2025 20:20:09 +0800 Subject: [PATCH 427/659] feat(upload): support rapid upload on web (#7851) --- drivers/alist_v3/driver.go | 9 +++++++++ server/handles/fsup.go | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index d078c5fb421..894bac64607 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -189,6 +189,15 @@ func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt req.Header.Set("Authorization", d.Token) req.Header.Set("File-Path", path.Join(dstDir.GetPath(), stream.GetName())) req.Header.Set("Password", d.MetaPassword) + if md5 := stream.GetHash().GetHash(utils.MD5); len(md5) > 0 { + req.Header.Set("X-File-Md5", md5) + } + if sha1 := stream.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { + req.Header.Set("X-File-Sha1", sha1) + } + if sha256 := stream.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { + req.Header.Set("X-File-Sha256", sha256) + } req.ContentLength = stream.GetSize() // client := base.NewHttpClient() diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 563afbcd54a..15a6328b60b 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -2,6 +2,7 @@ package handles import ( "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/pkg/utils" "io" "net/url" stdpath "path" @@ -55,11 +56,22 @@ func FsStream(c *gin.Context) { common.ErrorResp(c, err, 400) return } + h := make(map[*utils.HashType]string) + if md5 := c.GetHeader("X-File-Md5"); md5 != "" { + h[utils.MD5] = md5 + } + if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" { + h[utils.SHA1] = sha1 + } + if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { + h[utils.SHA256] = sha256 + } s := &stream.FileStream{ Obj: &model.Object{ Name: name, Size: size, Modified: getLastModified(c), + HashInfo: utils.NewHashInfoByMap(h), }, Reader: c.Request.Body, Mimetype: c.GetHeader("Content-Type"), @@ -128,11 +140,22 @@ func FsForm(c *gin.Context) { } defer f.Close() dir, name := stdpath.Split(path) + h := make(map[*utils.HashType]string) + if md5 := c.GetHeader("X-File-Md5"); md5 != "" { + h[utils.MD5] = md5 + } + if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" { + h[utils.SHA1] = sha1 + } + if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { + h[utils.SHA256] = sha256 + } s := stream.FileStream{ Obj: &model.Object{ Name: name, Size: file.Size, Modified: getLastModified(c), + HashInfo: utils.NewHashInfoByMap(h), }, Reader: f, Mimetype: file.Header.Get("Content-Type"), From 267120a8c8bdde8793ccc212fa311418c9520823 Mon Sep 17 00:00:00 2001 From: Shelton Zhu <498220739@qq.com> Date: Mon, 27 Jan 2025 20:20:55 +0800 Subject: [PATCH 428/659] fix(115): fix offline download (#7845 close #7794) * feat(115): use multi url for list files & change download url api * fix(115): fix offline download. (close #7794) --- drivers/115/driver.go | 2 +- drivers/115/util.go | 27 ++++++++++++------------ drivers/115_share/meta.go | 2 +- go.mod | 6 ++---- go.sum | 44 ++------------------------------------- 5 files changed, 19 insertions(+), 62 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 4f584cd7b51..0bf8a927a29 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -241,7 +241,7 @@ func (d *Pan115) OfflineList(ctx context.Context) ([]*driver115.OfflineTask, err } func (d *Pan115) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { - return d.client.AddOfflineTaskURIs(uris, dstDir.GetID()) + return d.client.AddOfflineTaskURIs(uris, dstDir.GetID(), driver115.WithAppVer(appVer)) } func (d *Pan115) DeleteOfflineTasks(ctx context.Context, hashes []string, deleteFiles bool) error { diff --git a/drivers/115/util.go b/drivers/115/util.go index d7a1adff71c..84cbd88f3ae 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -21,9 +21,9 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" + cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" + crypto "github.com/SheltonZhu/115driver/pkg/crypto/m115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" - crypto "github.com/gaoyb7/115drive-webdav/115" - "github.com/orzogc/fake115uploader/cipher" "github.com/pkg/errors" ) @@ -63,7 +63,7 @@ func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { if d.PageSize <= 0 { d.PageSize = driver115.FileListLimit } - files, err := d.client.ListWithLimit(fileId, d.PageSize) + files, err := d.client.ListWithLimit(fileId, d.PageSize, driver115.WithMultiUrls()) if err != nil { return nil, err } @@ -108,7 +108,7 @@ func (d *Pan115) getUA() string { func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { key := crypto.GenerateKey() result := driver115.DownloadResp{} - params, err := utils.Json.Marshal(map[string]string{"pickcode": pickCode}) + params, err := utils.Json.Marshal(map[string]string{"pick_code": pickCode}) if err != nil { return nil, err } @@ -116,7 +116,7 @@ func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, e data := crypto.Encode(params, key) bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) - reqUrl := fmt.Sprintf("%s?t=%s", driver115.ApiDownloadGetUrl, driver115.Now().String()) + reqUrl := fmt.Sprintf("%s?t=%s", driver115.AndroidApiDownloadGetUrl, driver115.Now().String()) req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("Cookie", d.Cookie) @@ -145,19 +145,18 @@ func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, e return nil, err } - downloadInfo := driver115.DownloadData{} + downloadInfo := struct { + Url string `json:"url"` + }{} if err := utils.Json.Unmarshal(bytes, &downloadInfo); err != nil { return nil, err } - for _, info := range downloadInfo { - if info.FileSize < 0 { - return nil, driver115.ErrDownloadEmpty - } - info.Header = resp.Request.Header - return info, nil - } - return nil, driver115.ErrUnexpected + info := &driver115.DownloadInfo{} + info.PickCode = pickCode + info.Header = resp.Request.Header + info.Url.Url = downloadInfo.Url + return info, nil } func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { diff --git a/drivers/115_share/meta.go b/drivers/115_share/meta.go index b3d2cc1fad7..92f8bf0ff08 100644 --- a/drivers/115_share/meta.go +++ b/drivers/115_share/meta.go @@ -18,7 +18,7 @@ type Addition struct { var config = driver.Config{ Name: "115 Share", - DefaultRoot: "", + DefaultRoot: "0", // OnlyProxy: true, // OnlyLocal: true, CheckStatus: false, diff --git a/go.mod b/go.mod index 0693dcd32c3..2bf4ba3e90c 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ toolchain go1.23.1 require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 - github.com/SheltonZhu/115driver v1.0.32 + github.com/SheltonZhu/115driver v1.0.34 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 @@ -29,7 +29,6 @@ require ( github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 - github.com/gaoyb7/115drive-webdav v0.1.8 github.com/gin-contrib/cors v1.7.2 github.com/gin-gonic/gin v1.10.0 github.com/go-resty/resty/v2 v2.14.0 @@ -50,7 +49,6 @@ require ( github.com/minio/sio v0.4.0 github.com/natefinch/lumberjack v2.0.0+incompatible github.com/ncw/swift/v2 v2.0.3 - github.com/orzogc/fake115uploader v0.6.2 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 github.com/pquerna/otp v1.4.0 @@ -103,6 +101,7 @@ require ( github.com/ipfs/boxo v0.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/klauspost/pgzip v1.2.6 // indirect + github.com/kr/text v0.2.0 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect @@ -139,7 +138,6 @@ require ( github.com/blevesearch/zapx/v13 v13.3.10 // indirect github.com/blevesearch/zapx/v14 v14.3.10 // indirect github.com/blevesearch/zapx/v15 v15.3.13 // indirect - github.com/bluele/gcache v0.0.2 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/go.sum b/go.sum index 9d92a935f4d..db58dea2956 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4 github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.0.32 h1:Taw1bnfcPJZW0xTdhDvEbBS1tccif7J7DslRp2NkDyQ= -github.com/SheltonZhu/115driver v1.0.32/go.mod h1:XXFi23pyhAgzUE8dUEKdGvIdUQKi3wv6zR7C1Do40D8= +github.com/SheltonZhu/115driver v1.0.34 h1:zhMLp4vgq7GksqvSxQQDOVfK6EOHldQl4b2n8tnZ+EE= +github.com/SheltonZhu/115driver v1.0.34/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= @@ -110,8 +110,6 @@ github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wy github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi/AUHjU= github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8= -github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= -github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= @@ -199,15 +197,12 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw= -github.com/gaoyb7/115drive-webdav v0.1.8/go.mod h1:BKbeY6j8SKs3+rzBFFALznGxbPmefEm3vA+dGhqgOGU= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw= github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= @@ -226,20 +221,14 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= -github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= @@ -248,7 +237,6 @@ github.com/go-webauthn/webauthn v0.11.1 h1:5G/+dg91/VcaJHTtJUfwIlNJkLwbJCcnUc4W8 github.com/go-webauthn/webauthn v0.11.1/go.mod h1:YXRm1WG0OtUyDFaVAgB5KG7kVqW+6dYCJ7FTQH4SxEE= github.com/go-webauthn/x v0.1.12 h1:RjQ5cvApzyU/xLCiP+rub0PE4HBZsLggbxGR5ZpUf/A= github.com/go-webauthn/x v0.1.12/go.mod h1:XlRcGkNH8PT45TfeJYc6gqpOtiOendHhVmnOxh+5yHs= -github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= @@ -390,8 +378,6 @@ github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgSh github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -400,7 +386,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/larksuite/oapi-sdk-go/v3 v3.3.1 h1:DLQQEgHUAGZB6RVlceB1f6A94O206exxW2RIMH+gMUc= github.com/larksuite/oapi-sdk-go/v3 v3.3.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= @@ -420,7 +405,6 @@ github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -481,19 +465,15 @@ github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= -github.com/orzogc/fake115uploader v0.6.2 h1:f4LzqeeXpmY7DjOMnzmAnnPTPMA/f/BUclq4ecffTvU= -github.com/orzogc/fake115uploader v0.6.2/go.mod h1:Mqqwv1+gUEjJhUfIQanco3DCTKp+7lSx8DJ3AoRwMoE= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= -github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -528,8 +508,6 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -548,7 +526,6 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= @@ -570,14 +547,12 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= @@ -603,8 +578,6 @@ github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uk github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys= github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs= -github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -616,8 +589,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco= github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= -github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= -github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= @@ -669,11 +640,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= @@ -734,7 +702,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -779,7 +746,6 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -789,14 +755,11 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -843,7 +806,6 @@ golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -928,7 +890,6 @@ google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= @@ -950,7 +911,6 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= From 99f39410f2aa34793f047fbe781bc03f726b4a35 Mon Sep 17 00:00:00 2001 From: Jiang Xiang <869914918@qq.com> Date: Mon, 27 Jan 2025 20:23:13 +0800 Subject: [PATCH 429/659] fix(s3): escape CopySource request header when copying files (#7860 close #7858) --- drivers/s3/util.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/s3/util.go b/drivers/s3/util.go index 31e658bdcab..99f271aa071 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "net/url" "path" "strings" @@ -198,7 +199,7 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { dstKey := getKey(dst, false) input := &s3.CopyObjectInput{ Bucket: &d.Bucket, - CopySource: aws.String("/" + d.Bucket + "/" + srcKey), + CopySource: aws.String(url.PathEscape("/" + d.Bucket + "/" + srcKey)), Key: &dstKey, } _, err := d.client.CopyObject(input) From 258b8f520f467b7f7be7cc18d70f1e86de95f182 Mon Sep 17 00:00:00 2001 From: Jealous Date: Mon, 27 Jan 2025 20:25:39 +0800 Subject: [PATCH 430/659] feat(recursive-move): add `overwrite` option to preventing unintentional overwriting (#7868 closes #7382,#7719) * feat(recursive-move): add `overwrite` option to preventing unintentional overwriting * chore: rearrange code order --- server/handles/fsbatch.go | 136 +++++++++++++++++++++++--------------- 1 file changed, 82 insertions(+), 54 deletions(-) diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index fa7971dfbe1..dd7b7e470b3 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -3,6 +3,7 @@ package handles import ( "fmt" "regexp" + "slices" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" @@ -14,56 +15,10 @@ import ( "github.com/pkg/errors" ) -type BatchRenameReq struct { - SrcDir string `json:"src_dir"` - RenameObjects []struct { - SrcName string `json:"src_name"` - NewName string `json:"new_name"` - } `json:"rename_objects"` -} - -func FsBatchRename(c *gin.Context) { - var req BatchRenameReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } - - reqPath, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } - - meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } - } - c.Set("meta", meta) - for _, renameObject := range req.RenameObjects { - if renameObject.SrcName == "" || renameObject.NewName == "" { - continue - } - filePath := fmt.Sprintf("%s/%s", reqPath, renameObject.SrcName) - if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { - common.ErrorResp(c, err, 500) - return - } - } - common.SuccessResp(c) -} - type RecursiveMoveReq struct { - SrcDir string `json:"src_dir"` - DstDir string `json:"dst_dir"` + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + Overwrite bool `json:"overwrite"` } func FsRecursiveMove(c *gin.Context) { @@ -104,9 +59,23 @@ func FsRecursiveMove(c *gin.Context) { return } + var existingFileNames []string + if !req.Overwrite { + dstFiles, err := fs.List(c, dstDir, &fs.ListArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + existingFileNames = make([]string, 0, len(dstFiles)) + for _, dstFile := range dstFiles { + existingFileNames = append(existingFileNames, dstFile.GetName()) + } + } + // record the file path filePathMap := make(map[model.Obj]string) movingFiles := generic.NewQueue[model.Obj]() + movingFileNames := make([]string, 0, len(rootFiles)) for _, file := range rootFiles { movingFiles.Push(file) filePathMap[file] = srcDir @@ -136,16 +105,75 @@ func FsRecursiveMove(c *gin.Context) { continue } - // move - err := fs.Move(c, movingFileName, dstDir, movingFiles.IsEmpty()) - if err != nil { - common.ErrorResp(c, err, 500) - return + if !req.Overwrite { + if slices.Contains(existingFileNames, movingFile.GetName()) { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", movingFile.GetName()), 403) + return + } + existingFileNames = append(existingFileNames, movingFile.GetName()) } + + movingFileNames = append(movingFileNames, movingFileName) + } + + } + + for i, fileName := range movingFileNames { + // move + err := fs.Move(c, fileName, dstDir, len(movingFileNames) > i+1) + if err != nil { + common.ErrorResp(c, err, 500) + return } + } + + common.SuccessResp(c) +} + +type BatchRenameReq struct { + SrcDir string `json:"src_dir"` + RenameObjects []struct { + SrcName string `json:"src_name"` + NewName string `json:"new_name"` + } `json:"rename_objects"` +} + +func FsBatchRename(c *gin.Context) { + var req BatchRenameReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if !user.CanRename() { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + reqPath, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("meta", meta) + for _, renameObject := range req.RenameObjects { + if renameObject.SrcName == "" || renameObject.NewName == "" { + continue + } + filePath := fmt.Sprintf("%s/%s", reqPath, renameObject.SrcName) + if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { + common.ErrorResp(c, err, 500) + return + } + } common.SuccessResp(c) } From bdd9774aa7684f7eb66f6758d537ee161ee14078 Mon Sep 17 00:00:00 2001 From: Sakana Date: Mon, 27 Jan 2025 20:28:44 +0800 Subject: [PATCH 431/659] feat(github_releases): add support for github_releases driver (#7844 close #7842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(github_releases): 添加对 GitHub Releases 的支持 * feat(github_releases): 增加目录大小和更新时间，增加请求缓存 * Feat(github_releases): 可选填入 GitHub token 来提高速率限制或访问私有仓库 * Fix(github_releases): 修复仓库无权限或不存在时的异常 * feat(github_releases): 支持显示所有版本，开启后不显示文件夹大小 * feat(github_releases): 兼容无子目录 --- drivers/all.go | 1 + drivers/github_releases/driver.go | 153 +++++++++++++++++++++ drivers/github_releases/meta.go | 34 +++++ drivers/github_releases/types.go | 68 ++++++++++ drivers/github_releases/util.go | 217 ++++++++++++++++++++++++++++++ 5 files changed, 473 insertions(+) create mode 100644 drivers/github_releases/driver.go create mode 100644 drivers/github_releases/meta.go create mode 100644 drivers/github_releases/types.go create mode 100644 drivers/github_releases/util.go diff --git a/drivers/all.go b/drivers/all.go index 8b253a08558..bd051168eaa 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -25,6 +25,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/github" + _ "github.com/alist-org/alist/v3/drivers/github_releases" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" _ "github.com/alist-org/alist/v3/drivers/halalcloud" diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go new file mode 100644 index 00000000000..79f2b582146 --- /dev/null +++ b/drivers/github_releases/driver.go @@ -0,0 +1,153 @@ +package github_releases + +import ( + "context" + "fmt" + "net/http" + "time" + + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type GithubReleases struct { + model.Storage + Addition + + releases []Release +} + +func (d *GithubReleases) Config() driver.Config { + return config +} + +func (d *GithubReleases) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GithubReleases) Init(ctx context.Context) error { + SetHeader(d.Addition.Token) + repos, err := ParseRepos(d.Addition.RepoStructure, d.Addition.ShowAllVersion) + if err != nil { + return err + } + d.releases = repos + return nil +} + +func (d *GithubReleases) Drop(ctx context.Context) error { + ClearCache() + return nil +} + +func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]File, 0) + path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) + + for _, repo := range d.releases { + if repo.Path == path { // 与仓库路径相同 + resp, err := GetRepoReleaseInfo(repo.RepoName, repo.ID, path, d.Storage.CacheExpiration) + if err != nil { + return nil, err + } + files = append(files, resp.Files...) + + if d.Addition.ShowReadme { + resp, err := GetGithubOtherFile(repo.RepoName, path, d.Storage.CacheExpiration) + if err != nil { + return nil, err + } + files = append(files, *resp...) + } + + } else if strings.HasPrefix(repo.Path, path) { // 仓库路径是目录的子目录 + nextDir := GetNextDir(repo.Path, path) + if nextDir == "" { + continue + } + if d.Addition.ShowAllVersion { + files = append(files, File{ + FileName: nextDir, + Size: 0, + CreateAt: time.Time{}, + UpdateAt: time.Time{}, + Url: "", + Type: "dir", + Path: fmt.Sprintf("%s/%s", path, nextDir), + }) + continue + } + + repo, _ := GetRepoReleaseInfo(repo.RepoName, repo.Version, path, d.Storage.CacheExpiration) + + hasSameDir := false + for index, file := range files { + if file.FileName == nextDir { + hasSameDir = true + files[index].Size += repo.Size + files[index].UpdateAt = func(a time.Time, b time.Time) time.Time { + if a.After(b) { + return a + } + return b + }(files[index].UpdateAt, repo.UpdateAt) + break + } + } + + if !hasSameDir { + files = append(files, File{ + FileName: nextDir, + Size: repo.Size, + CreateAt: repo.CreateAt, + UpdateAt: repo.UpdateAt, + Url: repo.Url, + Type: "dir", + Path: fmt.Sprintf("%s/%s", path, nextDir), + }) + } + } + } + + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return src, nil + }) +} + +func (d *GithubReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + link := model.Link{ + URL: file.GetID(), + Header: http.Header{}, + } + return &link, nil +} + +func (d *GithubReleases) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *GithubReleases) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *GithubReleases) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *GithubReleases) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *GithubReleases) Remove(ctx context.Context, obj model.Obj) error { + return errs.NotImplement +} + +func (d *GithubReleases) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*GithubReleases)(nil) diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go new file mode 100644 index 00000000000..ca6ca5dc8d0 --- /dev/null +++ b/drivers/github_releases/meta.go @@ -0,0 +1,34 @@ +package github_releases + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"/path/to/alist-gh:alistGo/alist\n/path/to2/alist-web-gh:AlistGo/alist-web" help:"structure:[path:]org/repo"` + ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` + Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` + ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` +} + +var config = driver.Config{ + Name: "GitHub Releases", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GithubReleases{} + }) +} diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go new file mode 100644 index 00000000000..733460dca5f --- /dev/null +++ b/drivers/github_releases/types.go @@ -0,0 +1,68 @@ +package github_releases + +import ( + "time" + + "github.com/alist-org/alist/v3/pkg/utils" +) + +type File struct { + FileName string `json:"name"` + Size int64 `json:"size"` + CreateAt time.Time `json:"time"` + UpdateAt time.Time `json:"chtime"` + Url string `json:"url"` + Type string `json:"type"` + Path string `json:"path"` +} + +func (f File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f File) GetPath() string { + return f.Path +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) ModTime() time.Time { + return f.UpdateAt +} + +func (f File) CreateTime() time.Time { + return f.CreateAt +} + +func (f File) IsDir() bool { + return f.Type == "dir" +} + +func (f File) GetID() string { + return f.Url +} + +func (f File) Thumb() string { + return "" +} + +type ReleasesData struct { + Files []File `json:"files"` + Size int64 `json:"size"` + UpdateAt time.Time `json:"chtime"` + CreateAt time.Time `json:"time"` + Url string `json:"url"` +} + +type Release struct { + Path string // 挂载路径 + RepoName string // 仓库名称 + Version string // 版本号, tag + ID string // 版本ID +} diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go new file mode 100644 index 00000000000..b2d79c0b3c1 --- /dev/null +++ b/drivers/github_releases/util.go @@ -0,0 +1,217 @@ +package github_releases + +import ( + "fmt" + "regexp" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" + jsoniter "github.com/json-iterator/go" + log "github.com/sirupsen/logrus" +) + +var ( + cache = make(map[string]*resty.Response) + created = make(map[string]time.Time) + mu sync.Mutex + req *resty.Request +) + +// 解析仓库列表 +func ParseRepos(text string, allVersion bool) ([]Release, error) { + lines := strings.Split(text, "\n") + var repos []Release + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, ":") + path, repo := "", "" + if len(parts) == 1 { + path = "/" + repo = parts[0] + } else if len(parts) == 2 { + path = fmt.Sprintf("/%s", strings.Trim(parts[0], "/")) + repo = parts[1] + } else { + return nil, fmt.Errorf("invalid format: %s", line) + } + + if allVersion { + releases, _ := GetAllVersion(repo, path) + repos = append(repos, *releases...) + } else { + repos = append(repos, Release{ + Path: path, + RepoName: repo, + Version: "latest", + ID: "latest", + }) + } + + } + return repos, nil +} + +// 获取下一级目录 +func GetNextDir(wholePath string, basePath string) string { + if !strings.HasSuffix(basePath, "/") { + basePath += "/" + } + if !strings.HasPrefix(wholePath, basePath) { + return "" + } + remainingPath := strings.TrimLeft(strings.TrimPrefix(wholePath, basePath), "/") + if remainingPath != "" { + parts := strings.Split(remainingPath, "/") + return parts[0] + } + return "" +} + +// 发送 GET 请求 +func GetRequest(url string, cacheExpiration int) (*resty.Response, error) { + mu.Lock() + if res, ok := cache[url]; ok && time.Now().Before(created[url].Add(time.Duration(cacheExpiration)*time.Minute)) { + mu.Unlock() + return res, nil + } + mu.Unlock() + + res, err := req.Get(url) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + log.Warn("failed to get request: ", res.StatusCode(), res.String()) + } + + mu.Lock() + cache[url] = res + created[url] = time.Now() + mu.Unlock() + + return res, nil +} + +// 获取 README、LICENSE 等文件 +func GetGithubOtherFile(repo string, basePath string, cacheExpiration int) (*[]File, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/contents/", strings.Trim(repo, "/")) + res, _ := GetRequest(url, cacheExpiration) + body := jsoniter.Get(res.Body()) + var files []File + for i := 0; i < body.Size(); i++ { + filename := body.Get(i, "name").ToString() + + re := regexp.MustCompile(`(?i)^(.*\.md|LICENSE)$`) + + if !re.MatchString(filename) { + continue + } + + files = append(files, File{ + FileName: filename, + Size: body.Get(i, "size").ToInt64(), + CreateAt: time.Time{}, + UpdateAt: time.Now(), + Url: body.Get(i, "download_url").ToString(), + Type: body.Get(i, "type").ToString(), + Path: fmt.Sprintf("%s/%s", basePath, filename), + }) + } + return &files, nil +} + +// 获取 GitHub Release 详细信息 +func GetRepoReleaseInfo(repo string, version string, basePath string, cacheExpiration int) (*ReleasesData, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/releases/%s", strings.Trim(repo, "/"), version) + res, _ := GetRequest(url, cacheExpiration) + body := res.Body() + + if jsoniter.Get(res.Body(), "status").ToInt64() != 0 { + return &ReleasesData{}, fmt.Errorf("%s", res.String()) + } + + assets := jsoniter.Get(res.Body(), "assets") + var files []File + + for i := 0; i < assets.Size(); i++ { + filename := assets.Get(i, "name").ToString() + + files = append(files, File{ + FileName: filename, + Size: assets.Get(i, "size").ToInt64(), + Url: assets.Get(i, "browser_download_url").ToString(), + Type: assets.Get(i, "content_type").ToString(), + Path: fmt.Sprintf("%s/%s", basePath, filename), + + CreateAt: func() time.Time { + t, _ := time.Parse(time.RFC3339, assets.Get(i, "created_at").ToString()) + return t + }(), + UpdateAt: func() time.Time { + t, _ := time.Parse(time.RFC3339, assets.Get(i, "updated_at").ToString()) + return t + }(), + }) + } + + return &ReleasesData{ + Files: files, + Url: jsoniter.Get(body, "html_url").ToString(), + + Size: func() int64 { + size := int64(0) + for _, file := range files { + size += file.Size + } + return size + }(), + UpdateAt: func() time.Time { + t, _ := time.Parse(time.RFC3339, jsoniter.Get(body, "published_at").ToString()) + return t + }(), + CreateAt: func() time.Time { + t, _ := time.Parse(time.RFC3339, jsoniter.Get(body, "created_at").ToString()) + return t + }(), + }, nil +} + +// 获取所有的版本号 +func GetAllVersion(repo string, path string) (*[]Release, error) { + url := fmt.Sprintf("https://api.github.com/repos/%s/releases", strings.Trim(repo, "/")) + res, _ := GetRequest(url, 0) + body := jsoniter.Get(res.Body()) + releases := make([]Release, 0) + for i := 0; i < body.Size(); i++ { + version := body.Get(i, "tag_name").ToString() + releases = append(releases, Release{ + Path: fmt.Sprintf("%s/%s", path, version), + Version: version, + RepoName: repo, + ID: body.Get(i, "id").ToString(), + }) + } + return &releases, nil +} + +func ClearCache() { + mu.Lock() + cache = make(map[string]*resty.Response) + created = make(map[string]time.Time) + mu.Unlock() +} + +func SetHeader(token string) { + req = base.RestyClient.R() + if token != "" { + req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)) + } + req.SetHeader("Accept", "application/vnd.github+json") + req.SetHeader("X-GitHub-Api-Version", "2022-11-28") +} From fd51f34efa70005ffd69378ddb05183f405911ee Mon Sep 17 00:00:00 2001 From: Snowykami Date: Mon, 27 Jan 2025 20:47:52 +0800 Subject: [PATCH 432/659] feat(misskey): add misskey driver (#7864) --- drivers/all.go | 1 + drivers/misskey/driver.go | 74 +++++++++++ drivers/misskey/meta.go | 35 ++++++ drivers/misskey/types.go | 35 ++++++ drivers/misskey/util.go | 256 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 401 insertions(+) create mode 100644 drivers/misskey/driver.go create mode 100644 drivers/misskey/meta.go create mode 100644 drivers/misskey/types.go create mode 100644 drivers/misskey/util.go diff --git a/drivers/all.go b/drivers/all.go index bd051168eaa..2746e1bf7cb 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -37,6 +37,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/local" _ "github.com/alist-org/alist/v3/drivers/mediatrack" _ "github.com/alist-org/alist/v3/drivers/mega" + _ "github.com/alist-org/alist/v3/drivers/misskey" _ "github.com/alist-org/alist/v3/drivers/mopan" _ "github.com/alist-org/alist/v3/drivers/netease_music" _ "github.com/alist-org/alist/v3/drivers/onedrive" diff --git a/drivers/misskey/driver.go b/drivers/misskey/driver.go new file mode 100644 index 00000000000..29797a01242 --- /dev/null +++ b/drivers/misskey/driver.go @@ -0,0 +1,74 @@ +package misskey + +import ( + "context" + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type Misskey struct { + model.Storage + Addition +} + +func (d *Misskey) Config() driver.Config { + return config +} + +func (d *Misskey) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Misskey) Init(ctx context.Context) error { + d.Endpoint = strings.TrimSuffix(d.Endpoint, "/") + if d.Endpoint == "" || d.AccessToken == "" { + return errs.EmptyToken + } else { + return nil + } +} + +func (d *Misskey) Drop(ctx context.Context) error { + return nil +} + +func (d *Misskey) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.list(dir) +} + +func (d *Misskey) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return d.link(file) +} + +func (d *Misskey) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return d.makeDir(parentDir, dirName) +} + +func (d *Misskey) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.move(srcObj, dstDir) +} + +func (d *Misskey) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return d.rename(srcObj, newName) +} + +func (d *Misskey) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.copy(srcObj, dstDir) +} + +func (d *Misskey) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(obj) +} + +func (d *Misskey) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return d.put(dstDir, stream, up) +} + +//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Misskey)(nil) diff --git a/drivers/misskey/meta.go b/drivers/misskey/meta.go new file mode 100644 index 00000000000..b8a80c159ff --- /dev/null +++ b/drivers/misskey/meta.go @@ -0,0 +1,35 @@ +package misskey + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // define other + // Field string `json:"field" type:"select" required:"true" options:"a,b,c" default:"a"` + Endpoint string `json:"endpoint" required:"true" default:"https://misskey.io"` + AccessToken string `json:"access_token" required:"true"` +} + +var config = driver.Config{ + Name: "Misskey", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Misskey{} + }) +} diff --git a/drivers/misskey/types.go b/drivers/misskey/types.go new file mode 100644 index 00000000000..e9adc8d2c6e --- /dev/null +++ b/drivers/misskey/types.go @@ -0,0 +1,35 @@ +package misskey + +type Resp struct { + Code int + Raw []byte +} + +type Properties struct { + Width int `json:"width"` + Height int `json:"height"` +} + +type MFile struct { + ID string `json:"id"` + CreatedAt string `json:"createdAt"` + Name string `json:"name"` + Type string `json:"type"` + MD5 string `json:"md5"` + Size int64 `json:"size"` + IsSensitive bool `json:"isSensitive"` + Blurhash string `json:"blurhash"` + Properties Properties `json:"properties"` + URL string `json:"url"` + ThumbnailURL string `json:"thumbnailUrl"` + Comment *string `json:"comment"` + FolderID *string `json:"folderId"` + Folder MFolder `json:"folder"` +} + +type MFolder struct { + ID string `json:"id"` + CreatedAt string `json:"createdAt"` + Name string `json:"name"` + ParentID *string `json:"parentId"` +} diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go new file mode 100644 index 00000000000..4d5a3b4d01f --- /dev/null +++ b/drivers/misskey/util.go @@ -0,0 +1,256 @@ +package misskey + +import ( + "bytes" + "context" + "errors" + "io" + "time" + + "github.com/go-resty/resty/v2" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// Base layer methods + +func (d *Misskey) request(path, method string, callback base.ReqCallback, resp interface{}) error { + url := d.Endpoint + "/api/drive" + path + req := base.RestyClient.R() + + req.SetAuthToken(d.AccessToken).SetHeader("Content-Type", "application/json") + + if callback != nil { + callback(req) + } else { + req.SetBody("{}") + } + + req.SetResult(resp) + + // 启用调试模式 + req.EnableTrace() + + response, err := req.Execute(method, url) + if err != nil { + return err + } + if !response.IsSuccess() { + return errors.New(response.String()) + } + return nil +} + +func (d *Misskey) getThumb(ctx context.Context, obj model.Obj) (io.Reader, error) { + // TODO return the thumb of obj, optional + return nil, errs.NotImplement +} + +func setBody(body interface{}) base.ReqCallback { + return func(req *resty.Request) { + req.SetBody(body) + } +} + +func handleFolderId(dir model.Obj) interface{} { + if dir.GetID() == "" { + return nil + } + return dir.GetID() +} + +// API layer methods + +func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { + var files []MFile + var body map[string]string + if dir.GetPath() != "/" { + body = map[string]string{"folderId": dir.GetID()} + } else { + body = map[string]string{} + } + err := d.request("/files", "POST", setBody(body), &files) + if err != nil { + return []model.Obj{}, err + } + return utils.SliceConvert(files, func(src MFile) (model.Obj, error) { + return mFile2Object(src), nil + }) +} + +func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) { + var folders []MFolder + var body map[string]string + if dir.GetPath() != "/" { + body = map[string]string{"folderId": dir.GetID()} + } else { + body = map[string]string{} + } + err := d.request("/folders", "POST", setBody(body), &folders) + if err != nil { + return []model.Obj{}, err + } + return utils.SliceConvert(folders, func(src MFolder) (model.Obj, error) { + return mFolder2Object(src), nil + }) +} + +func (d *Misskey) list(dir model.Obj) ([]model.Obj, error) { + files, _ := d.getFiles(dir) + folders, _ := d.getFolders(dir) + return append(files, folders...), nil +} + +func (d *Misskey) link(file model.Obj) (*model.Link, error) { + var mFile MFile + err := d.request("/files/show", "POST", setBody(map[string]string{"fileId": file.GetID()}), &mFile) + if err != nil { + return nil, err + } + return &model.Link{ + URL: mFile.URL, + }, nil +} + +func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error) { + var folder MFolder + err := d.request("/folders/create", "POST", setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) + if err != nil { + return nil, err + } + return mFolder2Object(folder), nil +} + +func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + var folder MFolder + err := d.request("/folders/update", "POST", setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) + return mFolder2Object(folder), err + } else { + var file MFile + err := d.request("/files/update", "POST", setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) + return mFile2Object(file), err + } +} + +func (d *Misskey) rename(srcObj model.Obj, newName string) (model.Obj, error) { + if srcObj.IsDir() { + var folder MFolder + err := d.request("/folders/update", "POST", setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) + return mFolder2Object(folder), err + } else { + var file MFile + err := d.request("/files/update", "POST", setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) + return mFile2Object(file), err + } +} + +func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + folder, err := d.makeDir(dstDir, srcObj.GetName()) + if err != nil { + return nil, err + } + list, err := d.list(srcObj) + if err != nil { + return nil, err + } + for _, obj := range list { + _, err := d.copy(obj, folder) + if err != nil { + return nil, err + } + } + return folder, nil + } else { + var file MFile + url, err := d.link(srcObj) + if err != nil { + return nil, err + } + err = d.request("/files/upload-from-url", "POST", setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) + if err != nil { + return nil, err + } + return mFile2Object(file), nil + } +} + +func (d *Misskey) remove(obj model.Obj) error { + if obj.IsDir() { + err := d.request("/folders/delete", "POST", setBody(map[string]string{"folderId": obj.GetID()}), nil) + return err + } else { + err := d.request("/files/delete", "POST", setBody(map[string]string{"fileId": obj.GetID()}), nil) + return err + } +} + +func (d *Misskey) put(dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var file MFile + + fileContent, err := io.ReadAll(stream) + if err != nil { + return nil, err + } + + req := base.RestyClient.R(). + SetFileReader("file", stream.GetName(), io.NopCloser(bytes.NewReader(fileContent))). + SetFormData(map[string]string{ + "folderId": handleFolderId(dstDir).(string), + "name": stream.GetName(), + "comment": "", + "isSensitive": "false", + "force": "false", + }). + SetResult(&file).SetAuthToken(d.AccessToken) + + resp, err := req.Post(d.Endpoint + "/api/drive/files/create") + if err != nil { + return nil, err + } + if !resp.IsSuccess() { + return nil, errors.New(resp.String()) + } + + return mFile2Object(file), nil +} + +func mFile2Object(file MFile) *model.ObjThumbURL { + ctime, err := time.Parse(time.RFC3339, file.CreatedAt) + if err != nil { + ctime = time.Time{} + } + return &model.ObjThumbURL{ + Object: model.Object{ + ID: file.ID, + Name: file.Name, + Ctime: ctime, + IsFolder: false, + Size: file.Size, + }, + Thumbnail: model.Thumbnail{ + Thumbnail: file.ThumbnailURL, + }, + Url: model.Url{ + Url: file.URL, + }, + } +} + +func mFolder2Object(folder MFolder) *model.Object { + ctime, err := time.Parse(time.RFC3339, folder.CreatedAt) + if err != nil { + ctime = time.Time{} + } + return &model.Object{ + ID: folder.ID, + Name: folder.Name, + Ctime: ctime, + IsFolder: true, + } +} From 027edcbe536230c4607f1d73ccff85c666ed146c Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Mon, 27 Jan 2025 20:49:24 +0800 Subject: [PATCH 433/659] refactor(patch): execute all patches in dev version (#7807) --- internal/bootstrap/patch.go | 6 ++++++ internal/bootstrap/patch/v3_41_0/grant_permission.go | 11 ++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/internal/bootstrap/patch.go b/internal/bootstrap/patch.go index 2d22d1b6388..5c7ca7583b6 100644 --- a/internal/bootstrap/patch.go +++ b/internal/bootstrap/patch.go @@ -2,6 +2,7 @@ package bootstrap import ( "fmt" + "github.com/alist-org/alist/v3/internal/bootstrap/patch" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/pkg/utils" @@ -40,6 +41,11 @@ func compareVersion(majorA, minorA, patchNumA, majorB, minorB, patchNumB int) bo func InitUpgradePatch() { if !strings.HasPrefix(conf.Version, "v") { + for _, vp := range patch.UpgradePatches { + for i, p := range vp.Patches { + safeCall(vp.Version, i, p) + } + } return } if LastLaunchedVersion == conf.Version { diff --git a/internal/bootstrap/patch/v3_41_0/grant_permission.go b/internal/bootstrap/patch/v3_41_0/grant_permission.go index e62d1e8fa90..60d8ab4fa3b 100644 --- a/internal/bootstrap/patch/v3_41_0/grant_permission.go +++ b/internal/bootstrap/patch/v3_41_0/grant_permission.go @@ -11,14 +11,11 @@ import ( // PR AlistGo/alist#7817. func GrantAdminPermissions() { admin, err := op.GetAdmin() - if err != nil { - utils.Log.Errorf("Cannot grant permissions to admin: %v", err) - } - if (admin.Permission & 0x33FF) == 0 { + if err == nil && (admin.Permission & 0x33FF) == 0 { admin.Permission |= 0x33FF err = op.UpdateUser(admin) - if err != nil { - utils.Log.Errorf("Cannot grant permissions to admin: %v", err) - } + } + if err != nil { + utils.Log.Errorf("Cannot grant permissions to admin: %v", err) } } From 226c34929a8bf7154bcf6c9f701c78da356c7ce5 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 27 Jan 2025 20:59:58 +0800 Subject: [PATCH 434/659] feat(ci): add build info for beta release --- .github/workflows/beta_release.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index c9cb7475780..3c52b4c40ca 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -87,12 +87,18 @@ jobs: run: bash build.sh dev web - name: Build - id: test-action uses: go-cross/cgo-actions@v1 with: targets: ${{ matrix.target }} musl-target-format: $os-$musl-$arch out-dir: build + x-flags: | + github.com/alist-org/alist/v3/internal/conf.BuiltAt=$built_at + github.com/alist-org/alist/v3/internal/conf.GoVersion=$go_version + github.com/alist-org/alist/v3/internal/conf.GitAuthor=Xhofe + github.com/alist-org/alist/v3/internal/conf.GitCommit=$git_commit + github.com/alist-org/alist/v3/internal/conf.Version=$tag + github.com/alist-org/alist/v3/internal/conf.WebVersion=dev - name: Compress run: | From f88fd83d4ac3372076215abf6fc2ccabde679d2b Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 28 Jan 2025 18:55:56 +0800 Subject: [PATCH 435/659] feat(ci): use `go-cross/cgo-actions` for dev build --- .github/workflows/build.yml | 44 ++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b059a20b0d9..fe037f43367 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,14 +15,17 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - go-version: [ '1.21' ] + target: + - darwin-amd64 + - darwin-arm64 + - windows-amd64 + - linux-arm64-musl + - linux-amd64-musl + - windows-arm64 + - android-arm64 name: Build runs-on: ${{ matrix.platform }} steps: - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go-version }} - name: Checkout uses: actions/checkout@v4 @@ -30,19 +33,30 @@ jobs: - uses: benjlevesque/short-sha@v3.0 id: short-sha - - name: Install dependencies - run: | - sudo snap install zig --classic --beta - docker pull crazymax/xgo:latest - go install github.com/crazy-max/xgo@latest - sudo apt install upx + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Setup web + run: bash build.sh dev web - name: Build - run: | - bash build.sh dev + uses: go-cross/cgo-actions@v1 + with: + targets: ${{ matrix.target }} + musl-target-format: $os-$musl-$arch + out-dir: build + x-flags: | + github.com/alist-org/alist/v3/internal/conf.BuiltAt=$built_at + github.com/alist-org/alist/v3/internal/conf.GoVersion=$go_version + github.com/alist-org/alist/v3/internal/conf.GitAuthor=Xhofe + github.com/alist-org/alist/v3/internal/conf.GitCommit=$git_commit + github.com/alist-org/alist/v3/internal/conf.Version=$tag + github.com/alist-org/alist/v3/internal/conf.WebVersion=dev - name: Upload artifact uses: actions/upload-artifact@v4 with: - name: alist_${{ env.SHA }} - path: dist \ No newline at end of file + name: alist_${{ env.SHA }}_${{ matrix.target }} + path: build/* \ No newline at end of file From d53eecc2292e84681e8a8e8641125b52f8c88954 Mon Sep 17 00:00:00 2001 From: Jiang Xiang <869914918@qq.com> Date: Thu, 30 Jan 2025 11:24:07 +0800 Subject: [PATCH 436/659] fix(febbox): panic due to slice out of range (#7898 close #7889) --- drivers/febbox/util.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/febbox/util.go b/drivers/febbox/util.go index ac072edbde8..ad2efe070e1 100644 --- a/drivers/febbox/util.go +++ b/drivers/febbox/util.go @@ -3,6 +3,7 @@ package febbox import ( "encoding/json" "errors" + "fmt" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/op" "github.com/go-resty/resty/v2" @@ -135,6 +136,9 @@ func (d *FebBox) getDownloadLink(id string, ip string) (string, error) { if err = json.Unmarshal(res, &fileDownloadResp); err != nil { return "", err } + if len(fileDownloadResp.Data) == 0 { + return "", fmt.Errorf("can not get download link, code:%d, msg:%s", fileDownloadResp.Code, fileDownloadResp.Msg) + } return fileDownloadResp.Data[0].DownloadURL, nil } From b9f397d29f0a4e75c72564da40d0f297ed8c5626 Mon Sep 17 00:00:00 2001 From: abc1763613206 Date: Thu, 30 Jan 2025 11:25:41 +0800 Subject: [PATCH 437/659] fix(139): restore the `Account` handling, partially reverts #7850 (#7900 close #7784) --- drivers/139/driver.go | 47 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index cf64a8fdb7d..1e2ba9c4d52 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -2,11 +2,13 @@ package _139 import ( "context" + "encoding/base64" "fmt" "io" "net/http" "path" "strconv" + "strings" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -69,29 +71,28 @@ func (d *Yun139) Init(ctx context.Context) error { default: return errs.NotImplement } - // if d.ref != nil { - // return nil - // } - // decode, err := base64.StdEncoding.DecodeString(d.Authorization) - // if err != nil { - // return err - // } - // decodeStr := string(decode) - // splits := strings.Split(decodeStr, ":") - // if len(splits) < 2 { - // return fmt.Errorf("authorization is invalid, splits < 2") - // } - // d.Account = splits[1] - // _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ - // "qryUserExternInfoReq": base.Json{ - // "commonAccountInfo": base.Json{ - // "account": d.getAccount(), - // "accountType": 1, - // }, - // }, - // }, nil) - // return err - return nil + if d.ref != nil { + return nil + } + decode, err := base64.StdEncoding.DecodeString(d.Authorization) + if err != nil { + return err + } + decodeStr := string(decode) + splits := strings.Split(decodeStr, ":") + if len(splits) < 2 { + return fmt.Errorf("authorization is invalid, splits < 2") + } + d.Account = splits[1] + _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ + "qryUserExternInfoReq": base.Json{ + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + }, + }, nil) + return err } func (d *Yun139) InitReference(storage driver.Driver) error { From 779c293f04a387cfef210b83632aeeb7c5fb69de Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Sat, 1 Feb 2025 17:29:55 +0800 Subject: [PATCH 438/659] fix(driver): implement canceling and updating progress for putting for some drivers (#7847) * fix(driver): additionally implement canceling and updating progress for putting for some drivers * refactor: add driver archive api into template * fix(123): use built-in MD5 to avoid caching full * . * fix build failed --- drivers/115/driver.go | 4 +- drivers/115/util.go | 31 ++++++-- drivers/123/driver.go | 58 ++++++++------ drivers/123/upload.go | 4 +- drivers/alist_v3/driver.go | 18 +++-- drivers/chaoxing/driver.go | 16 +++- drivers/ftp/driver.go | 14 +++- drivers/github/driver.go | 17 +++-- drivers/github/util.go | 16 ---- drivers/ilanzou/driver.go | 36 +++++---- drivers/ipfs_api/driver.go | 11 ++- drivers/kodbox/driver.go | 16 ++-- drivers/lanzou/driver.go | 10 ++- drivers/mediatrack/driver.go | 25 +++--- drivers/netease_music/driver.go | 2 +- drivers/netease_music/types.go | 16 ++++ drivers/netease_music/upload.go | 13 +++- drivers/netease_music/util.go | 32 ++++++-- drivers/pikpak/driver.go | 4 +- drivers/pikpak/util.go | 32 ++++++-- drivers/quqi/driver.go | 13 +++- drivers/s3/driver.go | 19 +++-- drivers/seafile/driver.go | 12 ++- drivers/template/driver.go | 24 +++++- drivers/thunder/driver.go | 22 +++--- drivers/thunderx/driver.go | 22 +++--- drivers/trainbit/driver.go | 18 ++--- drivers/trainbit/util.go | 11 --- drivers/uss/driver.go | 14 +++- drivers/webdav/driver.go | 16 ++-- drivers/weiyun/driver.go | 130 +++++++++++++++++--------------- drivers/wopan/driver.go | 1 + drivers/yandex_disk/driver.go | 16 ++-- internal/driver/driver.go | 6 +- internal/stream/stream.go | 14 ++++ 35 files changed, 457 insertions(+), 256 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 0bf8a927a29..0dcb64d8284 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -215,12 +215,12 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr var uploadResult *UploadResult // 闪传失败，上传 if stream.GetSize() <= 10*utils.MB { // 文件大小小于10MB，改用普通模式上传 - if uploadResult, err = d.UploadByOSS(&fastInfo.UploadOSSParams, stream, dirID); err != nil { + if uploadResult, err = d.UploadByOSS(ctx, &fastInfo.UploadOSSParams, stream, dirID, up); err != nil { return nil, err } } else { // 分片上传 - if uploadResult, err = d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID); err != nil { + if uploadResult, err = d.UploadByMultipart(ctx, &fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID, up); err != nil { return nil, err } } diff --git a/drivers/115/util.go b/drivers/115/util.go index 84cbd88f3ae..4d3cdd93ff1 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -2,17 +2,21 @@ package _115 import ( "bytes" + "context" "crypto/md5" "crypto/tls" "encoding/hex" "encoding/json" "fmt" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "net/url" "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/alist-org/alist/v3/internal/conf" @@ -271,7 +275,7 @@ func UploadDigestRange(stream model.FileStreamer, rangeSpec string) (result stri } // UploadByOSS use aliyun sdk to upload -func (c *Pan115) UploadByOSS(params *driver115.UploadOSSParams, r io.Reader, dirID string) (*UploadResult, error) { +func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSParams, s model.FileStreamer, dirID string, up driver.UpdateProgress) (*UploadResult, error) { ossToken, err := c.client.GetOSSToken() if err != nil { return nil, err @@ -286,6 +290,13 @@ func (c *Pan115) UploadByOSS(params *driver115.UploadOSSParams, r io.Reader, dir } var bodyBytes []byte + r := &stream.ReaderWithCtx{ + Reader: &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }, + Ctx: ctx, + } if err = bucket.PutObject(params.Object, r, append( driver115.OssOption(params, ossToken), oss.CallbackResult(&bodyBytes), @@ -301,7 +312,8 @@ func (c *Pan115) UploadByOSS(params *driver115.UploadOSSParams, r io.Reader, dir } // UploadByMultipart upload by mutipart blocks -func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize int64, stream model.FileStreamer, dirID string, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { +func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.UploadOSSParams, fileSize int64, s model.FileStreamer, + dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart @@ -313,7 +325,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i err error ) - tmpF, err := stream.CacheFullInTempFile() + tmpF, err := s.CacheFullInTempFile() if err != nil { return nil, err } @@ -372,6 +384,7 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i quit <- struct{}{} }() + completedNum := atomic.Int32{} // consumers for i := 0; i < options.ThreadsNum; i++ { go func(threadId int) { @@ -384,6 +397,8 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i var part oss.UploadPart // 出现错误就继续尝试，共尝试3次 for retry := 0; retry < 3; retry++ { select { + case <-ctx.Done(): + break case <-ticker.C: if ossToken, err = d.client.GetOSSToken(); err != nil { // 到时重新获取ossToken errCh <- errors.Wrap(err, "刷新token时出现错误") @@ -396,12 +411,18 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i continue } - if part, err = bucket.UploadPart(imur, bytes.NewBuffer(buf), chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { + if part, err = bucket.UploadPart(imur, &stream.ReaderWithCtx{ + Reader: bytes.NewBuffer(buf), + Ctx: ctx, + }, chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { break } } if err != nil { - errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误：%v", stream.GetName(), chunk.Number, err)) + errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误：%v", s.GetName(), chunk.Number, err)) + } else { + num := completedNum.Add(1) + up(float64(num) * 100.0 / float64(len(chunks))) } UploadedPartsCh <- part } diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 3828a59d9f0..1bf71ae64a8 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "net/url" @@ -185,32 +186,35 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error { } } -func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - // const DEFAULT int64 = 10485760 - h := md5.New() - // need to calculate md5 of the full content - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - defer func() { - _ = tempFile.Close() - }() - if _, err = utils.CopyWithBuffer(h, tempFile); err != nil { - return err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err +func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + etag := file.GetHash().GetHash(utils.MD5) + if len(etag) < utils.MD5.Width { + // const DEFAULT int64 = 10485760 + h := md5.New() + // need to calculate md5 of the full content + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return err + } + defer func() { + _ = tempFile.Close() + }() + if _, err = utils.CopyWithBuffer(h, tempFile); err != nil { + return err + } + _, err = tempFile.Seek(0, io.SeekStart) + if err != nil { + return err + } + etag = hex.EncodeToString(h.Sum(nil)) } - etag := hex.EncodeToString(h.Sum(nil)) data := base.Json{ "driveId": 0, "duplicate": 2, // 2->覆盖 1->重命名 0->默认 "etag": etag, - "fileName": stream.GetName(), + "fileName": file.GetName(), "parentFileId": dstDir.GetID(), - "size": stream.GetSize(), + "size": file.GetSize(), "type": 0, } var resp UploadResp @@ -225,7 +229,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return nil } if resp.Data.AccessKeyId == "" || resp.Data.SecretAccessKey == "" || resp.Data.SessionToken == "" { - err = d.newUpload(ctx, &resp, stream, tempFile, up) + err = d.newUpload(ctx, &resp, file, up) return err } else { cfg := &aws.Config{ @@ -239,15 +243,21 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } uploader := s3manager.NewUploader(s) - if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) } input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Key, - Body: tempFile, + Body: &stream.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }, } _, err = uploader.UploadWithContext(ctx, input) + if err != nil { + return err + } } _, err = d.Request(UploadComplete, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ diff --git a/drivers/123/upload.go b/drivers/123/upload.go index 66627b4cd94..a472df55240 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -69,7 +69,7 @@ func (d *Pan123) completeS3(ctx context.Context, upReq *UploadResp, file model.F return err } -func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, reader io.Reader, up driver.UpdateProgress) error { +func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, up driver.UpdateProgress) error { chunkSize := int64(1024 * 1024 * 16) // fetch s3 pre signed urls chunkCount := int(math.Ceil(float64(file.GetSize()) / float64(chunkSize))) @@ -103,7 +103,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi if j == chunkCount { curSize = file.GetSize() - (int64(chunkCount)-1)*chunkSize } - err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.LimitReader(reader, chunkSize), curSize, false, getS3UploadUrl) + err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.LimitReader(file, chunkSize), curSize, false, getS3UploadUrl) if err != nil { return err } diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index 894bac64607..679285e0d8f 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -3,6 +3,7 @@ package alist_v3 import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "path" @@ -181,25 +182,28 @@ func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", stream) +func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) if err != nil { return err } req.Header.Set("Authorization", d.Token) - req.Header.Set("File-Path", path.Join(dstDir.GetPath(), stream.GetName())) + req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) req.Header.Set("Password", d.MetaPassword) - if md5 := stream.GetHash().GetHash(utils.MD5); len(md5) > 0 { + if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { req.Header.Set("X-File-Md5", md5) } - if sha1 := stream.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { + if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { req.Header.Set("X-File-Sha1", sha1) } - if sha256 := stream.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { + if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { req.Header.Set("X-File-Sha256", sha256) } - req.ContentLength = stream.GetSize() + req.ContentLength = s.GetSize() // client := base.NewHttpClient() // client.Timeout = time.Hour * 6 res, err := base.HttpClient.Do(req) diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index 360c6e3d01d..9b526f8ac34 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "io" "mime/multipart" "net/http" @@ -215,7 +216,7 @@ func (d *ChaoXing) Remove(ctx context.Context, obj model.Obj) error { return nil } -func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { var resp UploadDataRsp _, err := d.request("https://noteyd.chaoxing.com/pc/files/getUploadConfig", http.MethodGet, func(req *resty.Request) { }, &resp) @@ -227,11 +228,11 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, stream model.FileS } body := &bytes.Buffer{} writer := multipart.NewWriter(body) - filePart, err := writer.CreateFormFile("file", stream.GetName()) + filePart, err := writer.CreateFormFile("file", file.GetName()) if err != nil { return err } - _, err = utils.CopyWithBuffer(filePart, stream) + _, err = utils.CopyWithBuffer(filePart, file) if err != nil { return err } @@ -248,7 +249,14 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, stream model.FileS if err != nil { return err } - req, err := http.NewRequest("POST", "https://pan-yz.chaoxing.com/upload", body) + r := &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: body, + Size: int64(body.Len()), + }, + UpdateProgress: up, + } + req, err := http.NewRequestWithContext(ctx, "POST", "https://pan-yz.chaoxing.com/upload", r) if err != nil { return err } diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index 05b9e49a91d..b3e95f9320f 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -2,6 +2,7 @@ package ftp import ( "context" + "github.com/alist-org/alist/v3/internal/stream" stdpath "path" "github.com/alist-org/alist/v3/internal/driver" @@ -114,13 +115,18 @@ func (d *FTP) Remove(ctx context.Context, obj model.Obj) error { } } -func (d *FTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *FTP) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { if err := d.login(); err != nil { return err } - // TODO: support cancel - path := stdpath.Join(dstDir.GetPath(), stream.GetName()) - return d.conn.Stor(encode(path, d.Encoding), stream) + path := stdpath.Join(dstDir.GetPath(), s.GetName()) + return d.conn.Stor(encode(path, d.Encoding), &stream.ReaderWithCtx{ + Reader: &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }, + Ctx: ctx, + }) } var _ driver.Driver = (*FTP)(nil) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index eed06882984..996c79c74e9 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -16,6 +16,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" @@ -649,15 +650,15 @@ func (d *Github) createGitKeep(path, message string) error { return nil } -func (d *Github) putBlob(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) (string, error) { +func (d *Github) putBlob(ctx context.Context, s model.FileStreamer, up driver.UpdateProgress) (string, error) { beforeContent := "{\"encoding\":\"base64\",\"content\":\"" afterContent := "\"}" - length := int64(len(beforeContent)) + calculateBase64Length(stream.GetSize()) + int64(len(afterContent)) + length := int64(len(beforeContent)) + calculateBase64Length(s.GetSize()) + int64(len(afterContent)) beforeContentReader := strings.NewReader(beforeContent) contentReader, contentWriter := io.Pipe() go func() { encoder := base64.NewEncoder(base64.StdEncoding, contentWriter) - if _, err := utils.CopyWithBuffer(encoder, stream); err != nil { + if _, err := utils.CopyWithBuffer(encoder, s); err != nil { _ = contentWriter.CloseWithError(err) return } @@ -667,10 +668,12 @@ func (d *Github) putBlob(ctx context.Context, stream model.FileStreamer, up driv afterContentReader := strings.NewReader(afterContent) req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("https://api.github.com/repos/%s/%s/git/blobs", d.Owner, d.Repo), - &ReaderWithProgress{ - Reader: io.MultiReader(beforeContentReader, contentReader, afterContentReader), - Length: length, - Progress: up, + &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: io.MultiReader(beforeContentReader, contentReader, afterContentReader), + Size: length, + }, + UpdateProgress: up, }) if err != nil { return "", err diff --git a/drivers/github/util.go b/drivers/github/util.go index 1e7f7fdbf36..85bc3cb9078 100644 --- a/drivers/github/util.go +++ b/drivers/github/util.go @@ -7,26 +7,10 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - "io" - "math" "strings" "text/template" ) -type ReaderWithProgress struct { - Reader io.Reader - Length int64 - Progress func(percentage float64) - offset int64 -} - -func (r *ReaderWithProgress) Read(p []byte) (int, error) { - n, err := r.Reader.Read(p) - r.offset += int64(n) - r.Progress(math.Min(100.0, float64(r.offset)/float64(r.Length)*100.0)) - return n, err -} - type MessageTemplateVars struct { UserName string ObjName string diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 90ef7c1a910..8681fed498e 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "encoding/hex" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "net/url" @@ -266,10 +267,10 @@ func (d *ILanZou) Remove(ctx context.Context, obj model.Obj) error { const DefaultPartSize = 1024 * 1024 * 8 -func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { h := md5.New() // need to calculate md5 of the full content - tempFile, err := stream.CacheFullInTempFile() + tempFile, err := s.CacheFullInTempFile() if err != nil { return nil, err } @@ -288,8 +289,8 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt res, err := d.proved("/7n/getUpToken", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileId": "", - "fileName": stream.GetName(), - "fileSize": stream.GetSize()/1024 + 1, + "fileName": s.GetName(), + "fileSize": s.GetSize()/1024 + 1, "folderId": dstDir.GetID(), "md5": etag, "type": 1, @@ -301,13 +302,20 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt upToken := utils.Json.Get(res, "upToken").ToString() now := time.Now() key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) + reader := &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: tempFile, + Size: s.GetSize(), + }, + UpdateProgress: up, + } var token string - if stream.GetSize() <= DefaultPartSize { - res, err := d.upClient.R().SetMultipartFormData(map[string]string{ + if s.GetSize() <= DefaultPartSize { + res, err := d.upClient.R().SetContext(ctx).SetMultipartFormData(map[string]string{ "token": upToken, "key": key, - "fname": stream.GetName(), - }).SetMultipartField("file", stream.GetName(), stream.GetMimetype(), tempFile). + "fname": s.GetName(), + }).SetMultipartField("file", s.GetName(), s.GetMimetype(), reader). Post("https://upload.qiniup.com/") if err != nil { return nil, err @@ -321,10 +329,10 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt } uploadId := utils.Json.Get(res.Body(), "uploadId").ToString() parts := make([]Part, 0) - partNum := (stream.GetSize() + DefaultPartSize - 1) / DefaultPartSize + partNum := (s.GetSize() + DefaultPartSize - 1) / DefaultPartSize for i := 1; i <= int(partNum); i++ { u := fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s/%d", d.conf.bucket, keyBase64, uploadId, i) - res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(io.LimitReader(tempFile, DefaultPartSize)).Put(u) + res, err = d.upClient.R().SetContext(ctx).SetHeader("Authorization", "UpToken "+upToken).SetBody(io.LimitReader(reader, DefaultPartSize)).Put(u) if err != nil { return nil, err } @@ -335,7 +343,7 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt }) } res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(base.Json{ - "fnmae": stream.GetName(), + "fnmae": s.GetName(), "parts": parts, }).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s", d.conf.bucket, keyBase64, uploadId)) if err != nil { @@ -373,9 +381,9 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt ID: strconv.FormatInt(file.FileId, 10), //Path: , Name: file.FileName, - Size: stream.GetSize(), - Modified: stream.ModTime(), - Ctime: stream.CreateTime(), + Size: s.GetSize(), + Modified: s.ModTime(), + Ctime: s.CreateTime(), IsFolder: false, HashInfo: utils.NewHashInfo(utils.MD5, etag), }, nil diff --git a/drivers/ipfs_api/driver.go b/drivers/ipfs_api/driver.go index f6f81305e20..61886b38b6f 100644 --- a/drivers/ipfs_api/driver.go +++ b/drivers/ipfs_api/driver.go @@ -3,6 +3,7 @@ package ipfs import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "net/url" stdpath "path" "path/filepath" @@ -108,9 +109,15 @@ func (d *IPFS) Remove(ctx context.Context, obj model.Obj) error { return d.sh.FilesRm(ctx, obj.GetPath(), true) } -func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { // TODO upload file, optional - _, err := d.sh.Add(stream, ToFiles(stdpath.Join(dstDir.GetPath(), stream.GetName()))) + _, err := d.sh.Add(&stream.ReaderWithCtx{ + Reader: &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }, + Ctx: ctx, + }, ToFiles(stdpath.Join(dstDir.GetPath(), s.GetName()))) return err } diff --git a/drivers/kodbox/driver.go b/drivers/kodbox/driver.go index eb5120a67c1..ff48ffb21cb 100644 --- a/drivers/kodbox/driver.go +++ b/drivers/kodbox/driver.go @@ -3,6 +3,7 @@ package kodbox import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" "net/http" @@ -225,14 +226,19 @@ func (d *KodBox) Remove(ctx context.Context, obj model.Obj) error { return nil } -func (d *KodBox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (d *KodBox) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { var resp *CommonResp _, err := d.request(http.MethodPost, "/?explorer/upload/fileUpload", func(req *resty.Request) { - req.SetFileReader("file", stream.GetName(), stream). + r := &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + } + req.SetFileReader("file", s.GetName(), r). SetResult(&resp). SetFormData(map[string]string{ "path": dstDir.GetPath(), - }) + }). + SetContext(ctx) }) if err != nil { return nil, err @@ -244,8 +250,8 @@ func (d *KodBox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return &model.ObjThumb{ Object: model.Object{ Path: resp.Info.(string), - Name: stream.GetName(), - Size: stream.GetSize(), + Name: s.GetName(), + Size: s.GetSize(), IsFolder: false, Modified: time.Now(), Ctime: time.Now(), diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index 9e73f0525c2..90635d16349 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -2,6 +2,7 @@ package lanzou import ( "context" + "github.com/alist-org/alist/v3/internal/stream" "net/http" "github.com/alist-org/alist/v3/drivers/base" @@ -208,7 +209,7 @@ func (d *LanZou) Remove(ctx context.Context, obj model.Obj) error { return errs.NotSupport } -func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if d.IsCookie() || d.IsAccount() { var resp RespText[[]FileOrFolder] _, err := d._post(d.BaseUrl+"/html5up.php", func(req *resty.Request) { @@ -217,9 +218,12 @@ func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "vie": "2", "ve": "2", "id": "WU_FILE_0", - "name": stream.GetName(), + "name": s.GetName(), "folder_id_bb_n": dstDir.GetID(), - }).SetFileReader("upload_file", stream.GetName(), stream).SetContext(ctx) + }).SetFileReader("upload_file", s.GetName(), &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }).SetContext(ctx) }, &resp, true) if err != nil { return nil, err diff --git a/drivers/mediatrack/driver.go b/drivers/mediatrack/driver.go index f0f1ded0087..ed53f8ee03a 100644 --- a/drivers/mediatrack/driver.go +++ b/drivers/mediatrack/driver.go @@ -5,6 +5,7 @@ import ( "crypto/md5" "encoding/hex" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "strconv" @@ -161,7 +162,7 @@ func (d *MediaTrack) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { src := "assets/" + uuid.New().String() var resp UploadResp _, err := d.request("https://jayce.api.mediatrack.cn/v3/storage/tokens/asset", http.MethodGet, func(req *resty.Request) { @@ -180,7 +181,7 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - tempFile, err := stream.CacheFullInTempFile() + tempFile, err := file.CacheFullInTempFile() if err != nil { return err } @@ -188,13 +189,19 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil _ = tempFile.Close() }() uploader := s3manager.NewUploader(s) - if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) } input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Object, - Body: tempFile, + Body: &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: tempFile, + Size: file.GetSize(), + }, + UpdateProgress: up, + }, } _, err = uploader.UploadWithContext(ctx, input) if err != nil { @@ -213,12 +220,12 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil hash := hex.EncodeToString(h.Sum(nil)) data := base.Json{ "category": 0, - "description": stream.GetName(), + "description": file.GetName(), "hash": hash, - "mime": stream.GetMimetype(), - "size": stream.GetSize(), + "mime": file.GetMimetype(), + "size": file.GetSize(), "src": src, - "title": stream.GetName(), + "title": file.GetName(), "type": 0, } _, err = d.request(url, http.MethodPost, func(req *resty.Request) { diff --git a/drivers/netease_music/driver.go b/drivers/netease_music/driver.go index c0d103de0d9..08460cceee9 100644 --- a/drivers/netease_music/driver.go +++ b/drivers/netease_music/driver.go @@ -88,7 +88,7 @@ func (d *NeteaseMusic) Remove(ctx context.Context, obj model.Obj) error { } func (d *NeteaseMusic) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - return d.putSongStream(stream) + return d.putSongStream(ctx, stream, up) } func (d *NeteaseMusic) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { diff --git a/drivers/netease_music/types.go b/drivers/netease_music/types.go index 0e156ad1579..332f75e94f6 100644 --- a/drivers/netease_music/types.go +++ b/drivers/netease_music/types.go @@ -2,6 +2,7 @@ package netease_music import ( "context" + "github.com/alist-org/alist/v3/internal/driver" "io" "net/http" "strconv" @@ -71,6 +72,8 @@ func (lrc *LyricObj) getLyricLink() *model.Link { type ReqOption struct { crypto string stream model.FileStreamer + up driver.UpdateProgress + ctx context.Context data map[string]string headers map[string]string cookies []*http.Cookie @@ -113,3 +116,16 @@ func (ch *Characteristic) merge(data map[string]string) map[string]interface{} { } return body } + +type InlineReadCloser struct { + io.Reader + io.Closer +} + +func (rc *InlineReadCloser) Read(p []byte) (int, error) { + return rc.Reader.Read(p) +} + +func (rc *InlineReadCloser) Close() error { + return rc.Closer.Close() +} diff --git a/drivers/netease_music/upload.go b/drivers/netease_music/upload.go index 7f580bd1744..3ff6216b7b9 100644 --- a/drivers/netease_music/upload.go +++ b/drivers/netease_music/upload.go @@ -1,8 +1,10 @@ package netease_music import ( + "context" "crypto/md5" "encoding/hex" + "github.com/alist-org/alist/v3/internal/driver" "io" "net/http" "strconv" @@ -47,9 +49,12 @@ func (u *uploader) init(stream model.FileStreamer) error { } h := md5.New() - utils.CopyWithBuffer(h, stream) + _, err := utils.CopyWithBuffer(h, stream) + if err != nil { + return err + } u.md5 = hex.EncodeToString(h.Sum(nil)) - _, err := u.file.Seek(0, io.SeekStart) + _, err = u.file.Seek(0, io.SeekStart) if err != nil { return err } @@ -167,7 +172,7 @@ func (u *uploader) publishInfo(resourceId string) error { return nil } -func (u *uploader) upload(stream model.FileStreamer) error { +func (u *uploader) upload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) error { bucket := "jd-musicrep-privatecloud-audio-public" token, err := u.allocToken(bucket) if err != nil { @@ -192,6 +197,8 @@ func (u *uploader) upload(stream model.FileStreamer) error { http.MethodPost, ReqOption{ stream: stream, + up: up, + ctx: ctx, headers: map[string]string{ "x-nos-token": token.token, "Content-Type": "audio/mpeg", diff --git a/drivers/netease_music/util.go b/drivers/netease_music/util.go index 4d0696eb82b..25efde77b9d 100644 --- a/drivers/netease_music/util.go +++ b/drivers/netease_music/util.go @@ -1,7 +1,9 @@ package netease_music import ( - "io" + "context" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/stream" "net/http" "path" "regexp" @@ -58,20 +60,38 @@ func (d *NeteaseMusic) request(url, method string, opt ReqOption) ([]byte, error url = "https://music.163.com/api/linux/forward" } + if opt.ctx != nil { + req.SetContext(opt.ctx) + } if method == http.MethodPost { if opt.stream != nil { + if opt.up == nil { + opt.up = func(_ float64) {} + } req.SetContentLength(true) - req.SetBody(io.ReadCloser(opt.stream)) + req.SetBody(&InlineReadCloser{ + Reader: &stream.ReaderUpdatingProgress{ + Reader: opt.stream, + UpdateProgress: opt.up, + }, + Closer: opt.stream, + }) } else { req.SetFormData(data) } res, err := req.Post(url) - return res.Body(), err + if err != nil { + return nil, err + } + return res.Body(), nil } if method == http.MethodGet { res, err := req.Get(url) - return res.Body(), err + if err != nil { + return nil, err + } + return res.Body(), nil } return nil, errs.NotImplement @@ -206,7 +226,7 @@ func (d *NeteaseMusic) removeSongObj(file model.Obj) error { return err } -func (d *NeteaseMusic) putSongStream(stream model.FileStreamer) error { +func (d *NeteaseMusic) putSongStream(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) error { tmp, err := stream.CacheFullInTempFile() if err != nil { return err @@ -231,7 +251,7 @@ func (d *NeteaseMusic) putSongStream(stream model.FileStreamer) error { } if u.meta.needUpload { - err = u.upload(stream) + err = u.upload(ctx, stream, up) if err != nil { return err } diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 3db273d652b..504b1d0e9f9 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -255,10 +255,10 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } if stream.GetSize() <= 10*utils.MB { // 文件大小 小于10MB，改用普通模式上传 - return d.UploadByOSS(¶ms, stream, up) + return d.UploadByOSS(ctx, ¶ms, stream, up) } // 分片上传 - return d.UploadByMultipart(¶ms, stream.GetSize(), stream, up) + return d.UploadByMultipart(ctx, ¶ms, stream.GetSize(), stream, up) } // 离线下载文件 diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index e8f3c854533..eb96a42ad29 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -2,6 +2,7 @@ package pikpak import ( "bytes" + "context" "crypto/md5" "crypto/sha1" "encoding/hex" @@ -9,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" jsoniter "github.com/json-iterator/go" @@ -19,6 +21,7 @@ import ( "regexp" "strings" "sync" + "sync/atomic" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -417,7 +420,7 @@ func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) err return nil } -func (d *PikPak) UploadByOSS(params *S3Params, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *PikPak) UploadByOSS(ctx context.Context, params *S3Params, s model.FileStreamer, up driver.UpdateProgress) error { ossClient, err := oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret) if err != nil { return err @@ -427,14 +430,20 @@ func (d *PikPak) UploadByOSS(params *S3Params, stream model.FileStreamer, up dri return err } - err = bucket.PutObject(params.Key, stream, OssOption(params)...) + err = bucket.PutObject(params.Key, &stream.ReaderWithCtx{ + Reader: &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }, + Ctx: ctx, + }, OssOption(params)...) if err != nil { return err } return nil } -func (d *PikPak) UploadByMultipart(params *S3Params, fileSize int64, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSize int64, s model.FileStreamer, up driver.UpdateProgress) error { var ( chunks []oss.FileChunk parts []oss.UploadPart @@ -444,7 +453,7 @@ func (d *PikPak) UploadByMultipart(params *S3Params, fileSize int64, stream mode err error ) - tmpF, err := stream.CacheFullInTempFile() + tmpF, err := s.CacheFullInTempFile() if err != nil { return err } @@ -488,6 +497,7 @@ func (d *PikPak) UploadByMultipart(params *S3Params, fileSize int64, stream mode quit <- struct{}{} }() + completedNum := atomic.Int32{} // consumers for i := 0; i < ThreadsNum; i++ { go func(threadId int) { @@ -500,6 +510,8 @@ func (d *PikPak) UploadByMultipart(params *S3Params, fileSize int64, stream mode var part oss.UploadPart // 出现错误就继续尝试，共尝试3次 for retry := 0; retry < 3; retry++ { select { + case <-ctx.Done(): + break case <-ticker.C: errCh <- errors.Wrap(err, "ossToken 过期") default: @@ -511,12 +523,18 @@ func (d *PikPak) UploadByMultipart(params *S3Params, fileSize int64, stream mode } b := bytes.NewBuffer(buf) - if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, OssOption(params)...); err == nil { + if part, err = bucket.UploadPart(imur, &stream.ReaderWithCtx{ + Reader: b, + Ctx: ctx, + }, chunk.Size, chunk.Number, OssOption(params)...); err == nil { break } } if err != nil { - errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误：%v", stream.GetName(), chunk.Number, err)) + errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误：%v", s.GetName(), chunk.Number, err)) + } else { + num := completedNum.Add(1) + up(float64(num) * 100.0 / float64(len(chunks))) } UploadedPartsCh <- part } @@ -547,7 +565,7 @@ LOOP: // EOF错误是xml的Unmarshal导致的，响应其实是json格式，所以实际上上传是成功的 if _, err = bucket.CompleteMultipartUpload(imur, parts, OssOption(params)...); err != nil && !errors.Is(err, io.EOF) { // 当文件名含有 &< 这两个字符之一时响应的xml解析会出现错误，实际上上传是成功的 - if filename := filepath.Base(stream.GetName()); !strings.ContainsAny(filename, "&<") { + if filename := filepath.Base(s.GetName()); !strings.ContainsAny(filename, "&<") { return err } } diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 51e54981a18..2ab972caff7 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -3,6 +3,7 @@ package quqi import ( "bytes" "context" + "errors" "io" "strconv" "strings" @@ -11,6 +12,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + istream "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" "github.com/aws/aws-sdk-go/aws" @@ -385,9 +387,16 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea } uploader := s3manager.NewUploader(s) buf := make([]byte, 1024*1024*2) + fup := &istream.ReaderUpdatingProgress{ + Reader: &istream.SimpleReaderWithSize{ + Reader: f, + Size: int64(len(buf)), + }, + UpdateProgress: up, + } for partNumber := int64(1); ; partNumber++ { - n, err := io.ReadFull(f, buf) - if err != nil && err != io.ErrUnexpectedEOF { + n, err := io.ReadFull(fup, buf) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { if err == io.EOF { break } diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 82c050a1fe8..a7e924e2e8c 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -163,18 +163,21 @@ func (d *S3) Remove(ctx context.Context, obj model.Obj) error { return d.removeFile(obj.GetPath()) } -func (d *S3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { uploader := s3manager.NewUploader(d.Session) - if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + if s.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = s.GetSize() / (s3manager.MaxUploadParts - 1) } - key := getKey(stdpath.Join(dstDir.GetPath(), stream.GetName()), false) - contentType := stream.GetMimetype() + key := getKey(stdpath.Join(dstDir.GetPath(), s.GetName()), false) + contentType := s.GetMimetype() log.Debugln("key:", key) input := &s3manager.UploadInput{ - Bucket: &d.Bucket, - Key: &key, - Body: stream, + Bucket: &d.Bucket, + Key: &key, + Body: &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }, ContentType: &contentType, } _, err := uploader.UploadWithContext(ctx, input) diff --git a/drivers/seafile/driver.go b/drivers/seafile/driver.go index 6d1f16dad3b..f23038d151d 100644 --- a/drivers/seafile/driver.go +++ b/drivers/seafile/driver.go @@ -3,6 +3,7 @@ package seafile import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "net/http" "strings" "time" @@ -197,7 +198,7 @@ func (d *Seafile) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { repo, path, err := d.getRepoAndPath(dstDir.GetPath()) if err != nil { return err @@ -214,11 +215,16 @@ func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt u := string(res) u = u[1 : len(u)-1] // remove quotes _, err = d.request(http.MethodPost, u, func(req *resty.Request) { - req.SetFileReader("file", stream.GetName(), stream). + r := &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + } + req.SetFileReader("file", s.GetName(), r). SetFormData(map[string]string{ "parent_dir": path, "replace": "1", - }) + }). + SetContext(ctx) }) return err } diff --git a/drivers/template/driver.go b/drivers/template/driver.go index 439f57f35f9..ff3648db3b0 100644 --- a/drivers/template/driver.go +++ b/drivers/template/driver.go @@ -66,11 +66,33 @@ func (d *Template) Remove(ctx context.Context, obj model.Obj) error { return errs.NotImplement } -func (d *Template) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (d *Template) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // TODO upload file, optional return nil, errs.NotImplement } +func (d *Template) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Template) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Template) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Template) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 8403f2617a6..1b7f0af6a33 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -3,6 +3,7 @@ package thunder import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "net/http" "strconv" "strings" @@ -332,16 +333,16 @@ func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error { return err } -func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - hi := stream.GetHash() +func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + hi := file.GetHash() gcid := hi.GetHash(hash_extend.GCID) if len(gcid) < hash_extend.GCID.Width { - tFile, err := stream.CacheFullInTempFile() + tFile, err := file.CacheFullInTempFile() if err != nil { return err } - gcid, err = utils.HashFile(hash_extend.GCID, tFile, stream.GetSize()) + gcid, err = utils.HashFile(hash_extend.GCID, tFile, file.GetSize()) if err != nil { return err } @@ -353,8 +354,8 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, stream model. r.SetBody(&base.Json{ "kind": FILE, "parent_id": dstDir.GetID(), - "name": stream.GetName(), - "size": stream.GetSize(), + "name": file.GetName(), + "size": file.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, }) @@ -375,14 +376,17 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, stream model. return err } uploader := s3manager.NewUploader(s) - if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) } _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), - Body: stream, + Body: &stream.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }, }) return err } diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index b9ee668c2f9..93e07ca98ea 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -8,6 +8,7 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" @@ -363,16 +364,16 @@ func (xc *XunLeiXCommon) Remove(ctx context.Context, obj model.Obj) error { return err } -func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - hi := stream.GetHash() +func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + hi := file.GetHash() gcid := hi.GetHash(hash_extend.GCID) if len(gcid) < hash_extend.GCID.Width { - tFile, err := stream.CacheFullInTempFile() + tFile, err := file.CacheFullInTempFile() if err != nil { return err } - gcid, err = utils.HashFile(hash_extend.GCID, tFile, stream.GetSize()) + gcid, err = utils.HashFile(hash_extend.GCID, tFile, file.GetSize()) if err != nil { return err } @@ -384,8 +385,8 @@ func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, stream model r.SetBody(&base.Json{ "kind": FILE, "parent_id": dstDir.GetID(), - "name": stream.GetName(), - "size": stream.GetSize(), + "name": file.GetName(), + "size": file.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, }) @@ -406,14 +407,17 @@ func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, stream model return err } uploader := s3manager.NewUploader(s) - if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) } _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), - Body: stream, + Body: &stream.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }, }) return err } diff --git a/drivers/trainbit/driver.go b/drivers/trainbit/driver.go index 795b2fb8a2e..2b1815ed66f 100644 --- a/drivers/trainbit/driver.go +++ b/drivers/trainbit/driver.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "net/url" @@ -114,23 +115,18 @@ func (d *Trainbit) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { endpoint, _ := url.Parse("https://tb28.trainbit.com/api/upload/send_raw/") query := &url.Values{} query.Add("q", strings.Split(dstDir.GetID(), "_")[1]) query.Add("guid", guid) - query.Add("name", url.QueryEscape(local2provider(stream.GetName(), false)+".")) + query.Add("name", url.QueryEscape(local2provider(s.GetName(), false)+".")) endpoint.RawQuery = query.Encode() - var total int64 - total = 0 - progressReader := &ProgressReader{ - stream, - func(byteNum int) { - total += int64(byteNum) - up(float64(total) / float64(stream.GetSize()) * 100) - }, + progressReader := &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, } - req, err := http.NewRequest(http.MethodPost, endpoint.String(), progressReader) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), progressReader) if err != nil { return err } diff --git a/drivers/trainbit/util.go b/drivers/trainbit/util.go index afc111a8290..486e88516ec 100644 --- a/drivers/trainbit/util.go +++ b/drivers/trainbit/util.go @@ -13,17 +13,6 @@ import ( "github.com/alist-org/alist/v3/internal/model" ) -type ProgressReader struct { - io.Reader - reporter func(byteNum int) -} - -func (progressReader *ProgressReader) Read(data []byte) (int, error) { - byteNum, err := progressReader.Reader.Read(data) - progressReader.reporter(byteNum) - return byteNum, err -} - func get(url string, apiKey string, AUSHELLPORTAL string) (*http.Response, error) { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { diff --git a/drivers/uss/driver.go b/drivers/uss/driver.go index 447515d8d36..3c54797c43e 100644 --- a/drivers/uss/driver.go +++ b/drivers/uss/driver.go @@ -3,6 +3,7 @@ package uss import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "net/url" "path" "strings" @@ -122,11 +123,16 @@ func (d *USS) Remove(ctx context.Context, obj model.Obj) error { }) } -func (d *USS) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - // TODO not support cancel?? +func (d *USS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { return d.client.Put(&upyun.PutObjectConfig{ - Path: getKey(path.Join(dstDir.GetPath(), stream.GetName()), false), - Reader: stream, + Path: getKey(path.Join(dstDir.GetPath(), s.GetName()), false), + Reader: &stream.ReaderWithCtx{ + Reader: &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }, + Ctx: ctx, + }, }) } diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index b402b1db0fa..35240c498e8 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -2,6 +2,7 @@ package webdav import ( "context" + "github.com/alist-org/alist/v3/internal/stream" "net/http" "os" "path" @@ -93,13 +94,18 @@ func (d *WebDav) Remove(ctx context.Context, obj model.Obj) error { return d.client.RemoveAll(getPath(obj)) } -func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { callback := func(r *http.Request) { - r.Header.Set("Content-Type", stream.GetMimetype()) - r.ContentLength = stream.GetSize() + r.Header.Set("Content-Type", s.GetMimetype()) + r.ContentLength = s.GetSize() } - // TODO: support cancel - err := d.client.WriteStream(path.Join(dstDir.GetPath(), stream.GetName()), stream, 0644, callback) + err := d.client.WriteStream(path.Join(dstDir.GetPath(), s.GetName()), &stream.ReaderWithCtx{ + Reader: &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }, + Ctx: ctx, + }, 0644, callback) return err } diff --git a/drivers/weiyun/driver.go b/drivers/weiyun/driver.go index e6d5897c313..59bd7237088 100644 --- a/drivers/weiyun/driver.go +++ b/drivers/weiyun/driver.go @@ -7,6 +7,7 @@ import ( "math" "net/http" "strconv" + "sync/atomic" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -311,77 +312,82 @@ func (d *WeiYun) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr // NOTE: // 秒传需要sha1最后一个状态,但sha1无法逆运算需要读完整个文件(或许可以??) // 服务器支持上传进度恢复,不需要额外实现 - if folder, ok := dstDir.(*Folder); ok { - file, err := stream.CacheFullInTempFile() - if err != nil { - return nil, err - } + var folder *Folder + var ok bool + if folder, ok = dstDir.(*Folder); !ok { + return nil, errs.NotSupport + } + file, err := stream.CacheFullInTempFile() + if err != nil { + return nil, err + } - // step 1. - preData, err := d.client.PreUpload(ctx, weiyunsdkgo.UpdloadFileParam{ - PdirKey: folder.GetPKey(), - DirKey: folder.DirKey, + // step 1. + preData, err := d.client.PreUpload(ctx, weiyunsdkgo.UpdloadFileParam{ + PdirKey: folder.GetPKey(), + DirKey: folder.DirKey, - FileName: stream.GetName(), - FileSize: stream.GetSize(), - File: file, + FileName: stream.GetName(), + FileSize: stream.GetSize(), + File: file, - ChannelCount: 4, - FileExistOption: 1, - }) - if err != nil { - return nil, err - } - - // not fast upload - if !preData.FileExist { - // step.2 增加上传通道 - if len(preData.ChannelList) < d.uploadThread { - newCh, err := d.client.AddUploadChannel(len(preData.ChannelList), d.uploadThread, preData.UploadAuthData) - if err != nil { - return nil, err - } - preData.ChannelList = append(preData.ChannelList, newCh.AddChannels...) - } - // step.3 上传 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, len(preData.ChannelList), - retry.Attempts(3), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - - for _, channel := range preData.ChannelList { - if utils.IsCanceled(upCtx) { - break - } + ChannelCount: 4, + FileExistOption: 1, + }) + if err != nil { + return nil, err + } - var channel = channel - threadG.Go(func(ctx context.Context) error { - for { - channel.Len = int(math.Min(float64(stream.GetSize()-channel.Offset), float64(channel.Len))) - upData, err := d.client.UploadFile(upCtx, channel, preData.UploadAuthData, - io.NewSectionReader(file, channel.Offset, int64(channel.Len))) - if err != nil { - return err - } - // 上传完成 - if upData.UploadState != 1 { - return nil - } - channel = upData.Channel - } - }) - } - if err = threadG.Wait(); err != nil { + // not fast upload + if !preData.FileExist { + // step.2 增加上传通道 + if len(preData.ChannelList) < d.uploadThread { + newCh, err := d.client.AddUploadChannel(len(preData.ChannelList), d.uploadThread, preData.UploadAuthData) + if err != nil { return nil, err } + preData.ChannelList = append(preData.ChannelList, newCh.AddChannels...) } + // step.3 上传 + threadG, upCtx := errgroup.NewGroupWithContext(ctx, len(preData.ChannelList), + retry.Attempts(3), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + total := atomic.Int64{} + for _, channel := range preData.ChannelList { + if utils.IsCanceled(upCtx) { + break + } - return &File{ - PFolder: folder, - File: preData.File, - }, nil + var channel = channel + threadG.Go(func(ctx context.Context) error { + for { + channel.Len = int(math.Min(float64(stream.GetSize()-channel.Offset), float64(channel.Len))) + upData, err := d.client.UploadFile(upCtx, channel, preData.UploadAuthData, + io.NewSectionReader(file, channel.Offset, int64(channel.Len))) + if err != nil { + return err + } + cur := total.Add(int64(channel.Len)) + up(float64(cur) * 100.0 / float64(stream.GetSize())) + // 上传完成 + if upData.UploadState != 1 { + return nil + } + channel = upData.Channel + } + }) + } + if err = threadG.Wait(); err != nil { + return nil, err + } } - return nil, errs.NotSupport + + return &File{ + PFolder: folder, + File: preData.File, + }, nil } // func (d *WeiYun) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index bccce4b1c0a..86093fc14fa 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -161,6 +161,7 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre OnProgress: func(current, total int64) { up(100 * float64(current) / float64(total)) }, + Ctx: ctx, }) return err } diff --git a/drivers/yandex_disk/driver.go b/drivers/yandex_disk/driver.go index 5af9f2e4fb0..fe858519a48 100644 --- a/drivers/yandex_disk/driver.go +++ b/drivers/yandex_disk/driver.go @@ -2,6 +2,7 @@ package yandex_disk import ( "context" + "github.com/alist-org/alist/v3/internal/stream" "net/http" "path" "strconv" @@ -106,25 +107,30 @@ func (d *YandexDisk) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *YandexDisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *YandexDisk) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { var resp UploadResp _, err := d.request("/upload", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "path": path.Join(dstDir.GetPath(), stream.GetName()), + "path": path.Join(dstDir.GetPath(), s.GetName()), "overwrite": "true", }) }, &resp) if err != nil { return err } - req, err := http.NewRequest(resp.Method, resp.Href, stream) + req, err := http.NewRequestWithContext(ctx, resp.Method, resp.Href, &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) if err != nil { return err } - req = req.WithContext(ctx) - req.Header.Set("Content-Length", strconv.FormatInt(stream.GetSize(), 10)) + req.Header.Set("Content-Length", strconv.FormatInt(s.GetSize(), 10)) req.Header.Set("Content-Type", "application/octet-stream") res, err := base.HttpClient.Do(req) + if err != nil { + return err + } _ = res.Body.Close() return err } diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 09fd42e7658..292f8e6a480 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -77,7 +77,7 @@ type Remove interface { } type Put interface { - Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error + Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up UpdateProgress) error } type PutURL interface { @@ -113,7 +113,7 @@ type CopyResult interface { } type PutResult interface { - Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error) + Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up UpdateProgress) (model.Obj, error) } type PutURLResult interface { @@ -159,7 +159,7 @@ type ArchiveDecompressResult interface { ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) } -type UpdateProgress model.UpdateProgress +type UpdateProgress = model.UpdateProgress type Progress struct { Total int64 diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 1962fb46745..74646bfbccd 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -562,3 +562,17 @@ func (f *FileReadAtSeeker) Seek(offset int64, whence int) (int64, error) { func (f *FileReadAtSeeker) Close() error { return f.ss.Close() } + +type ReaderWithCtx struct { + io.Reader + Ctx context.Context +} + +func (r *ReaderWithCtx) Read(p []byte) (n int, err error) { + select { + case <-r.Ctx.Done(): + return 0, r.Ctx.Err() + default: + return r.Reader.Read(p) + } +} From 39bde328ee10e226a6ee1689c3ae979ee10566cb Mon Sep 17 00:00:00 2001 From: Sakana Date: Sat, 1 Feb 2025 17:32:58 +0800 Subject: [PATCH 439/659] fix(lenovonas_share): the size of the directory (#7914) --- drivers/lenovonas_share/types.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/lenovonas_share/types.go b/drivers/lenovonas_share/types.go index 77b966d3bee..37ff1465524 100644 --- a/drivers/lenovonas_share/types.go +++ b/drivers/lenovonas_share/types.go @@ -47,7 +47,11 @@ func (f File) GetPath() string { } func (f File) GetSize() int64 { - return f.Size + if f.IsDir() { + return 0 + } else { + return f.Size + } } func (f File) GetName() string { @@ -70,10 +74,6 @@ func (f File) GetID() string { return f.GetPath() } -func (f File) Thumb() string { - return "" -} - type Files struct { Data struct { List []File `json:"list"` From 6164e4577b68caa53da3e85e29a2a24244f4022f Mon Sep 17 00:00:00 2001 From: hshpy Date: Wed, 5 Feb 2025 19:22:10 +0800 Subject: [PATCH 440/659] fix: missing args when using alias driver (#7941 close #7932) --- server/handles/fsread.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 7c580f635e4..0a62f1ffba2 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -303,9 +303,10 @@ func FsGet(c *gin.Context) { } else { // if storage is not proxy, use raw url by fs.Link link, _, err := fs.Link(c, reqPath, model.LinkArgs{ - IP: c.ClientIP(), - Header: c.Request.Header, - HttpReq: c.Request, + IP: c.ClientIP(), + Header: c.Request.Header, + HttpReq: c.Request, + Redirect: true, }) if err != nil { common.ErrorResp(c, err, 500) From f7958077532be308a4c38d7adb27e4a2e4cc10a1 Mon Sep 17 00:00:00 2001 From: Sakana Date: Sun, 9 Feb 2025 18:30:38 +0800 Subject: [PATCH 441/659] feat(github_releases): support dir size for show all version (#7938) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor * 修改默认 RepoStructure * feat: 支持使用 gh-proxy --- drivers/github_releases/driver.go | 156 ++++++++++++---------- drivers/github_releases/meta.go | 3 +- drivers/github_releases/models.go | 86 ++++++++++++ drivers/github_releases/types.go | 201 ++++++++++++++++++++++++---- drivers/github_releases/util.go | 210 ++++++------------------------ 5 files changed, 385 insertions(+), 271 deletions(-) create mode 100644 drivers/github_releases/models.go diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index 79f2b582146..b35aa57a41c 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -4,8 +4,6 @@ import ( "context" "fmt" "net/http" - "time" - "strings" "github.com/alist-org/alist/v3/internal/driver" @@ -18,7 +16,7 @@ type GithubReleases struct { model.Storage Addition - releases []Release + points []MountPoint } func (d *GithubReleases) Config() driver.Config { @@ -30,17 +28,11 @@ func (d *GithubReleases) GetAddition() driver.Additional { } func (d *GithubReleases) Init(ctx context.Context) error { - SetHeader(d.Addition.Token) - repos, err := ParseRepos(d.Addition.RepoStructure, d.Addition.ShowAllVersion) - if err != nil { - return err - } - d.releases = repos + d.ParseRepos(d.Addition.RepoStructure) return nil } func (d *GithubReleases) Drop(ctx context.Context) error { - ClearCache() return nil } @@ -48,67 +40,83 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis files := make([]File, 0) path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) - for _, repo := range d.releases { - if repo.Path == path { // 与仓库路径相同 - resp, err := GetRepoReleaseInfo(repo.RepoName, repo.ID, path, d.Storage.CacheExpiration) - if err != nil { - return nil, err - } - files = append(files, resp.Files...) + for i := range d.points { + point := &d.points[i] - if d.Addition.ShowReadme { - resp, err := GetGithubOtherFile(repo.RepoName, path, d.Storage.CacheExpiration) - if err != nil { - return nil, err + if !d.Addition.ShowAllVersion { // latest + point.RequestRelease(d.GetRequest, args.Refresh) + + if point.Point == path { // 与仓库路径相同 + files = append(files, point.GetLatestRelease()...) + if d.Addition.ShowReadme { + files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir == "" { + continue } - files = append(files, *resp...) - } - } else if strings.HasPrefix(repo.Path, path) { // 仓库路径是目录的子目录 - nextDir := GetNextDir(repo.Path, path) - if nextDir == "" { - continue - } - if d.Addition.ShowAllVersion { - files = append(files, File{ - FileName: nextDir, - Size: 0, - CreateAt: time.Time{}, - UpdateAt: time.Time{}, - Url: "", - Type: "dir", - Path: fmt.Sprintf("%s/%s", path, nextDir), - }) - continue + hasSameDir := false + for index := range files { + if files[index].GetName() == nextDir { + hasSameDir = true + files[index].Size += point.GetLatestSize() + break + } + } + if !hasSameDir { + files = append(files, File{ + Path: path + "/" + nextDir, + FileName: nextDir, + Size: point.GetLatestSize(), + UpdateAt: point.Release.PublishedAt, + CreateAt: point.Release.CreatedAt, + Type: "dir", + Url: "", + }) + } } + } else { // all version + point.RequestReleases(d.GetRequest, args.Refresh) - repo, _ := GetRepoReleaseInfo(repo.RepoName, repo.Version, path, d.Storage.CacheExpiration) - - hasSameDir := false - for index, file := range files { - if file.FileName == nextDir { - hasSameDir = true - files[index].Size += repo.Size - files[index].UpdateAt = func(a time.Time, b time.Time) time.Time { - if a.After(b) { - return a - } - return b - }(files[index].UpdateAt, repo.UpdateAt) - break + if point.Point == path { // 与仓库路径相同 + files = append(files, point.GetAllVersion()...) + if d.Addition.ShowReadme { + files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir == "" { + continue } - } - if !hasSameDir { - files = append(files, File{ - FileName: nextDir, - Size: repo.Size, - CreateAt: repo.CreateAt, - UpdateAt: repo.UpdateAt, - Url: repo.Url, - Type: "dir", - Path: fmt.Sprintf("%s/%s", path, nextDir), - }) + hasSameDir := false + for index := range files { + if files[index].GetName() == nextDir { + hasSameDir = true + files[index].Size += point.GetAllVersionSize() + break + } + } + if !hasSameDir { + files = append(files, File{ + FileName: nextDir, + Path: path + "/" + nextDir, + Size: point.GetAllVersionSize(), + UpdateAt: (*point.Releases)[0].PublishedAt, + CreateAt: (*point.Releases)[0].CreatedAt, + Type: "dir", + Url: "", + }) + } + } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 + tagName := GetNextDir(path, point.Point) + if tagName == "" { + continue + } + + files = append(files, point.GetReleaseByTagName(tagName)...) } } } @@ -119,35 +127,41 @@ func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.Lis } func (d *GithubReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url := file.GetID() + gh_proxy := strings.TrimSpace(d.Addition.GitHubProxy) + + if gh_proxy != "" { + url = strings.Replace(url, "https://github.com", gh_proxy, 1) + } + link := model.Link{ - URL: file.GetID(), + URL: url, Header: http.Header{}, } return &link, nil } func (d *GithubReleases) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + // TODO create folder, optional return nil, errs.NotImplement } func (d *GithubReleases) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO move obj, optional return nil, errs.NotImplement } func (d *GithubReleases) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + // TODO rename obj, optional return nil, errs.NotImplement } func (d *GithubReleases) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional return nil, errs.NotImplement } func (d *GithubReleases) Remove(ctx context.Context, obj model.Obj) error { + // TODO remove obj, optional return errs.NotImplement } - -func (d *GithubReleases) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - return nil, errs.NotImplement -} - -var _ driver.Driver = (*GithubReleases)(nil) diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go index ca6ca5dc8d0..47b84d37927 100644 --- a/drivers/github_releases/meta.go +++ b/drivers/github_releases/meta.go @@ -7,10 +7,11 @@ import ( type Addition struct { driver.RootID - RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"/path/to/alist-gh:alistGo/alist\n/path/to2/alist-web-gh:AlistGo/alist-web" help:"structure:[path:]org/repo"` + RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"` ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` + GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` } var config = driver.Config{ diff --git a/drivers/github_releases/models.go b/drivers/github_releases/models.go new file mode 100644 index 00000000000..a9a0e493c44 --- /dev/null +++ b/drivers/github_releases/models.go @@ -0,0 +1,86 @@ +package github_releases + +type Release struct { + Url string `json:"url"` + AssetsUrl string `json:"assets_url"` + UploadUrl string `json:"upload_url"` + HtmlUrl string `json:"html_url"` + Id int `json:"id"` + Author User `json:"author"` + NodeId string `json:"node_id"` + TagName string `json:"tag_name"` + TargetCommitish string `json:"target_commitish"` + Name string `json:"name"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + CreatedAt string `json:"created_at"` + PublishedAt string `json:"published_at"` + Assets []Asset `json:"assets"` + TarballUrl string `json:"tarball_url"` + ZipballUrl string `json:"zipball_url"` + Body string `json:"body"` + Reactions Reactions `json:"reactions"` +} + +type User struct { + Login string `json:"login"` + Id int `json:"id"` + NodeId string `json:"node_id"` + AvatarUrl string `json:"avatar_url"` + GravatarId string `json:"gravatar_id"` + Url string `json:"url"` + HtmlUrl string `json:"html_url"` + FollowersUrl string `json:"followers_url"` + FollowingUrl string `json:"following_url"` + GistsUrl string `json:"gists_url"` + StarredUrl string `json:"starred_url"` + SubscriptionsUrl string `json:"subscriptions_url"` + OrganizationsUrl string `json:"organizations_url"` + ReposUrl string `json:"repos_url"` + EventsUrl string `json:"events_url"` + ReceivedEventsUrl string `json:"received_events_url"` + Type string `json:"type"` + UserViewType string `json:"user_view_type"` + SiteAdmin bool `json:"site_admin"` +} + +type Asset struct { + Url string `json:"url"` + Id int `json:"id"` + NodeId string `json:"node_id"` + Name string `json:"name"` + Label string `json:"label"` + Uploader User `json:"uploader"` + ContentType string `json:"content_type"` + State string `json:"state"` + Size int64 `json:"size"` + DownloadCount int `json:"download_count"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + BrowserDownloadUrl string `json:"browser_download_url"` +} + +type Reactions struct { + Url string `json:"url"` + TotalCount int `json:"total_count"` + PlusOne int `json:"+1"` + MinusOne int `json:"-1"` + Laugh int `json:"laugh"` + Hooray int `json:"hooray"` + Confused int `json:"confused"` + Heart int `json:"heart"` + Rocket int `json:"rocket"` + Eyes int `json:"eyes"` +} + +type FileInfo struct { + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + Size int64 `json:"size"` + Url string `json:"url"` + HtmlUrl string `json:"html_url"` + GitUrl string `json:"git_url"` + DownloadUrl string `json:"download_url"` + Type string `json:"type"` +} diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index 733460dca5f..b0a9ee619e0 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -1,19 +1,181 @@ package github_releases import ( + "encoding/json" + "strings" "time" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" ) +type MountPoint struct { + Point string // 挂载点 + Repo string // 仓库名 owner/repo + Release *Release // Release 指针 latest + Releases *[]Release // []Release 指针 + OtherFile *[]FileInfo // 仓库根目录下的其他文件 +} + +// 请求最新版本 +func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) { + if m.Repo == "" { + return + } + + if m.Release == nil || refresh { + resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") + m.Release = new(Release) + json.Unmarshal(resp.Body(), m.Release) + } +} + +// 请求所有版本 +func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) { + if m.Repo == "" { + return + } + + if m.Releases == nil || refresh { + resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases") + m.Releases = new([]Release) + json.Unmarshal(resp.Body(), m.Releases) + } +} + +// 获取最新版本 +func (m *MountPoint) GetLatestRelease() []File { + files := make([]File, 0) + for _, asset := range m.Release.Assets { + files = append(files, File{ + Path: m.Point + "/" + asset.Name, + FileName: asset.Name, + Size: asset.Size, + Type: "file", + UpdateAt: asset.UpdatedAt, + CreateAt: asset.CreatedAt, + Url: asset.BrowserDownloadUrl, + }) + } + return files +} + +// 获取最新版本大小 +func (m *MountPoint) GetLatestSize() int64 { + size := int64(0) + for _, asset := range m.Release.Assets { + size += asset.Size + } + return size +} + +// 获取所有版本 +func (m *MountPoint) GetAllVersion() []File { + files := make([]File, 0) + for _, release := range *m.Releases { + file := File{ + Path: m.Point + "/" + release.TagName, + FileName: release.TagName, + Size: m.GetSizeByTagName(release.TagName), + Type: "dir", + UpdateAt: release.PublishedAt, + CreateAt: release.CreatedAt, + Url: release.HtmlUrl, + } + for _, asset := range release.Assets { + file.Size += asset.Size + } + files = append(files, file) + } + return files +} + +// 根据版本号获取版本 +func (m *MountPoint) GetReleaseByTagName(tagName string) []File { + for _, item := range *m.Releases { + if item.TagName == tagName { + files := make([]File, 0) + for _, asset := range item.Assets { + files = append(files, File{ + Path: m.Point + "/" + tagName + "/" + asset.Name, + FileName: asset.Name, + Size: asset.Size, + Type: "file", + UpdateAt: asset.UpdatedAt, + CreateAt: asset.CreatedAt, + Url: asset.BrowserDownloadUrl, + }) + } + return files + } + } + return nil +} + +// 根据版本号获取版本大小 +func (m *MountPoint) GetSizeByTagName(tagName string) int64 { + if m.Releases == nil { + return 0 + } + for _, item := range *m.Releases { + if item.TagName == tagName { + size := int64(0) + for _, asset := range item.Assets { + size += asset.Size + } + return size + } + } + return 0 +} + +// 获取所有版本大小 +func (m *MountPoint) GetAllVersionSize() int64 { + if m.Releases == nil { + return 0 + } + size := int64(0) + for _, release := range *m.Releases { + for _, asset := range release.Assets { + size += asset.Size + } + } + return size +} + +func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) []File { + if m.OtherFile == nil || refresh { + resp, _ := get("https://api.github.com/repos/" + m.Repo + "/contents") + m.OtherFile = new([]FileInfo) + json.Unmarshal(resp.Body(), m.OtherFile) + } + + files := make([]File, 0) + defaultTime := "1970-01-01T00:00:00Z" + for _, file := range *m.OtherFile { + if strings.HasSuffix(file.Name, ".md") || strings.HasPrefix(file.Name, "LICENSE") { + files = append(files, File{ + Path: m.Point + "/" + file.Name, + FileName: file.Name, + Size: file.Size, + Type: "file", + UpdateAt: defaultTime, + CreateAt: defaultTime, + Url: file.DownloadUrl, + }) + } + } + return files +} + type File struct { - FileName string `json:"name"` - Size int64 `json:"size"` - CreateAt time.Time `json:"time"` - UpdateAt time.Time `json:"chtime"` - Url string `json:"url"` - Type string `json:"type"` - Path string `json:"path"` + Path string // 文件路径 + FileName string // 文件名 + Size int64 // 文件大小 + Type string // 文件类型 + UpdateAt string // 更新时间 eg:"2025-01-27T16:10:16Z" + CreateAt string // 创建时间 + Url string // 下载链接 } func (f File) GetHash() utils.HashInfo { @@ -33,11 +195,13 @@ func (f File) GetName() string { } func (f File) ModTime() time.Time { - return f.UpdateAt + t, _ := time.Parse(time.RFC3339, f.CreateAt) + return t } func (f File) CreateTime() time.Time { - return f.CreateAt + t, _ := time.Parse(time.RFC3339, f.CreateAt) + return t } func (f File) IsDir() bool { @@ -47,22 +211,3 @@ func (f File) IsDir() bool { func (f File) GetID() string { return f.Url } - -func (f File) Thumb() string { - return "" -} - -type ReleasesData struct { - Files []File `json:"files"` - Size int64 `json:"size"` - UpdateAt time.Time `json:"chtime"` - CreateAt time.Time `json:"time"` - Url string `json:"url"` -} - -type Release struct { - Path string // 挂载路径 - RepoName string // 仓库名称 - Version string // 版本号, tag - ID string // 版本ID -} diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go index b2d79c0b3c1..df846e8a109 100644 --- a/drivers/github_releases/util.go +++ b/drivers/github_releases/util.go @@ -2,28 +2,36 @@ package github_releases import ( "fmt" - "regexp" + "path/filepath" "strings" - "sync" - "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/go-resty/resty/v2" - jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" ) -var ( - cache = make(map[string]*resty.Response) - created = make(map[string]time.Time) - mu sync.Mutex - req *resty.Request -) +// 发送 GET 请求 +func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) { + req := base.RestyClient.R() + req.SetHeader("Accept", "application/vnd.github+json") + req.SetHeader("X-GitHub-Api-Version", "2022-11-28") + if d.Addition.Token != "" { + req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", d.Addition.Token)) + } + res, err := req.Get(url) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + log.Warn("failed to get request: ", res.StatusCode(), res.String()) + } + return res, nil +} -// 解析仓库列表 -func ParseRepos(text string, allVersion bool) ([]Release, error) { +// 解析挂载结构 +func (d *GithubReleases) ParseRepos(text string) ([]MountPoint, error) { lines := strings.Split(text, "\n") - var repos []Release + points := make([]MountPoint, 0) for _, line := range lines { line = strings.TrimSpace(line) if line == "" { @@ -41,177 +49,37 @@ func ParseRepos(text string, allVersion bool) ([]Release, error) { return nil, fmt.Errorf("invalid format: %s", line) } - if allVersion { - releases, _ := GetAllVersion(repo, path) - repos = append(repos, *releases...) - } else { - repos = append(repos, Release{ - Path: path, - RepoName: repo, - Version: "latest", - ID: "latest", - }) - } - + points = append(points, MountPoint{ + Point: path, + Repo: repo, + Release: nil, + Releases: nil, + }) } - return repos, nil + d.points = points + return points, nil } // 获取下一级目录 func GetNextDir(wholePath string, basePath string) string { - if !strings.HasSuffix(basePath, "/") { - basePath += "/" - } + basePath = fmt.Sprintf("%s/", strings.TrimRight(basePath, "/")) if !strings.HasPrefix(wholePath, basePath) { return "" } remainingPath := strings.TrimLeft(strings.TrimPrefix(wholePath, basePath), "/") if remainingPath != "" { parts := strings.Split(remainingPath, "/") - return parts[0] - } - return "" -} - -// 发送 GET 请求 -func GetRequest(url string, cacheExpiration int) (*resty.Response, error) { - mu.Lock() - if res, ok := cache[url]; ok && time.Now().Before(created[url].Add(time.Duration(cacheExpiration)*time.Minute)) { - mu.Unlock() - return res, nil - } - mu.Unlock() - - res, err := req.Get(url) - if err != nil { - return nil, err - } - if res.StatusCode() != 200 { - log.Warn("failed to get request: ", res.StatusCode(), res.String()) - } - - mu.Lock() - cache[url] = res - created[url] = time.Now() - mu.Unlock() - - return res, nil -} - -// 获取 README、LICENSE 等文件 -func GetGithubOtherFile(repo string, basePath string, cacheExpiration int) (*[]File, error) { - url := fmt.Sprintf("https://api.github.com/repos/%s/contents/", strings.Trim(repo, "/")) - res, _ := GetRequest(url, cacheExpiration) - body := jsoniter.Get(res.Body()) - var files []File - for i := 0; i < body.Size(); i++ { - filename := body.Get(i, "name").ToString() - - re := regexp.MustCompile(`(?i)^(.*\.md|LICENSE)$`) - - if !re.MatchString(filename) { - continue + nextDir := parts[0] + if strings.HasPrefix(wholePath, strings.TrimRight(basePath, "/")+"/"+nextDir) { + return nextDir } - - files = append(files, File{ - FileName: filename, - Size: body.Get(i, "size").ToInt64(), - CreateAt: time.Time{}, - UpdateAt: time.Now(), - Url: body.Get(i, "download_url").ToString(), - Type: body.Get(i, "type").ToString(), - Path: fmt.Sprintf("%s/%s", basePath, filename), - }) - } - return &files, nil -} - -// 获取 GitHub Release 详细信息 -func GetRepoReleaseInfo(repo string, version string, basePath string, cacheExpiration int) (*ReleasesData, error) { - url := fmt.Sprintf("https://api.github.com/repos/%s/releases/%s", strings.Trim(repo, "/"), version) - res, _ := GetRequest(url, cacheExpiration) - body := res.Body() - - if jsoniter.Get(res.Body(), "status").ToInt64() != 0 { - return &ReleasesData{}, fmt.Errorf("%s", res.String()) - } - - assets := jsoniter.Get(res.Body(), "assets") - var files []File - - for i := 0; i < assets.Size(); i++ { - filename := assets.Get(i, "name").ToString() - - files = append(files, File{ - FileName: filename, - Size: assets.Get(i, "size").ToInt64(), - Url: assets.Get(i, "browser_download_url").ToString(), - Type: assets.Get(i, "content_type").ToString(), - Path: fmt.Sprintf("%s/%s", basePath, filename), - - CreateAt: func() time.Time { - t, _ := time.Parse(time.RFC3339, assets.Get(i, "created_at").ToString()) - return t - }(), - UpdateAt: func() time.Time { - t, _ := time.Parse(time.RFC3339, assets.Get(i, "updated_at").ToString()) - return t - }(), - }) - } - - return &ReleasesData{ - Files: files, - Url: jsoniter.Get(body, "html_url").ToString(), - - Size: func() int64 { - size := int64(0) - for _, file := range files { - size += file.Size - } - return size - }(), - UpdateAt: func() time.Time { - t, _ := time.Parse(time.RFC3339, jsoniter.Get(body, "published_at").ToString()) - return t - }(), - CreateAt: func() time.Time { - t, _ := time.Parse(time.RFC3339, jsoniter.Get(body, "created_at").ToString()) - return t - }(), - }, nil -} - -// 获取所有的版本号 -func GetAllVersion(repo string, path string) (*[]Release, error) { - url := fmt.Sprintf("https://api.github.com/repos/%s/releases", strings.Trim(repo, "/")) - res, _ := GetRequest(url, 0) - body := jsoniter.Get(res.Body()) - releases := make([]Release, 0) - for i := 0; i < body.Size(); i++ { - version := body.Get(i, "tag_name").ToString() - releases = append(releases, Release{ - Path: fmt.Sprintf("%s/%s", path, version), - Version: version, - RepoName: repo, - ID: body.Get(i, "id").ToString(), - }) } - return &releases, nil -} - -func ClearCache() { - mu.Lock() - cache = make(map[string]*resty.Response) - created = make(map[string]time.Time) - mu.Unlock() + return "" } -func SetHeader(token string) { - req = base.RestyClient.R() - if token != "" { - req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", token)) - } - req.SetHeader("Accept", "application/vnd.github+json") - req.SetHeader("X-GitHub-Api-Version", "2022-11-28") +// 判断当前目录是否是目标目录的祖先目录 +func IsAncestorDir(parentDir string, targetDir string) bool { + absTargetDir, _ := filepath.Abs(targetDir) + absParentDir, _ := filepath.Abs(parentDir) + return strings.HasPrefix(absTargetDir, absParentDir) } From d983a4ebcb481eb3fef9080c1262b8e64997220b Mon Sep 17 00:00:00 2001 From: "Feng.YJ" <32027253+huiyifyj@users.noreply.github.com> Date: Sun, 9 Feb 2025 18:30:56 +0800 Subject: [PATCH 442/659] refactor(cmd): use std `runtime` package to get go version info (#7964) * refactor(cmd): use std `runtime` package to get go version info - Remove the `GoVersion` variable. - Remove overriding `GoVersion` by ldflags in `build.sh`. - Get go version, OS and arch from the constants in the std `runtime` package instead of compile time. * chore(ci): remove `GoVersion` flag from workflows Remove GoVersion flag from beta_release.yml and build.yml workflows. > Reduce compile-time dependencies. --- .github/workflows/beta_release.yml | 1 - .github/workflows/build.yml | 1 - build.sh | 2 -- cmd/version.go | 6 ++++-- internal/conf/var.go | 1 - 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 3c52b4c40ca..485942c4a9b 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -94,7 +94,6 @@ jobs: out-dir: build x-flags: | github.com/alist-org/alist/v3/internal/conf.BuiltAt=$built_at - github.com/alist-org/alist/v3/internal/conf.GoVersion=$go_version github.com/alist-org/alist/v3/internal/conf.GitAuthor=Xhofe github.com/alist-org/alist/v3/internal/conf.GitCommit=$git_commit github.com/alist-org/alist/v3/internal/conf.Version=$tag diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe037f43367..a2c934e7a5e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,7 +49,6 @@ jobs: out-dir: build x-flags: | github.com/alist-org/alist/v3/internal/conf.BuiltAt=$built_at - github.com/alist-org/alist/v3/internal/conf.GoVersion=$go_version github.com/alist-org/alist/v3/internal/conf.GitAuthor=Xhofe github.com/alist-org/alist/v3/internal/conf.GitCommit=$git_commit github.com/alist-org/alist/v3/internal/conf.Version=$tag diff --git a/build.sh b/build.sh index d6e001c204f..2dee8e20773 100644 --- a/build.sh +++ b/build.sh @@ -1,6 +1,5 @@ appName="alist" builtAt="$(date +'%F %T %z')" -goVersion=$(go version | sed 's/go version //') gitAuthor="Xhofe " gitCommit=$(git log --pretty=format:"%h" -1) @@ -22,7 +21,6 @@ echo "frontend version: $webVersion" ldflags="\ -w -s \ -X 'github.com/alist-org/alist/v3/internal/conf.BuiltAt=$builtAt' \ --X 'github.com/alist-org/alist/v3/internal/conf.GoVersion=$goVersion' \ -X 'github.com/alist-org/alist/v3/internal/conf.GitAuthor=$gitAuthor' \ -X 'github.com/alist-org/alist/v3/internal/conf.GitCommit=$gitCommit' \ -X 'github.com/alist-org/alist/v3/internal/conf.Version=$version' \ diff --git a/cmd/version.go b/cmd/version.go index cdf4d71fcee..a758816ed96 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" "os" + "runtime" "github.com/alist-org/alist/v3/internal/conf" "github.com/spf13/cobra" @@ -16,14 +17,15 @@ var VersionCmd = &cobra.Command{ Use: "version", Short: "Show current version of AList", Run: func(cmd *cobra.Command, args []string) { + goVersion := fmt.Sprintf("%s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH) + fmt.Printf(`Built At: %s Go Version: %s Author: %s Commit ID: %s Version: %s WebVersion: %s -`, - conf.BuiltAt, conf.GoVersion, conf.GitAuthor, conf.GitCommit, conf.Version, conf.WebVersion) +`, conf.BuiltAt, goVersion, conf.GitAuthor, conf.GitCommit, conf.Version, conf.WebVersion) os.Exit(0) }, } diff --git a/internal/conf/var.go b/internal/conf/var.go index 0a8eb16fcd1..7ae1a5abfb9 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -7,7 +7,6 @@ import ( var ( BuiltAt string - GoVersion string GitAuthor string GitCommit string Version string = "dev" From 0219c4e15a452cf9a3c881dd4e49ed654e17a772 Mon Sep 17 00:00:00 2001 From: Jealous Date: Sun, 9 Feb 2025 18:31:43 +0800 Subject: [PATCH 443/659] fix(index): fix the issue where ignored paths are not updated (#7907) --- internal/search/util.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/search/util.go b/internal/search/util.go index 8d03b740c33..2e6ac8da1cd 100644 --- a/internal/search/util.go +++ b/internal/search/util.go @@ -38,7 +38,7 @@ func WriteProgress(progress *model.IndexProgress) { } } -func updateIgnorePaths() { +func updateIgnorePaths(customIgnorePaths string) { storages := op.GetAllStorages() ignorePaths := make([]string, 0) var skipDrivers = []string{"AList V2", "AList V3", "Virtual"} @@ -66,7 +66,6 @@ func updateIgnorePaths() { } } } - customIgnorePaths := setting.GetStr(conf.IgnorePaths) if customIgnorePaths != "" { ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...) } @@ -84,13 +83,13 @@ func isIgnorePath(path string) bool { func init() { op.RegisterSettingItemHook(conf.IgnorePaths, func(item *model.SettingItem) error { - updateIgnorePaths() + updateIgnorePaths(item.Value) return nil }) op.RegisterStorageHook(func(typ string, storage driver.Driver) { var skipDrivers = []string{"AList V2", "AList V3", "Virtual"} if utils.SliceContains(skipDrivers, storage.Config().Name) { - updateIgnorePaths() + updateIgnorePaths(setting.GetStr(conf.IgnorePaths)) } }) } From b9ad18bd0a668a9f2839f480d24022e3ca5cf0aa Mon Sep 17 00:00:00 2001 From: Jealous Date: Sun, 9 Feb 2025 18:32:57 +0800 Subject: [PATCH 444/659] feat(recursive-move): Advanced conflict policy for preventing unintentional overwriting (#7906) --- server/common/common.go | 26 +++++++++++++++++--------- server/handles/const.go | 7 +++++++ server/handles/fsbatch.go | 22 +++++++++++++--------- 3 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 server/handles/const.go diff --git a/server/common/common.go b/server/common/common.go index e231ffe6e88..33ae704e86b 100644 --- a/server/common/common.go +++ b/server/common/common.go @@ -68,21 +68,29 @@ func ErrorStrResp(c *gin.Context, str string, code int, l ...bool) { } func SuccessResp(c *gin.Context, data ...interface{}) { - if len(data) == 0 { - c.JSON(200, Resp[interface{}]{ - Code: 200, - Message: "success", - Data: nil, - }) - return + SuccessWithMsgResp(c, "success", data...) +} + +func SuccessWithMsgResp(c *gin.Context, msg string, data ...interface{}) { + var respData interface{} + if len(data) > 0 { + respData = data[0] } + c.JSON(200, Resp[interface{}]{ Code: 200, - Message: "success", - Data: data[0], + Message: msg, + Data: respData, }) } +func Pluralize(count int, singular, plural string) string { + if count == 1 { + return singular + } + return plural +} + func GetHttpReq(ctx context.Context) *http.Request { if c, ok := ctx.(*gin.Context); ok { return c.Request diff --git a/server/handles/const.go b/server/handles/const.go new file mode 100644 index 00000000000..b108c9da9bc --- /dev/null +++ b/server/handles/const.go @@ -0,0 +1,7 @@ +package handles + +const ( + CANCEL = "cancel" + OVERWRITE = "overwrite" + SKIP = "skip" +) diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index dd7b7e470b3..3841bff5a34 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -16,9 +16,9 @@ import ( ) type RecursiveMoveReq struct { - SrcDir string `json:"src_dir"` - DstDir string `json:"dst_dir"` - Overwrite bool `json:"overwrite"` + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + ConflictPolicy string `json:"conflict_policy"` } func FsRecursiveMove(c *gin.Context) { @@ -60,7 +60,7 @@ func FsRecursiveMove(c *gin.Context) { } var existingFileNames []string - if !req.Overwrite { + if req.ConflictPolicy != OVERWRITE { dstFiles, err := fs.List(c, dstDir, &fs.ListArgs{}) if err != nil { common.ErrorResp(c, err, 500) @@ -99,25 +99,28 @@ func FsRecursiveMove(c *gin.Context) { filePathMap[subFile] = subFilePath } } else { - if movingFilePath == dstDir { // same directory, don't move continue } - if !req.Overwrite { - if slices.Contains(existingFileNames, movingFile.GetName()) { + if slices.Contains(existingFileNames, movingFile.GetName()) { + if req.ConflictPolicy == CANCEL { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", movingFile.GetName()), 403) return + } else if req.ConflictPolicy == SKIP { + continue } + } else if req.ConflictPolicy != OVERWRITE { existingFileNames = append(existingFileNames, movingFile.GetName()) } - movingFileNames = append(movingFileNames, movingFileName) + } } + var count = 0 for i, fileName := range movingFileNames { // move err := fs.Move(c, fileName, dstDir, len(movingFileNames) > i+1) @@ -125,9 +128,10 @@ func FsRecursiveMove(c *gin.Context) { common.ErrorResp(c, err, 500) return } + count++ } - common.SuccessResp(c) + common.SuccessWithMsgResp(c, fmt.Sprintf("Successfully moved %d %s", count, common.Pluralize(count, "file", "files"))) } type BatchRenameReq struct { From 3f9bed3d5f54f559807af8190a9313962aebf982 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sun, 9 Feb 2025 18:33:38 +0800 Subject: [PATCH 445/659] feat(bootstrap): add `.url` to proxy types (#7928) --- internal/bootstrap/data/setting.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index bcb64f792d7..5e8a2be4271 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -114,7 +114,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.VideoTypes, Value: "mp4,mkv,avi,mov,rmvb,webm,flv,m3u8", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ImageTypes, Value: "jpg,tiff,jpeg,png,gif,bmp,svg,ico,swf,webp", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, //{Key: conf.OfficeTypes, Value: "doc,docx,xls,xlsx,ppt,pptx", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, - {Key: conf.ProxyTypes, Value: "m3u8", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, + {Key: conf.ProxyTypes, Value: "m3u8,url", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ProxyIgnoreHeaders, Value: "authorization,referer", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: "external_previews", Value: `{}`, Type: conf.TypeText, Group: model.PREVIEW}, {Key: "iframe_previews", Value: `{ From ec3fc945a35d08357a1109c54f7adf46b6cc379c Mon Sep 17 00:00:00 2001 From: Sakana Date: Sun, 9 Feb 2025 18:35:39 +0800 Subject: [PATCH 446/659] fix(feiji): modify the request header (#7902 close #7890) --- drivers/ilanzou/util.go | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index b8fd5280c77..ea942795e4f 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -73,6 +73,7 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr "Referer": d.conf.site + "/", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", "Accept-Encoding": "gzip, deflate, br, zstd", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,mt;q=0.5", }) if callback != nil { From f25be154c692eeffcde9fe3589848f6af48465f5 Mon Sep 17 00:00:00 2001 From: YangRucheng Date: Sun, 16 Feb 2025 12:20:28 +0800 Subject: [PATCH 447/659] fix(ilanzou): add header `X-Forwarded-For` to solve IP ban (#7977) * fix: warning * feat: ip header * fix: ip header for fs link --- drivers/ilanzou/driver.go | 17 ++++++++++++----- drivers/ilanzou/meta.go | 1 + drivers/ilanzou/util.go | 4 ++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 8681fed498e..22d1589f208 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -6,7 +6,6 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "net/url" @@ -14,6 +13,8 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -121,7 +122,7 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err != nil { return nil, err } - ts, ts_str, err := getTimestamp(d.conf.secret) + ts, ts_str, _ := getTimestamp(d.conf.secret) params := []string{ "uuid=" + url.QueryEscape(d.UUID), @@ -150,11 +151,17 @@ func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) u.RawQuery = strings.Join(params, "&") realURL := u.String() // get the url after redirect - res, err := base.NoRedirectClient.R().SetHeaders(map[string]string{ - //"Origin": d.conf.site, + req := base.NoRedirectClient.R() + + req.SetHeaders(map[string]string{ "Referer": d.conf.site + "/", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", - }).Get(realURL) + }) + if d.Addition.Ip != "" { + req.SetHeader("X-Forwarded-For", d.Addition.Ip) + } + + res, err := req.Get(realURL) if err != nil { return nil, err } diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go index f15fc01a492..7a4a00fb655 100644 --- a/drivers/ilanzou/meta.go +++ b/drivers/ilanzou/meta.go @@ -9,6 +9,7 @@ type Addition struct { driver.RootID Username string `json:"username" type:"string" required:"true"` Password string `json:"password" type:"string" required:"true"` + Ip string `json:"ip" type:"string"` Token string UUID string diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go index ea942795e4f..81773afbc80 100644 --- a/drivers/ilanzou/util.go +++ b/drivers/ilanzou/util.go @@ -76,6 +76,10 @@ func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, pr "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,mt;q=0.5", }) + if d.Addition.Ip != "" { + req.SetHeader("X-Forwarded-For", d.Addition.Ip) + } + if callback != nil { callback(req) } From 36b42046230cdc078744302eb47937eefb863ee5 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Sun, 16 Feb 2025 12:21:03 +0800 Subject: [PATCH 448/659] feat(github): support github proxy (#7979 close #7963) --- drivers/github/driver.go | 17 ++++++++++++++--- drivers/github/meta.go | 3 ++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index 996c79c74e9..dee4cbbf2db 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -85,10 +85,13 @@ func (d *Github) Init(ctx context.Context) error { } d.client = base.NewRestyClient(). SetHeader("Accept", "application/vnd.github.object+json"). - SetHeader("Authorization", "Bearer "+d.Token). SetHeader("X-GitHub-Api-Version", "2022-11-28"). SetLogger(log.StandardLogger()). SetDebug(false) + token := strings.TrimSpace(d.Token) + if token != "" { + d.client = d.client.SetHeader("Authorization", "Bearer "+token) + } if d.Ref == "" { repo, err := d.getRepo() if err != nil { @@ -149,8 +152,13 @@ func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if obj.Type == "submodule" { return nil, errors.New("cannot download a submodule") } + url := obj.DownloadURL + ghProxy := strings.TrimSpace(d.Addition.GitHubProxy) + if ghProxy != "" { + url = strings.Replace(url, "https://raw.githubusercontent.com", ghProxy, 1) + } return &model.Link{ - URL: obj.DownloadURL, + URL: url, }, nil } @@ -679,8 +687,11 @@ func (d *Github) putBlob(ctx context.Context, s model.FileStreamer, up driver.Up return "", err } req.Header.Set("Accept", "application/vnd.github+json") - req.Header.Set("Authorization", "Bearer "+d.Token) req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + token := strings.TrimSpace(d.Token) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } req.ContentLength = length res, err := base.HttpClient.Do(req) diff --git a/drivers/github/meta.go b/drivers/github/meta.go index 0df4aa60988..05e704be8e0 100644 --- a/drivers/github/meta.go +++ b/drivers/github/meta.go @@ -7,10 +7,11 @@ import ( type Addition struct { driver.RootPath - Token string `json:"token" type:"string" required:"true"` + Token string `json:"token" type:"string"` Owner string `json:"owner" type:"string" required:"true"` Repo string `json:"repo" type:"string" required:"true"` Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` + GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` CommitterName string `json:"committer_name" type:"string"` CommitterEmail string `json:"committer_email" type:"string"` AuthorName string `json:"author_name" type:"string"` From 399336b33c344768109596cf5d088270d3e1f522 Mon Sep 17 00:00:00 2001 From: foxxorcat <95907542+foxxorcat@users.noreply.github.com> Date: Sun, 16 Feb 2025 12:21:34 +0800 Subject: [PATCH 449/659] fix(189pc): transfer rename (#7958) * fix(189pc): transfer rename * fix: OverwriteUpload * fix: change search method * fix * fix --- drivers/189pc/driver.go | 73 ++++++++----- drivers/189pc/help.go | 10 ++ drivers/189pc/utils.go | 220 +++++++++++++++++++++++----------------- pkg/utils/time.go | 25 +++-- 4 files changed, 201 insertions(+), 127 deletions(-) diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index 6b502de08b8..c91caf2fb4f 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -1,8 +1,8 @@ package _189pc import ( - "container/ring" "context" + "fmt" "net/http" "strconv" "strings" @@ -14,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + "github.com/google/uuid" ) type Cloud189PC struct { @@ -29,7 +30,7 @@ type Cloud189PC struct { uploadThread int - familyTransferFolder *ring.Ring + familyTransferFolder *Cloud189Folder cleanFamilyTransferFile func() storageConfig driver.Config @@ -48,9 +49,18 @@ func (y *Cloud189PC) GetAddition() driver.Additional { } func (y *Cloud189PC) Init(ctx context.Context) (err error) { - // 兼容旧上传接口 - y.storageConfig.NoOverwriteUpload = y.isFamily() && (y.Addition.RapidUpload || y.Addition.UploadMethod == "old") - + y.storageConfig = config + if y.isFamily() { + // 兼容旧上传接口 + if y.Addition.RapidUpload || y.Addition.UploadMethod == "old" { + y.storageConfig.NoOverwriteUpload = true + } + } else { + // 家庭云转存，不支持覆盖上传 + if y.Addition.FamilyTransfer { + y.storageConfig.NoOverwriteUpload = true + } + } // 处理个人云和家庭云参数 if y.isFamily() && y.RootFolderID == "-11" { y.RootFolderID = "" @@ -91,13 +101,14 @@ func (y *Cloud189PC) Init(ctx context.Context) (err error) { } } - // 创建中转文件夹,防止重名文件 + // 创建中转文件夹 if y.FamilyTransfer { - if y.familyTransferFolder, err = y.createFamilyTransferFolder(32); err != nil { + if err := y.createFamilyTransferFolder(); err != nil { return err } } + // 清理转存文件节流 y.cleanFamilyTransferFile = utils.NewThrottle2(time.Minute, func() { if err := y.cleanFamilyTransfer(context.TODO()); err != nil { utils.Log.Errorf("cleanFamilyTransferFolderError:%s", err) @@ -327,35 +338,49 @@ func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if !isFamily && y.FamilyTransfer { // 修改上传目标为家庭云文件夹 transferDstDir := dstDir - dstDir = (y.familyTransferFolder.Value).(*Cloud189Folder) - y.familyTransferFolder = y.familyTransferFolder.Next() + dstDir = y.familyTransferFolder + // 使用临时文件名 + srcName := stream.GetName() + stream = &WrapFileStreamer{ + FileStreamer: stream, + Name: fmt.Sprintf("0%s.transfer", uuid.NewString()), + } + + // 使用家庭云上传 isFamily = true overwrite = false defer func() { if newObj != nil { - // 批量任务有概率删不掉 - y.cleanFamilyTransferFile() - // 转存家庭云文件到个人云 err = y.SaveFamilyFileToPersonCloud(context.TODO(), y.FamilyID, newObj, transferDstDir, true) - - task := BatchTaskInfo{ - FileId: newObj.GetID(), - FileName: newObj.GetName(), - IsFolder: BoolToNumber(newObj.IsDir()), + // 删除家庭云源文件 + go y.Delete(context.TODO(), y.FamilyID, newObj) + // 批量任务有概率删不掉 + go y.cleanFamilyTransferFile() + // 转存失败返回错误 + if err != nil { + return } - // 删除源文件 - if resp, err := y.CreateBatchTask("DELETE", y.FamilyID, "", nil, task); err == nil { - y.WaitBatchTask("DELETE", resp.TaskID, time.Second) - // 永久删除 - if resp, err := y.CreateBatchTask("CLEAR_RECYCLE", y.FamilyID, "", nil, task); err == nil { - y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) + // 查找转存文件 + var file *Cloud189File + file, err = y.findFileByName(context.TODO(), newObj.GetName(), transferDstDir.GetID(), false) + if err != nil { + if err == errs.ObjectNotFound { + err = fmt.Errorf("unknown error: No transfer file obtained %s", newObj.GetName()) } + return } - newObj = nil + + // 重命名转存文件 + newObj, err = y.Rename(context.TODO(), file, srcName) + if err != nil { + // 重命名失败删除源文件 + _ = y.Delete(context.TODO(), "", file) + } + return } }() } diff --git a/drivers/189pc/help.go b/drivers/189pc/help.go index 49f957fab1d..bac8880a897 100644 --- a/drivers/189pc/help.go +++ b/drivers/189pc/help.go @@ -18,6 +18,7 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils/random" ) @@ -208,3 +209,12 @@ func IF[V any](o bool, t V, f V) V { } return f } + +type WrapFileStreamer struct { + model.FileStreamer + Name string +} + +func (w *WrapFileStreamer) GetName() string { + return w.Name +} diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 0c3e54045d0..6f3c4dcf098 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -2,7 +2,6 @@ package _189pc import ( "bytes" - "container/ring" "context" "crypto/md5" "encoding/base64" @@ -23,6 +22,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" @@ -185,39 +185,9 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str return body, nil } func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) ([]model.Obj, error) { - fullUrl := API_URL - if isFamily { - fullUrl += "/family/file" - } - fullUrl += "/listFiles.action" - - res := make([]model.Obj, 0, 130) + res := make([]model.Obj, 0, 100) for pageNum := 1; ; pageNum++ { - var resp Cloud189FilesResp - _, err := y.get(fullUrl, func(r *resty.Request) { - r.SetContext(ctx) - r.SetQueryParams(map[string]string{ - "folderId": fileId, - "fileType": "0", - "mediaAttr": "0", - "iconOption": "5", - "pageNum": fmt.Sprint(pageNum), - "pageSize": "130", - }) - if isFamily { - r.SetQueryParams(map[string]string{ - "familyId": y.FamilyID, - "orderBy": toFamilyOrderBy(y.OrderBy), - "descending": toDesc(y.OrderDirection), - }) - } else { - r.SetQueryParams(map[string]string{ - "recursive": "0", - "orderBy": y.OrderBy, - "descending": toDesc(y.OrderDirection), - }) - } - }, &resp, isFamily) + resp, err := y.getFilesWithPage(ctx, fileId, isFamily, pageNum, 1000, y.OrderBy, y.OrderDirection) if err != nil { return nil, err } @@ -236,6 +206,63 @@ func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) return res, nil } +func (y *Cloud189PC) getFilesWithPage(ctx context.Context, fileId string, isFamily bool, pageNum int, pageSize int, orderBy string, orderDirection string) (*Cloud189FilesResp, error) { + fullUrl := API_URL + if isFamily { + fullUrl += "/family/file" + } + fullUrl += "/listFiles.action" + + var resp Cloud189FilesResp + _, err := y.get(fullUrl, func(r *resty.Request) { + r.SetContext(ctx) + r.SetQueryParams(map[string]string{ + "folderId": fileId, + "fileType": "0", + "mediaAttr": "0", + "iconOption": "5", + "pageNum": fmt.Sprint(pageNum), + "pageSize": fmt.Sprint(pageSize), + }) + if isFamily { + r.SetQueryParams(map[string]string{ + "familyId": y.FamilyID, + "orderBy": toFamilyOrderBy(orderBy), + "descending": toDesc(orderDirection), + }) + } else { + r.SetQueryParams(map[string]string{ + "recursive": "0", + "orderBy": orderBy, + "descending": toDesc(orderDirection), + }) + } + }, &resp, isFamily) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (y *Cloud189PC) findFileByName(ctx context.Context, searchName string, folderId string, isFamily bool) (*Cloud189File, error) { + for pageNum := 1; ; pageNum++ { + resp, err := y.getFilesWithPage(ctx, folderId, isFamily, pageNum, 10, "filename", "asc") + if err != nil { + return nil, err + } + // 获取完毕跳出 + if resp.FileListAO.Count == 0 { + return nil, errs.ObjectNotFound + } + for i := 0; i < len(resp.FileListAO.FileList); i++ { + file := resp.FileListAO.FileList[i] + if file.Name == searchName { + return &file, nil + } + } + } +} + func (y *Cloud189PC) login() (err error) { // 初始化登陆所需参数 if y.loginParam == nil { @@ -902,8 +929,7 @@ func (y *Cloud189PC) isLogin() bool { } // 创建家庭云中转文件夹 -func (y *Cloud189PC) createFamilyTransferFolder(count int) (*ring.Ring, error) { - folders := ring.New(count) +func (y *Cloud189PC) createFamilyTransferFolder() error { var rootFolder Cloud189Folder _, err := y.post(API_URL+"/family/file/createFolder.action", func(req *resty.Request) { req.SetQueryParams(map[string]string{ @@ -912,81 +938,61 @@ func (y *Cloud189PC) createFamilyTransferFolder(count int) (*ring.Ring, error) { }) }, &rootFolder, true) if err != nil { - return nil, err - } - - folderCount := 0 - - // 获取已有目录 - files, err := y.getFiles(context.TODO(), rootFolder.GetID(), true) - if err != nil { - return nil, err - } - for _, file := range files { - if folder, ok := file.(*Cloud189Folder); ok { - folders.Value = folder - folders = folders.Next() - folderCount++ - } - } - - // 创建新的目录 - for folderCount < count { - var newFolder Cloud189Folder - _, err := y.post(API_URL+"/family/file/createFolder.action", func(req *resty.Request) { - req.SetQueryParams(map[string]string{ - "folderName": uuid.NewString(), - "familyId": y.FamilyID, - "parentId": rootFolder.GetID(), - }) - }, &newFolder, true) - if err != nil { - return nil, err - } - folders.Value = &newFolder - folders = folders.Next() - folderCount++ + return err } - return folders, nil + y.familyTransferFolder = &rootFolder + return nil } // 清理中转文件夹 func (y *Cloud189PC) cleanFamilyTransfer(ctx context.Context) error { - var tasks []BatchTaskInfo - r := y.familyTransferFolder - for p := r.Next(); p != r; p = p.Next() { - folder := p.Value.(*Cloud189Folder) - - files, err := y.getFiles(ctx, folder.GetID(), true) + transferFolderId := y.familyTransferFolder.GetID() + for pageNum := 1; ; pageNum++ { + resp, err := y.getFilesWithPage(ctx, transferFolderId, true, pageNum, 100, "lastOpTime", "asc") if err != nil { return err } - for _, file := range files { + // 获取完毕跳出 + if resp.FileListAO.Count == 0 { + break + } + + var tasks []BatchTaskInfo + for i := 0; i < len(resp.FileListAO.FolderList); i++ { + folder := resp.FileListAO.FolderList[i] + tasks = append(tasks, BatchTaskInfo{ + FileId: folder.GetID(), + FileName: folder.GetName(), + IsFolder: BoolToNumber(folder.IsDir()), + }) + } + for i := 0; i < len(resp.FileListAO.FileList); i++ { + file := resp.FileListAO.FileList[i] tasks = append(tasks, BatchTaskInfo{ FileId: file.GetID(), FileName: file.GetName(), IsFolder: BoolToNumber(file.IsDir()), }) } - } - if len(tasks) > 0 { - // 删除 - resp, err := y.CreateBatchTask("DELETE", y.FamilyID, "", nil, tasks...) - if err != nil { - return err - } - err = y.WaitBatchTask("DELETE", resp.TaskID, time.Second) - if err != nil { - return err - } - // 永久删除 - resp, err = y.CreateBatchTask("CLEAR_RECYCLE", y.FamilyID, "", nil, tasks...) - if err != nil { + if len(tasks) > 0 { + // 删除 + resp, err := y.CreateBatchTask("DELETE", y.FamilyID, "", nil, tasks...) + if err != nil { + return err + } + err = y.WaitBatchTask("DELETE", resp.TaskID, time.Second) + if err != nil { + return err + } + // 永久删除 + resp, err = y.CreateBatchTask("CLEAR_RECYCLE", y.FamilyID, "", nil, tasks...) + if err != nil { + return err + } + err = y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) return err } - err = y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) - return err } return nil } @@ -1063,6 +1069,34 @@ func (y *Cloud189PC) SaveFamilyFileToPersonCloud(ctx context.Context, familyId s } } +// 永久删除文件 +func (y *Cloud189PC) Delete(ctx context.Context, familyId string, srcObj model.Obj) error { + task := BatchTaskInfo{ + FileId: srcObj.GetID(), + FileName: srcObj.GetName(), + IsFolder: BoolToNumber(srcObj.IsDir()), + } + // 删除源文件 + resp, err := y.CreateBatchTask("DELETE", familyId, "", nil, task) + if err != nil { + return err + } + err = y.WaitBatchTask("DELETE", resp.TaskID, time.Second) + if err != nil { + return err + } + // 清除回收站 + resp, err = y.CreateBatchTask("CLEAR_RECYCLE", familyId, "", nil, task) + if err != nil { + return err + } + err = y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) + if err != nil { + return err + } + return nil +} + func (y *Cloud189PC) CreateBatchTask(aType string, familyID string, targetFolderId string, other map[string]string, taskInfos ...BatchTaskInfo) (*CreateBatchTaskResp, error) { var resp CreateBatchTaskResp _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { diff --git a/pkg/utils/time.go b/pkg/utils/time.go index aa7069282fb..36573b4ecf4 100644 --- a/pkg/utils/time.go +++ b/pkg/utils/time.go @@ -34,31 +34,36 @@ func NewDebounce2(interval time.Duration, f func()) func() { if timer == nil { timer = time.AfterFunc(interval, f) } - (*time.Timer)(timer).Reset(interval) + timer.Reset(interval) } } func NewThrottle(interval time.Duration) func(func()) { var lastCall time.Time - + var lock sync.Mutex return func(fn func()) { + lock.Lock() + defer lock.Unlock() + now := time.Now() - if now.Sub(lastCall) < interval { - return + if now.Sub(lastCall) >= interval { + lastCall = now + go fn() } - time.AfterFunc(interval, fn) - lastCall = now } } func NewThrottle2(interval time.Duration, fn func()) func() { var lastCall time.Time + var lock sync.Mutex return func() { + lock.Lock() + defer lock.Unlock() + now := time.Now() - if now.Sub(lastCall) < interval { - return + if now.Sub(lastCall) >= interval { + lastCall = now + go fn() } - time.AfterFunc(interval, fn) - lastCall = now } } From 3b71500f237c4cb3427ce11cb6675fa9ba7006b6 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Sun, 16 Feb 2025 12:22:11 +0800 Subject: [PATCH 450/659] feat(traffic): support limit task worker count & file stream rate (#7948) * feat: set task workers num & client stream rate limit * feat: server stream rate limit * upgrade xhofe/tache * . --- cmd/common.go | 1 + drivers/115/util.go | 26 ++--- drivers/123/driver.go | 5 +- drivers/123/upload.go | 3 +- drivers/139/driver.go | 6 +- drivers/189/util.go | 6 +- drivers/189pc/utils.go | 19 +++- drivers/alist_v3/driver.go | 4 +- drivers/aliyundrive/driver.go | 13 ++- drivers/aliyundrive_open/upload.go | 7 +- drivers/baidu_netdisk/driver.go | 10 +- drivers/baidu_photo/driver.go | 10 +- drivers/base/client.go | 13 +-- drivers/chaoxing/driver.go | 7 +- drivers/cloudreve/driver.go | 6 +- drivers/cloudreve/util.go | 13 +-- drivers/dropbox/driver.go | 9 +- drivers/ftp/driver.go | 12 +-- drivers/github/driver.go | 8 +- drivers/google_drive/driver.go | 3 +- drivers/google_drive/util.go | 8 +- drivers/google_photo/driver.go | 2 +- drivers/halalcloud/driver.go | 3 +- drivers/ilanzou/driver.go | 6 +- drivers/ipfs_api/driver.go | 12 +-- drivers/kodbox/driver.go | 9 +- drivers/lanzou/driver.go | 10 +- drivers/lark/driver.go | 12 ++- drivers/mediatrack/driver.go | 7 +- drivers/mega/driver.go | 3 +- drivers/misskey/driver.go | 2 +- drivers/misskey/util.go | 18 ++-- drivers/mopan/driver.go | 12 ++- drivers/netease_music/types.go | 13 --- drivers/netease_music/util.go | 14 +-- drivers/onedrive/util.go | 10 +- drivers/onedrive_app/util.go | 10 +- drivers/pikpak/util.go | 19 ++-- drivers/quark_uc/driver.go | 14 +-- drivers/quark_uc/util.go | 9 +- drivers/quqi/driver.go | 14 ++- drivers/s3/driver.go | 11 +-- drivers/seafile/driver.go | 5 +- drivers/sftp/driver.go | 2 +- drivers/smb/driver.go | 2 +- drivers/teambition/driver.go | 2 +- drivers/teambition/util.go | 22 +++-- drivers/terabox/driver.go | 2 +- drivers/thunder/driver.go | 5 +- drivers/thunder_browser/driver.go | 2 +- drivers/thunderx/driver.go | 5 +- drivers/trainbit/driver.go | 7 +- drivers/url_tree/driver.go | 2 +- drivers/uss/driver.go | 11 +-- drivers/vtencent/util.go | 3 +- drivers/webdav/driver.go | 13 +-- drivers/weiyun/driver.go | 7 +- drivers/wopan/driver.go | 2 +- drivers/yandex_disk/driver.go | 4 +- go.mod | 6 +- go.sum | 8 +- internal/bootstrap/data/setting.go | 17 +++- internal/bootstrap/stream_limit.go | 53 ++++++++++ internal/bootstrap/task.go | 39 ++++++-- internal/conf/const.go | 12 +++ internal/driver/driver.go | 68 ++++++++----- internal/driver/utils.go | 62 ++++++++++++ internal/model/setting.go | 1 + internal/net/serve.go | 14 ++- internal/op/setting.go | 9 ++ internal/stream/limit.go | 152 +++++++++++++++++++++++++++++ internal/stream/stream.go | 60 +++++++----- internal/stream/util.go | 20 ++++ server/common/proxy.go | 27 ++++- server/ftp/fsread.go | 7 +- server/ftp/fsup.go | 19 +++- server/middlewares/limit.go | 36 +++++++ server/router.go | 17 ++-- server/webdav.go | 8 +- 79 files changed, 803 insertions(+), 327 deletions(-) create mode 100644 internal/bootstrap/stream_limit.go create mode 100644 internal/driver/utils.go create mode 100644 internal/stream/limit.go diff --git a/cmd/common.go b/cmd/common.go index 47a25f3f266..8a73f9b0582 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -17,6 +17,7 @@ func Init() { bootstrap.Log() bootstrap.InitDB() data.InitData() + bootstrap.InitStreamLimit() bootstrap.InitIndex() bootstrap.InitUpgradePatch() } diff --git a/drivers/115/util.go b/drivers/115/util.go index 4d3cdd93ff1..7298f565d0c 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -8,8 +8,6 @@ import ( "encoding/hex" "encoding/json" "fmt" - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "net/url" @@ -20,6 +18,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" @@ -144,7 +143,7 @@ func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, e return nil, err } - bytes, err := crypto.Decode(string(result.EncodedData), key) + b, err := crypto.Decode(string(result.EncodedData), key) if err != nil { return nil, err } @@ -152,7 +151,7 @@ func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, e downloadInfo := struct { Url string `json:"url"` }{} - if err := utils.Json.Unmarshal(bytes, &downloadInfo); err != nil { + if err := utils.Json.Unmarshal(b, &downloadInfo); err != nil { return nil, err } @@ -290,13 +289,10 @@ func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSPar } var bodyBytes []byte - r := &stream.ReaderWithCtx{ - Reader: &stream.ReaderUpdatingProgress{ - Reader: s, - UpdateProgress: up, - }, - Ctx: ctx, - } + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) if err = bucket.PutObject(params.Object, r, append( driver115.OssOption(params, ossToken), oss.CallbackResult(&bodyBytes), @@ -405,16 +401,12 @@ func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.Upload } default: } - buf := make([]byte, chunk.Size) if _, err = tmpF.ReadAt(buf, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { continue } - - if part, err = bucket.UploadPart(imur, &stream.ReaderWithCtx{ - Reader: bytes.NewBuffer(buf), - Ctx: ctx, - }, chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { + if part, err = bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(buf)), + chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { break } } diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 1bf71ae64a8..7d457138fde 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,7 +6,6 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "net/url" @@ -249,10 +248,10 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStrea input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Key, - Body: &stream.ReaderUpdatingProgress{ + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, - }, + }), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { diff --git a/drivers/123/upload.go b/drivers/123/upload.go index a472df55240..dc148c4c93f 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -81,6 +81,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi batchSize = 10 getS3UploadUrl = d.getS3PreSignedUrls } + limited := driver.NewLimitedUploadStream(ctx, file) for i := 1; i <= chunkCount; i += batchSize { if utils.IsCanceled(ctx) { return ctx.Err() @@ -103,7 +104,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi if j == chunkCount { curSize = file.GetSize() - (int64(chunkCount)-1)*chunkSize } - err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.LimitReader(file, chunkSize), curSize, false, getS3UploadUrl) + err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.LimitReader(limited, chunkSize), curSize, false, getS3UploadUrl) if err != nil { return err } diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 1e2ba9c4d52..c6b30335770 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -631,12 +631,13 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr // Progress p := driver.NewProgress(stream.GetSize(), up) + rateLimited := driver.NewLimitedUploadStream(ctx, stream) // 上传所有分片 for _, uploadPartInfo := range uploadPartInfos { index := uploadPartInfo.PartNumber - 1 partSize := partInfos[index].PartSize log.Debugf("[139] uploading part %+v/%+v", index, len(uploadPartInfos)) - limitReader := io.LimitReader(stream, partSize) + limitReader := io.LimitReader(rateLimited, partSize) // Update Progress r := io.TeeReader(limitReader, p) @@ -787,6 +788,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr if part == 0 { part = 1 } + rateLimited := driver.NewLimitedUploadStream(ctx, stream) for i := int64(0); i < part; i++ { if utils.IsCanceled(ctx) { return ctx.Err() @@ -798,7 +800,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr byteSize = partSize } - limitReader := io.LimitReader(stream, byteSize) + limitReader := io.LimitReader(rateLimited, byteSize) // Update Progress r := io.TeeReader(limitReader, p) req, err := http.NewRequest("POST", resp.Data.UploadResult.RedirectionURL, r) diff --git a/drivers/189/util.go b/drivers/189/util.go index 0b4c0633d7b..16a5aa3996e 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -365,7 +365,7 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F log.Debugf("uploadData: %+v", uploadData) requestURL := uploadData.RequestURL uploadHeaders := strings.Split(decodeURIComponent(uploadData.RequestHeader), "&") - req, err := http.NewRequest(http.MethodPut, requestURL, bytes.NewReader(byteData)) + req, err := http.NewRequest(http.MethodPut, requestURL, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { return err } @@ -375,11 +375,11 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F req.Header.Set(v[0:i], v[i+1:]) } r, err := base.HttpClient.Do(req) - log.Debugf("%+v %+v", r, r.Request.Header) - r.Body.Close() if err != nil { return err } + log.Debugf("%+v %+v", r, r.Request.Header) + _ = r.Body.Close() up(float64(i) * 100 / float64(count)) } fileMd5 := hex.EncodeToString(md5Sum.Sum(nil)) diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 6f3c4dcf098..290d2e568aa 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -19,6 +19,8 @@ import ( "strings" "time" + "golang.org/x/sync/semaphore" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" @@ -174,8 +176,8 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str } var erron RespErr - jsoniter.Unmarshal(body, &erron) - xml.Unmarshal(body, &erron) + _ = jsoniter.Unmarshal(body, &erron) + _ = xml.Unmarshal(body, &erron) if erron.HasError() { return nil, &erron } @@ -508,6 +510,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) + sem := semaphore.NewWeighted(3) fileMd5 := md5.New() silceMd5 := md5.New() @@ -517,7 +520,9 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if utils.IsCanceled(upCtx) { break } - + if err = sem.Acquire(ctx, 1); err != nil { + break + } byteData := make([]byte, sliceSize) if i == count { byteData = byteData[:lastPartSize] @@ -526,6 +531,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo // 读取块 silceMd5.Reset() if _, err := io.ReadFull(io.TeeReader(file, io.MultiWriter(fileMd5, silceMd5)), byteData); err != io.EOF && err != nil { + sem.Release(1) return nil, err } @@ -535,6 +541,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo partInfo := fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) threadG.Go(func(ctx context.Context) error { + defer sem.Release(1) uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo) if err != nil { return err @@ -542,7 +549,8 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo // step.4 上传切片 uploadUrl := uploadUrls[0] - _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, bytes.NewReader(byteData), isFamily) + _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData)), isFamily) if err != nil { return err } @@ -794,6 +802,7 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model if err != nil { return nil, err } + rateLimited := driver.NewLimitedUploadStream(ctx, io.NopCloser(tempFile)) // 创建上传会话 uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize()), isFamily) @@ -820,7 +829,7 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model header["Edrive-UploadFileId"] = fmt.Sprint(status.UploadFileId) } - _, err := y.put(ctx, status.FileUploadUrl, header, true, io.NopCloser(tempFile), isFamily) + _, err := y.put(ctx, status.FileUploadUrl, header, true, rateLimited, isFamily) if err, ok := err.(*RespErr); ok && err.Code != "InputStreamReadError" { return nil, err } diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index 679285e0d8f..5a299ea0aec 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -3,7 +3,6 @@ package alist_v3 import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "path" @@ -183,10 +182,11 @@ func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { } func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", &stream.ReaderUpdatingProgress{ + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", reader) if err != nil { return err } diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index 2a977aa35e5..105e28b2e98 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -14,13 +14,12 @@ import ( "os" "time" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -194,7 +193,10 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil } if d.RapidUpload { buf := bytes.NewBuffer(make([]byte, 0, 1024)) - utils.CopyWithBufferN(buf, file, 1024) + _, err := utils.CopyWithBufferN(buf, file, 1024) + if err != nil { + return err + } reqBody["pre_hash"] = utils.HashData(utils.SHA1, buf.Bytes()) if localFile != nil { if _, err := localFile.Seek(0, io.SeekStart); err != nil { @@ -286,6 +288,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil file.Reader = localFile } + rateLimited := driver.NewLimitedUploadStream(ctx, file) for i, partInfo := range resp.PartInfoList { if utils.IsCanceled(ctx) { return ctx.Err() @@ -294,7 +297,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil if d.InternalUpload { url = partInfo.InternalUploadUrl } - req, err := http.NewRequest("PUT", url, io.LimitReader(file, DEFAULT)) + req, err := http.NewRequest("PUT", url, io.LimitReader(rateLimited, DEFAULT)) if err != nil { return err } @@ -303,7 +306,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() if count > 0 { up(float64(i) * 100 / float64(count)) } diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 653a2442346..fb730de6966 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -77,7 +77,7 @@ func (d *AliyundriveOpen) uploadPart(ctx context.Context, r io.Reader, partInfo if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusConflict { return fmt.Errorf("upload status: %d", res.StatusCode) } @@ -251,8 +251,9 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m rd = utils.NewMultiReadable(srd) } err = retry.Do(func() error { - rd.Reset() - return d.uploadPart(ctx, rd, createResp.PartInfoList[i]) + _ = rd.Reset() + rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd) + return d.uploadPart(ctx, rateLimitedRd, createResp.PartInfoList[i]) }, retry.Attempts(3), retry.DelayType(retry.BackOffDelay), diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index ad52a4b5438..e0ba98fa9b5 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -12,6 +12,8 @@ import ( "strconv" "time" + "golang.org/x/sync/semaphore" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -263,16 +265,21 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) + sem := semaphore.NewWeighted(3) for i, partseq := range precreateResp.BlockList { if utils.IsCanceled(upCtx) { break } + if err = sem.Acquire(ctx, 1); err != nil { + break + } i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize if partseq+1 == count { byteSize = lastBlockSize } threadG.Go(func(ctx context.Context) error { + defer sem.Release(1) params := map[string]string{ "method": "upload", "access_token": d.AccessToken, @@ -281,7 +288,8 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F "uploadid": precreateResp.Uploadid, "partseq": strconv.Itoa(partseq), } - err := d.uploadSlice(ctx, params, stream.GetName(), io.NewSectionReader(tempFile, offset, byteSize)) + err := d.uploadSlice(ctx, params, stream.GetName(), + driver.NewLimitedUploadStream(ctx, io.NewSectionReader(tempFile, offset, byteSize))) if err != nil { return err } diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index b584c9a3bb2..9ee0a7ae860 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "golang.org/x/sync/semaphore" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -314,10 +316,14 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) + sem := semaphore.NewWeighted(3) for i, partseq := range precreateResp.BlockList { if utils.IsCanceled(upCtx) { break } + if err = sem.Acquire(ctx, 1); err != nil { + break + } i, partseq, offset, byteSize := i, partseq, int64(partseq)*DEFAULT, DEFAULT if partseq+1 == count { @@ -325,6 +331,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil } threadG.Go(func(ctx context.Context) error { + defer sem.Release(1) uploadParams := map[string]string{ "method": "upload", "path": params["path"], @@ -335,7 +342,8 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil _, err = d.Post("https://c3.pcs.baidu.com/rest/2.0/pcs/superfile2", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(uploadParams) - r.SetFileReader("file", stream.GetName(), io.NewSectionReader(tempFile, offset, byteSize)) + r.SetFileReader("file", stream.GetName(), + driver.NewLimitedUploadStream(ctx, io.NewSectionReader(tempFile, offset, byteSize))) }, nil) if err != nil { return err diff --git a/drivers/base/client.go b/drivers/base/client.go index 8bf8f421eea..538c43a66c5 100644 --- a/drivers/base/client.go +++ b/drivers/base/client.go @@ -6,6 +6,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/net" "github.com/go-resty/resty/v2" ) @@ -26,7 +27,7 @@ func InitClient() { NoRedirectClient.SetHeader("user-agent", UserAgent) RestyClient = NewRestyClient() - HttpClient = NewHttpClient() + HttpClient = net.NewHttpClient() } func NewRestyClient() *resty.Client { @@ -38,13 +39,3 @@ func NewRestyClient() *resty.Client { SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) return client } - -func NewHttpClient() *http.Client { - return &http.Client{ - Timeout: time.Hour * 48, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, - }, - } -} diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go index 9b526f8ac34..bf01a83b732 100644 --- a/drivers/chaoxing/driver.go +++ b/drivers/chaoxing/driver.go @@ -6,7 +6,6 @@ import ( "encoding/json" "errors" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "mime/multipart" "net/http" @@ -249,13 +248,13 @@ func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStr if err != nil { return err } - r := &stream.ReaderUpdatingProgress{ - Reader: &stream.SimpleReaderWithSize{ + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ Reader: body, Size: int64(body.Len()), }, UpdateProgress: up, - } + }) req, err := http.NewRequestWithContext(ctx, "POST", "https://pan-yz.chaoxing.com/upload", r) if err != nil { return err diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 8fc117aca2c..73fc3fea2af 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -1,7 +1,9 @@ package cloudreve import ( + "bytes" "context" + "errors" "io" "net/http" "path" @@ -173,7 +175,7 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File var n int buf = make([]byte, chunkSize) n, err = io.ReadAtLeast(stream, buf, chunkSize) - if err != nil && err != io.ErrUnexpectedEOF { + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { if err == io.EOF { return nil } @@ -186,7 +188,7 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { req.SetHeader("Content-Type", "application/octet-stream") req.SetHeader("Content-Length", strconv.Itoa(n)) - req.SetBody(buf) + req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(buf))) }, nil) if err != nil { break diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index b5b71153e12..8a90a42fee7 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -100,7 +100,7 @@ func (d *Cloudreve) login() error { if err == nil { break } - if err != nil && err.Error() != "CAPTCHA not match." { + if err.Error() != "CAPTCHA not match." { break } } @@ -202,7 +202,8 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U if err != nil { return err } - req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), bytes.NewBuffer(byteData)) + req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) if err != nil { return err } @@ -214,7 +215,7 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() up(float64(finish) * 100 / float64(stream.GetSize())) chunk++ } @@ -241,7 +242,7 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(byteData)) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) if err != nil { return err } @@ -256,10 +257,10 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { data, _ := io.ReadAll(res.Body) - res.Body.Close() + _ = res.Body.Close() return errors.New(string(data)) } - res.Body.Close() + _ = res.Body.Close() up(float64(finish) * 100 / float64(stream.GetSize())) } // 上传成功发送回调请求 diff --git a/drivers/dropbox/driver.go b/drivers/dropbox/driver.go index 9b1717b04d9..fbaecc4a991 100644 --- a/drivers/dropbox/driver.go +++ b/drivers/dropbox/driver.go @@ -191,7 +191,7 @@ func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt } url := d.contentBase + "/2/files/upload_session/append_v2" - reader := io.LimitReader(stream, PartSize) + reader := driver.NewLimitedUploadStream(ctx, io.LimitReader(stream, PartSize)) req, err := http.NewRequest(http.MethodPost, url, reader) if err != nil { log.Errorf("failed to update file when append to upload session, err: %+v", err) @@ -219,13 +219,8 @@ func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt return err } _ = res.Body.Close() - - if count > 0 { - up(float64(i+1) * 100 / float64(count)) - } - + up(float64(i+1) * 100 / float64(count)) offset += byteSize - } // 3.finish toPath := dstDir.GetPath() + "/" + stream.GetName() diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index b3e95f9320f..8f30b780e75 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -2,7 +2,6 @@ package ftp import ( "context" - "github.com/alist-org/alist/v3/internal/stream" stdpath "path" "github.com/alist-org/alist/v3/internal/driver" @@ -120,13 +119,10 @@ func (d *FTP) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, u return err } path := stdpath.Join(dstDir.GetPath(), s.GetName()) - return d.conn.Stor(encode(path, d.Encoding), &stream.ReaderWithCtx{ - Reader: &stream.ReaderUpdatingProgress{ - Reader: s, - UpdateProgress: up, - }, - Ctx: ctx, - }) + return d.conn.Stor(encode(path, d.Encoding), driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + })) } var _ driver.Driver = (*FTP)(nil) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index dee4cbbf2db..d1cfd9fbc02 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -16,7 +16,6 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" @@ -676,13 +675,13 @@ func (d *Github) putBlob(ctx context.Context, s model.FileStreamer, up driver.Up afterContentReader := strings.NewReader(afterContent) req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("https://api.github.com/repos/%s/%s/git/blobs", d.Owner, d.Repo), - &stream.ReaderUpdatingProgress{ - Reader: &stream.SimpleReaderWithSize{ + driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ Reader: io.MultiReader(beforeContentReader, contentReader, afterContentReader), Size: length, }, UpdateProgress: up, - }) + })) if err != nil { return "", err } @@ -698,6 +697,7 @@ func (d *Github) putBlob(ctx context.Context, s model.FileStreamer, up driver.Up if err != nil { return "", err } + defer res.Body.Close() resBody, err := io.ReadAll(res.Body) if err != nil { return "", err diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index dccdcea902f..c8afb08499b 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -158,7 +158,8 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi putUrl := res.Header().Get("location") if stream.GetSize() < d.ChunkSize*1024*1024 { _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { - req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)).SetBody(stream) + req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). + SetBody(driver.NewLimitedUploadStream(ctx, stream)) }, nil) } else { err = d.chunkUpload(ctx, stream, putUrl) diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 0d3801127a4..0fe543468b8 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -11,10 +11,10 @@ import ( "strconv" "time" - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" "github.com/golang-jwt/jwt/v4" @@ -126,8 +126,7 @@ func (d *GoogleDrive) refreshToken() error { } d.AccessToken = resp.AccessToken return nil - } - if gdsaFileErr != nil && os.IsExist(gdsaFileErr) { + } else if os.IsExist(gdsaFileErr) { return gdsaFileErr } url := "https://www.googleapis.com/oauth2/v4/token" @@ -229,6 +228,7 @@ func (d *GoogleDrive) chunkUpload(ctx context.Context, stream model.FileStreamer if err != nil { return err } + reader = driver.NewLimitedUploadStream(ctx, reader) _, err = d.request(url, http.MethodPut, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Content-Length": strconv.FormatInt(chunkSize, 10), diff --git a/drivers/google_photo/driver.go b/drivers/google_photo/driver.go index b54132ef9ed..e6f0abc6416 100644 --- a/drivers/google_photo/driver.go +++ b/drivers/google_photo/driver.go @@ -124,7 +124,7 @@ func (d *GooglePhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fi } resp, err := d.request(postUrl, http.MethodPost, func(req *resty.Request) { - req.SetBody(stream).SetContext(ctx) + req.SetBody(driver.NewLimitedUploadStream(ctx, stream)).SetContext(ctx) }, nil, postHeaders) if err != nil { diff --git a/drivers/halalcloud/driver.go b/drivers/halalcloud/driver.go index d3235828201..26832760117 100644 --- a/drivers/halalcloud/driver.go +++ b/drivers/halalcloud/driver.go @@ -392,10 +392,11 @@ func (d *HalalCloud) put(ctx context.Context, dstDir model.Obj, fileStream model if fileStream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { uploader.PartSize = fileStream.GetSize() / (s3manager.MaxUploadParts - 1) } + reader := driver.NewLimitedUploadStream(ctx, fileStream) _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(result.Bucket), Key: aws.String(result.Key), - Body: io.TeeReader(fileStream, driver.NewProgress(fileStream.GetSize(), up)), + Body: io.TeeReader(reader, driver.NewProgress(fileStream.GetSize(), up)), }) return nil, err diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 22d1589f208..697d85b1b3b 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -309,13 +309,13 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame upToken := utils.Json.Get(res, "upToken").ToString() now := time.Now() key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) - reader := &stream.ReaderUpdatingProgress{ - Reader: &stream.SimpleReaderWithSize{ + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ Reader: tempFile, Size: s.GetSize(), }, UpdateProgress: up, - } + }) var token string if s.GetSize() <= DefaultPartSize { res, err := d.upClient.R().SetContext(ctx).SetMultipartFormData(map[string]string{ diff --git a/drivers/ipfs_api/driver.go b/drivers/ipfs_api/driver.go index 61886b38b6f..777606564a2 100644 --- a/drivers/ipfs_api/driver.go +++ b/drivers/ipfs_api/driver.go @@ -3,7 +3,6 @@ package ipfs import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "net/url" stdpath "path" "path/filepath" @@ -111,13 +110,10 @@ func (d *IPFS) Remove(ctx context.Context, obj model.Obj) error { func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { // TODO upload file, optional - _, err := d.sh.Add(&stream.ReaderWithCtx{ - Reader: &stream.ReaderUpdatingProgress{ - Reader: s, - UpdateProgress: up, - }, - Ctx: ctx, - }, ToFiles(stdpath.Join(dstDir.GetPath(), s.GetName()))) + _, err := d.sh.Add(driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }), ToFiles(stdpath.Join(dstDir.GetPath(), s.GetName()))) return err } diff --git a/drivers/kodbox/driver.go b/drivers/kodbox/driver.go index ff48ffb21cb..c536c916d7a 100644 --- a/drivers/kodbox/driver.go +++ b/drivers/kodbox/driver.go @@ -3,9 +3,6 @@ package kodbox import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/go-resty/resty/v2" "net/http" "path/filepath" "strings" @@ -13,6 +10,8 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" ) type KodBox struct { @@ -229,10 +228,10 @@ func (d *KodBox) Remove(ctx context.Context, obj model.Obj) error { func (d *KodBox) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { var resp *CommonResp _, err := d.request(http.MethodPost, "/?explorer/upload/fileUpload", func(req *resty.Request) { - r := &stream.ReaderUpdatingProgress{ + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, - } + }) req.SetFileReader("file", s.GetName(), r). SetResult(&resp). SetFormData(map[string]string{ diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index 90635d16349..877e72bb3d1 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -2,7 +2,6 @@ package lanzou import ( "context" - "github.com/alist-org/alist/v3/internal/stream" "net/http" "github.com/alist-org/alist/v3/drivers/base" @@ -213,6 +212,10 @@ func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer if d.IsCookie() || d.IsAccount() { var resp RespText[[]FileOrFolder] _, err := d._post(d.BaseUrl+"/html5up.php", func(req *resty.Request) { + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) req.SetFormData(map[string]string{ "task": "1", "vie": "2", @@ -220,10 +223,7 @@ func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer "id": "WU_FILE_0", "name": s.GetName(), "folder_id_bb_n": dstDir.GetID(), - }).SetFileReader("upload_file", s.GetName(), &stream.ReaderUpdatingProgress{ - Reader: s, - UpdateProgress: up, - }).SetContext(ctx) + }).SetFileReader("upload_file", s.GetName(), reader).SetContext(ctx) }, &resp, true) if err != nil { return nil, err diff --git a/drivers/lark/driver.go b/drivers/lark/driver.go index d2672300444..fbf7529afe3 100644 --- a/drivers/lark/driver.go +++ b/drivers/lark/driver.go @@ -320,7 +320,10 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea Build() // 发起请求 - uploadLimit.Wait(ctx) + err := uploadLimit.Wait(ctx) + if err != nil { + return nil, err + } resp, err := c.client.Drive.File.UploadPrepare(ctx, req) if err != nil { return nil, err @@ -341,7 +344,7 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea length = stream.GetSize() - int64(i*blockSize) } - reader := io.LimitReader(stream, length) + reader := driver.NewLimitedUploadStream(ctx, io.LimitReader(stream, length)) req := larkdrive.NewUploadPartFileReqBuilder(). Body(larkdrive.NewUploadPartFileReqBodyBuilder(). @@ -353,7 +356,10 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea Build() // 发起请求 - uploadLimit.Wait(ctx) + err = uploadLimit.Wait(ctx) + if err != nil { + return nil, err + } resp, err := c.client.Drive.File.UploadPart(ctx, req) if err != nil { diff --git a/drivers/mediatrack/driver.go b/drivers/mediatrack/driver.go index ed53f8ee03a..50ef9799506 100644 --- a/drivers/mediatrack/driver.go +++ b/drivers/mediatrack/driver.go @@ -5,7 +5,6 @@ import ( "crypto/md5" "encoding/hex" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "strconv" @@ -195,13 +194,13 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, file model.FileS input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Object, - Body: &stream.ReaderUpdatingProgress{ - Reader: &stream.SimpleReaderWithSize{ + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ Reader: tempFile, Size: file.GetSize(), }, UpdateProgress: up, - }, + }), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index 198c1f9864c..f76bfeefd70 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -156,6 +156,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea return err } + reader := driver.NewLimitedUploadStream(ctx, stream) for id := 0; id < u.Chunks(); id++ { if utils.IsCanceled(ctx) { return ctx.Err() @@ -165,7 +166,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea return err } chunk := make([]byte, chkSize) - n, err := io.ReadFull(stream, chunk) + n, err := io.ReadFull(reader, chunk) if err != nil && err != io.EOF { return err } diff --git a/drivers/misskey/driver.go b/drivers/misskey/driver.go index 29797a01242..b5c753f3c65 100644 --- a/drivers/misskey/driver.go +++ b/drivers/misskey/driver.go @@ -64,7 +64,7 @@ func (d *Misskey) Remove(ctx context.Context, obj model.Obj) error { } func (d *Misskey) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - return d.put(dstDir, stream, up) + return d.put(ctx, dstDir, stream, up) } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go index 4d5a3b4d01f..f8baeafa6cb 100644 --- a/drivers/misskey/util.go +++ b/drivers/misskey/util.go @@ -1,7 +1,6 @@ package misskey import ( - "bytes" "context" "errors" "io" @@ -190,16 +189,16 @@ func (d *Misskey) remove(obj model.Obj) error { } } -func (d *Misskey) put(dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { var file MFile - fileContent, err := io.ReadAll(stream) - if err != nil { - return nil, err - } - + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: stream, + UpdateProgress: up, + }) req := base.RestyClient.R(). - SetFileReader("file", stream.GetName(), io.NopCloser(bytes.NewReader(fileContent))). + SetContext(ctx). + SetFileReader("file", stream.GetName(), reader). SetFormData(map[string]string{ "folderId": handleFolderId(dstDir).(string), "name": stream.GetName(), @@ -207,7 +206,8 @@ func (d *Misskey) put(dstDir model.Obj, stream model.FileStreamer, up driver.Upd "isSensitive": "false", "force": "false", }). - SetResult(&file).SetAuthToken(d.AccessToken) + SetResult(&file). + SetAuthToken(d.AccessToken) resp, err := req.Post(d.Endpoint + "/api/drive/files/create") if err != nil { diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index 369ec83b64d..2cbabe46b36 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "golang.org/x/sync/semaphore" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" @@ -301,6 +303,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) + sem := semaphore.NewWeighted(3) // step.3 parts, err := d.client.GetAllMultiUploadUrls(initUpdload.UploadFileID, initUpdload.PartInfos) @@ -312,6 +315,9 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if utils.IsCanceled(upCtx) { break } + if err = sem.Acquire(ctx, 1); err != nil { + break + } i, part, byteSize := i, part, initUpdload.PartSize if part.PartNumber == uploadPartData.PartTotal { byteSize = initUpdload.LastPartSize @@ -319,7 +325,9 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre // step.4 threadG.Go(func(ctx context.Context) error { - req, err := part.NewRequest(ctx, io.NewSectionReader(file, int64(part.PartNumber-1)*initUpdload.PartSize, byteSize)) + defer sem.Release(1) + reader := io.NewSectionReader(file, int64(part.PartNumber-1)*initUpdload.PartSize, byteSize) + req, err := part.NewRequest(ctx, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } @@ -328,7 +336,7 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if err != nil { return err } - resp.Body.Close() + _ = resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("upload err,code=%d", resp.StatusCode) } diff --git a/drivers/netease_music/types.go b/drivers/netease_music/types.go index 332f75e94f6..12afeb7a67e 100644 --- a/drivers/netease_music/types.go +++ b/drivers/netease_music/types.go @@ -116,16 +116,3 @@ func (ch *Characteristic) merge(data map[string]string) map[string]interface{} { } return body } - -type InlineReadCloser struct { - io.Reader - io.Closer -} - -func (rc *InlineReadCloser) Read(p []byte) (int, error) { - return rc.Reader.Read(p) -} - -func (rc *InlineReadCloser) Close() error { - return rc.Closer.Close() -} diff --git a/drivers/netease_music/util.go b/drivers/netease_music/util.go index 25efde77b9d..2e78be14b97 100644 --- a/drivers/netease_music/util.go +++ b/drivers/netease_music/util.go @@ -2,8 +2,6 @@ package netease_music import ( "context" - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/stream" "net/http" "path" "regexp" @@ -12,6 +10,7 @@ import ( "time" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" @@ -69,13 +68,10 @@ func (d *NeteaseMusic) request(url, method string, opt ReqOption) ([]byte, error opt.up = func(_ float64) {} } req.SetContentLength(true) - req.SetBody(&InlineReadCloser{ - Reader: &stream.ReaderUpdatingProgress{ - Reader: opt.stream, - UpdateProgress: opt.up, - }, - Closer: opt.stream, - }) + req.SetBody(driver.NewLimitedUploadStream(opt.ctx, &driver.ReaderUpdatingProgress{ + Reader: opt.stream, + UpdateProgress: opt.up, + })) } else { req.SetFormData(data) } diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 95f92db6433..9350a681cbd 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -152,12 +152,8 @@ func (d *Onedrive) upSmall(ctx context.Context, dstDir model.Obj, stream model.F // 1. upload new file // ApiDoc: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online url := d.GetMetaUrl(false, filepath) + "/content" - data, err := io.ReadAll(stream) - if err != nil { - return err - } - _, err = d.Request(url, http.MethodPut, func(req *resty.Request) { - req.SetBody(data).SetContext(ctx) + _, err := d.Request(url, http.MethodPut, func(req *resty.Request) { + req.SetBody(driver.NewLimitedUploadStream(ctx, stream)).SetContext(ctx) }, nil) if err != nil { return fmt.Errorf("onedrive: Failed to upload new file(path=%v): %w", filepath, err) @@ -225,7 +221,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(byteData)) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) if err != nil { return err } diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index d036e131757..a6793520269 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -140,12 +140,8 @@ func (d *OnedriveAPP) GetFile(path string) (*File, error) { func (d *OnedriveAPP) upSmall(ctx context.Context, dstDir model.Obj, stream model.FileStreamer) error { url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/content" - data, err := io.ReadAll(stream) - if err != nil { - return err - } - _, err = d.Request(url, http.MethodPut, func(req *resty.Request) { - req.SetBody(data).SetContext(ctx) + _, err := d.Request(url, http.MethodPut, func(req *resty.Request) { + req.SetBody(driver.NewLimitedUploadStream(ctx, stream)).SetContext(ctx) }, nil) return err } @@ -175,7 +171,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(byteData)) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) if err != nil { return err } diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index eb96a42ad29..f2594e78f5e 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -10,7 +10,6 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/aliyun/aliyun-oss-go-sdk/oss" jsoniter "github.com/json-iterator/go" @@ -430,13 +429,10 @@ func (d *PikPak) UploadByOSS(ctx context.Context, params *S3Params, s model.File return err } - err = bucket.PutObject(params.Key, &stream.ReaderWithCtx{ - Reader: &stream.ReaderUpdatingProgress{ - Reader: s, - UpdateProgress: up, - }, - Ctx: ctx, - }, OssOption(params)...) + err = bucket.PutObject(params.Key, driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }), OssOption(params)...) if err != nil { return err } @@ -522,11 +518,8 @@ func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSi continue } - b := bytes.NewBuffer(buf) - if part, err = bucket.UploadPart(imur, &stream.ReaderWithCtx{ - Reader: b, - Ctx: ctx, - }, chunk.Size, chunk.Number, OssOption(params)...); err == nil { + b := driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(buf)) + if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, OssOption(params)...); err == nil { break } } diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 8674fbab26f..04757b1b1cf 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -1,6 +1,7 @@ package quark import ( + "bytes" "context" "crypto/md5" "crypto/sha1" @@ -178,7 +179,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File } // part up partSize := pre.Metadata.PartSize - var bytes []byte + var part []byte md5s := make([]string, 0) defaultBytes := make([]byte, partSize) total := stream.GetSize() @@ -189,17 +190,18 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File return ctx.Err() } if left > int64(partSize) { - bytes = defaultBytes + part = defaultBytes } else { - bytes = make([]byte, left) + part = make([]byte, left) } - _, err := io.ReadFull(tempFile, bytes) + _, err := io.ReadFull(tempFile, part) if err != nil { return err } - left -= int64(len(bytes)) + left -= int64(len(part)) log.Debugf("left: %d", left) - m, err := d.upPart(ctx, pre, stream.GetMimetype(), partNumber, bytes) + reader := driver.NewLimitedUploadStream(ctx, bytes.NewReader(part)) + m, err := d.upPart(ctx, pre, stream.GetMimetype(), partNumber, reader) //m, err := driver.UpPart(pre, file.GetMIMEType(), partNumber, bytes, account, md5Str, sha1Str) if err != nil { return err diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index df27af6714f..9a3bdc1c00a 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "io" "net/http" "strconv" "strings" @@ -119,7 +120,7 @@ func (d *QuarkOrUC) upHash(md5, sha1, taskId string) (bool, error) { return resp.Data.Finish, err } -func (d *QuarkOrUC) upPart(ctx context.Context, pre UpPreResp, mineType string, partNumber int, bytes []byte) (string, error) { +func (d *QuarkOrUC) upPart(ctx context.Context, pre UpPreResp, mineType string, partNumber int, bytes io.Reader) (string, error) { //func (driver QuarkOrUC) UpPart(pre UpPreResp, mineType string, partNumber int, bytes []byte, account *model.Account, md5Str, sha1Str string) (string, error) { timeStr := time.Now().UTC().Format(http.TimeFormat) data := base.Json{ @@ -163,6 +164,9 @@ x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit "partNumber": strconv.Itoa(partNumber), "uploadId": pre.Data.UploadId, }).SetBody(bytes).Put(u) + if err != nil { + return "", err + } if res.StatusCode() != 200 { return "", fmt.Errorf("up status: %d, error: %s", res.StatusCode(), res.String()) } @@ -230,6 +234,9 @@ x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit SetQueryParams(map[string]string{ "uploadId": pre.Data.UploadId, }).SetBody(body).Post(u) + if err != nil { + return err + } if res.StatusCode() != 200 { return fmt.Errorf("up status: %d, error: %s", res.StatusCode(), res.String()) } diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 2ab972caff7..0fa64041d26 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -12,7 +12,6 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - istream "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" "github.com/aws/aws-sdk-go/aws" @@ -387,8 +386,8 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea } uploader := s3manager.NewUploader(s) buf := make([]byte, 1024*1024*2) - fup := &istream.ReaderUpdatingProgress{ - Reader: &istream.SimpleReaderWithSize{ + fup := &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ Reader: f, Size: int64(len(buf)), }, @@ -402,12 +401,19 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea } return nil, err } + reader := bytes.NewReader(buf[:n]) _, err = uploader.S3.UploadPartWithContext(ctx, &s3.UploadPartInput{ UploadId: &uploadInitResp.Data.UploadID, Key: &uploadInitResp.Data.Key, Bucket: &uploadInitResp.Data.Bucket, PartNumber: aws.Int64(partNumber), - Body: bytes.NewReader(buf[:n]), + Body: struct { + *driver.RateLimitReader + io.Seeker + }{ + RateLimitReader: driver.NewLimitedUploadStream(ctx, reader), + Seeker: reader, + }, }) if err != nil { return nil, err diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index a7e924e2e8c..b741148983e 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -4,18 +4,17 @@ import ( "bytes" "context" "fmt" - "github.com/alist-org/alist/v3/server/common" "io" "net/url" stdpath "path" "strings" "time" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/pkg/cron" - "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/server/common" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -174,10 +173,10 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up input := &s3manager.UploadInput{ Bucket: &d.Bucket, Key: &key, - Body: &stream.ReaderUpdatingProgress{ + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, - }, + }), ContentType: &contentType, } _, err := uploader.UploadWithContext(ctx, input) diff --git a/drivers/seafile/driver.go b/drivers/seafile/driver.go index f23038d151d..239f57dd949 100644 --- a/drivers/seafile/driver.go +++ b/drivers/seafile/driver.go @@ -3,7 +3,6 @@ package seafile import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "net/http" "strings" "time" @@ -215,10 +214,10 @@ func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame u := string(res) u = u[1 : len(u)-1] // remove quotes _, err = d.request(http.MethodPost, u, func(req *resty.Request) { - r := &stream.ReaderUpdatingProgress{ + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, - } + }) req.SetFileReader("file", s.GetName(), r). SetFormData(map[string]string{ "parent_dir": path, diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go index 1f216598d2d..7498ce39f66 100644 --- a/drivers/sftp/driver.go +++ b/drivers/sftp/driver.go @@ -111,7 +111,7 @@ func (d *SFTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea defer func() { _ = dstFile.Close() }() - err = utils.CopyWithCtx(ctx, dstFile, stream, stream.GetSize(), up) + err = utils.CopyWithCtx(ctx, dstFile, driver.NewLimitedUploadStream(ctx, stream), stream.GetSize(), up) return err } diff --git a/drivers/smb/driver.go b/drivers/smb/driver.go index 9632f24e0eb..c292e92e03a 100644 --- a/drivers/smb/driver.go +++ b/drivers/smb/driver.go @@ -186,7 +186,7 @@ func (d *SMB) Put(ctx context.Context, dstDir model.Obj, stream model.FileStream _ = d.fs.Remove(fullPath) } }() - err = utils.CopyWithCtx(ctx, out, stream, stream.GetSize(), up) + err = utils.CopyWithCtx(ctx, out, driver.NewLimitedUploadStream(ctx, stream), stream.GetSize(), up) if err != nil { return err } diff --git a/drivers/teambition/driver.go b/drivers/teambition/driver.go index c75d2ac00b6..b37c324b288 100644 --- a/drivers/teambition/driver.go +++ b/drivers/teambition/driver.go @@ -148,7 +148,7 @@ func (d *Teambition) Put(ctx context.Context, dstDir model.Obj, stream model.Fil var newFile *FileUpload if stream.GetSize() <= 20971520 { // post upload - newFile, err = d.upload(ctx, stream, token) + newFile, err = d.upload(ctx, stream, token, up) } else { // chunk upload //err = base.ErrNotImplement diff --git a/drivers/teambition/util.go b/drivers/teambition/util.go index 181cc58f64c..01c12cb17a1 100644 --- a/drivers/teambition/util.go +++ b/drivers/teambition/util.go @@ -1,6 +1,7 @@ package teambition import ( + "bytes" "context" "errors" "fmt" @@ -120,11 +121,15 @@ func (d *Teambition) getFiles(parentId string) ([]model.Obj, error) { return files, nil } -func (d *Teambition) upload(ctx context.Context, file model.FileStreamer, token string) (*FileUpload, error) { +func (d *Teambition) upload(ctx context.Context, file model.FileStreamer, token string, up driver.UpdateProgress) (*FileUpload, error) { prefix := "tcs" if d.isInternational() { prefix = "us-tcs" } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) var newFile FileUpload res, err := base.RestyClient.R(). SetContext(ctx). @@ -134,7 +139,8 @@ func (d *Teambition) upload(ctx context.Context, file model.FileStreamer, token "type": file.GetMimetype(), "size": strconv.FormatInt(file.GetSize(), 10), "lastModifiedDate": time.Now().Format("Mon Jan 02 2006 15:04:05 GMT+0800 (中国标准时间)"), - }).SetMultipartField("file", file.GetName(), file.GetMimetype(), file). + }). + SetMultipartField("file", file.GetName(), file.GetMimetype(), reader). Post(fmt.Sprintf("https://%s.teambition.net/upload", prefix)) if err != nil { return nil, err @@ -183,10 +189,9 @@ func (d *Teambition) chunkUpload(ctx context.Context, file model.FileStreamer, t "Authorization": token, "Content-Type": "application/octet-stream", "Referer": referer, - }).SetBody(chunkData).Post(u) - if err != nil { - return nil, err - } + }). + SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(chunkData))). + Post(u) if err != nil { return nil, err } @@ -252,7 +257,10 @@ func (d *Teambition) newUpload(ctx context.Context, dstDir model.Obj, stream mod Key: &uploadToken.Upload.Key, ContentDisposition: &uploadToken.Upload.ContentDisposition, ContentType: &uploadToken.Upload.ContentType, - Body: stream, + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: stream, + UpdateProgress: up, + }), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index 362de69e0a0..82962b8148a 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -228,7 +228,7 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt res, err := base.RestyClient.R(). SetContext(ctx). SetQueryParams(params). - SetFileReader("file", stream.GetName(), bytes.NewReader(byteData)). + SetFileReader("file", stream.GetName(), driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))). SetHeader("Cookie", d.Cookie). Post(u) if err != nil { diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 1b7f0af6a33..7f41d003838 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -3,7 +3,6 @@ package thunder import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "net/http" "strconv" "strings" @@ -383,10 +382,10 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.Fi Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), - Body: &stream.ReaderUpdatingProgress{ + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, - }, + }), }) return err } diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index 96dd7e8ecce..7ce71f7d265 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -508,7 +508,7 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), - Body: io.TeeReader(stream, driver.NewProgress(stream.GetSize(), up)), + Body: driver.NewLimitedUploadStream(ctx, io.TeeReader(stream, driver.NewProgress(stream.GetSize(), up))), }) return err } diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index 93e07ca98ea..2194bdc6e9c 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -8,7 +8,6 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" @@ -414,10 +413,10 @@ func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.F Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), - Body: &stream.ReaderUpdatingProgress{ + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: file, UpdateProgress: up, - }, + }), }) return err } diff --git a/drivers/trainbit/driver.go b/drivers/trainbit/driver.go index 2b1815ed66f..f4f4bf3fa90 100644 --- a/drivers/trainbit/driver.go +++ b/drivers/trainbit/driver.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/http" "net/url" @@ -59,7 +58,7 @@ func (d *Trainbit) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } var jsonData any - json.Unmarshal(data, &jsonData) + err = json.Unmarshal(data, &jsonData) if err != nil { return nil, err } @@ -122,10 +121,10 @@ func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, s model.FileStream query.Add("guid", guid) query.Add("name", url.QueryEscape(local2provider(s.GetName(), false)+".")) endpoint.RawQuery = query.Encode() - progressReader := &stream.ReaderUpdatingProgress{ + progressReader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, - } + }) req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), progressReader) if err != nil { return err diff --git a/drivers/url_tree/driver.go b/drivers/url_tree/driver.go index 569b3fba5c7..f97d5cc5e76 100644 --- a/drivers/url_tree/driver.go +++ b/drivers/url_tree/driver.go @@ -3,7 +3,6 @@ package url_tree import ( "context" "errors" - "github.com/alist-org/alist/v3/internal/op" stdpath "path" "strings" "sync" @@ -11,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" log "github.com/sirupsen/logrus" ) diff --git a/drivers/uss/driver.go b/drivers/uss/driver.go index 3c54797c43e..2e219050649 100644 --- a/drivers/uss/driver.go +++ b/drivers/uss/driver.go @@ -126,13 +126,10 @@ func (d *USS) Remove(ctx context.Context, obj model.Obj) error { func (d *USS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { return d.client.Put(&upyun.PutObjectConfig{ Path: getKey(path.Join(dstDir.GetPath(), s.GetName()), false), - Reader: &stream.ReaderWithCtx{ - Reader: &stream.ReaderUpdatingProgress{ - Reader: s, - UpdateProgress: up, - }, - Ctx: ctx, - }, + Reader: driver.NewLimitedUploadStream(ctx, &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }), }) } diff --git a/drivers/vtencent/util.go b/drivers/vtencent/util.go index ba87f1abe51..91db54b7ad5 100644 --- a/drivers/vtencent/util.go +++ b/drivers/vtencent/util.go @@ -278,7 +278,8 @@ func (d *Vtencent) FileUpload(ctx context.Context, dstDir model.Obj, stream mode input := &s3manager.UploadInput{ Bucket: aws.String(fmt.Sprintf("%s-%d", params.StorageBucket, params.StorageAppID)), Key: ¶ms.Video.StoragePath, - Body: io.TeeReader(stream, io.MultiWriter(hash, driver.NewProgress(stream.GetSize(), up))), + Body: driver.NewLimitedUploadStream(ctx, + io.TeeReader(stream, io.MultiWriter(hash, driver.NewProgress(stream.GetSize(), up)))), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index 35240c498e8..45150fca57d 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -2,7 +2,6 @@ package webdav import ( "context" - "github.com/alist-org/alist/v3/internal/stream" "net/http" "os" "path" @@ -99,13 +98,11 @@ func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer r.Header.Set("Content-Type", s.GetMimetype()) r.ContentLength = s.GetSize() } - err := d.client.WriteStream(path.Join(dstDir.GetPath(), s.GetName()), &stream.ReaderWithCtx{ - Reader: &stream.ReaderUpdatingProgress{ - Reader: s, - UpdateProgress: up, - }, - Ctx: ctx, - }, 0644, callback) + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + err := d.client.WriteStream(path.Join(dstDir.GetPath(), s.GetName()), reader, 0644, callback) return err } diff --git a/drivers/weiyun/driver.go b/drivers/weiyun/driver.go index 59bd7237088..90793d333f8 100644 --- a/drivers/weiyun/driver.go +++ b/drivers/weiyun/driver.go @@ -70,7 +70,7 @@ func (d *WeiYun) Init(ctx context.Context) error { if d.client.LoginType() == 1 { d.cron = cron.NewCron(time.Minute * 5) d.cron.Do(func() { - d.client.KeepAlive() + _ = d.client.KeepAlive() }) } @@ -364,12 +364,13 @@ func (d *WeiYun) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr threadG.Go(func(ctx context.Context) error { for { channel.Len = int(math.Min(float64(stream.GetSize()-channel.Offset), float64(channel.Len))) + len64 := int64(channel.Len) upData, err := d.client.UploadFile(upCtx, channel, preData.UploadAuthData, - io.NewSectionReader(file, channel.Offset, int64(channel.Len))) + driver.NewLimitedUploadStream(ctx, io.NewSectionReader(file, channel.Offset, len64))) if err != nil { return err } - cur := total.Add(int64(channel.Len)) + cur := total.Add(len64) up(float64(cur) * 100.0 / float64(stream.GetSize())) // 上传完成 if upData.UploadState != 1 { diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index 86093fc14fa..82ec05a919e 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -155,7 +155,7 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre _, err := d.client.Upload2C(d.getSpaceType(), wopan.Upload2CFile{ Name: stream.GetName(), Size: stream.GetSize(), - Content: stream, + Content: driver.NewLimitedUploadStream(ctx, stream), ContentType: stream.GetMimetype(), }, dstDir.GetID(), d.FamilyID, wopan.Upload2COption{ OnProgress: func(current, total int64) { diff --git a/drivers/yandex_disk/driver.go b/drivers/yandex_disk/driver.go index fe858519a48..6e5ca05c7d0 100644 --- a/drivers/yandex_disk/driver.go +++ b/drivers/yandex_disk/driver.go @@ -2,7 +2,6 @@ package yandex_disk import ( "context" - "github.com/alist-org/alist/v3/internal/stream" "net/http" "path" "strconv" @@ -118,10 +117,11 @@ func (d *YandexDisk) Put(ctx context.Context, dstDir model.Obj, s model.FileStre if err != nil { return err } - req, err := http.NewRequestWithContext(ctx, resp.Method, resp.Href, &stream.ReaderUpdatingProgress{ + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, }) + req, err := http.NewRequestWithContext(ctx, resp.Method, resp.Href, reader) if err != nil { return err } diff --git a/go.mod b/go.mod index 2bf4ba3e90c..7bf8a4bb846 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 - github.com/xhofe/tache v0.1.3 + github.com/xhofe/tache v0.1.5 github.com/xhofe/wopan-sdk-go v0.1.3 github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 @@ -102,6 +102,7 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/text v0.2.0 // indirect + github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect @@ -170,7 +171,6 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/pgx/v5 v5.5.5 // indirect - github.com/jaevor/go-nanoid v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -240,7 +240,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.10.0 // indirect + golang.org/x/sync v0.10.0 golang.org/x/sys v0.28.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 diff --git a/go.sum b/go.sum index db58dea2956..a51e0c6a28c 100644 --- a/go.sum +++ b/go.sum @@ -337,8 +337,6 @@ github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jaevor/go-nanoid v1.3.0 h1:nD+iepesZS6pr3uOVf20vR9GdGgJW1HPaR46gtrxzkg= -github.com/jaevor/go-nanoid v1.3.0/go.mod h1:SI+jFaPuddYkqkVQoNGHs81navCtH388TcrH0RqFKgY= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -403,6 +401,8 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= +github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -596,8 +596,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= -github.com/xhofe/tache v0.1.3 h1:MipxzlljYX29E1YI/SLC7hVomVF+51iP1OUzlsuq1wE= -github.com/xhofe/tache v0.1.3/go.mod h1:iKumPFvywf30FRpAHHCt64G0JHLMzT0K+wyGedHsmTQ= +github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= +github.com/xhofe/tache v0.1.5/go.mod h1:PYt6I/XUKliSg1uHlgsk6ha+le/f6PAvjUtFZAVl3a8= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 5e8a2be4271..de3b8af91a3 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -11,6 +11,7 @@ import ( "github.com/alist-org/alist/v3/pkg/utils/random" "github.com/pkg/errors" "gorm.io/gorm" + "strconv" ) var initialSettingItems []model.SettingItem @@ -191,12 +192,12 @@ func InitialSettings() []model.SettingItem { {Key: conf.LdapDefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.LDAP, Flag: model.PRIVATE}, {Key: conf.LdapLoginTips, Value: "login with ldap", Type: conf.TypeString, Group: model.LDAP, Flag: model.PUBLIC}, - //s3 settings + // s3 settings {Key: conf.S3AccessKeyId, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3SecretAccessKey, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, {Key: conf.S3Buckets, Value: "[]", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, - //ftp settings + // ftp settings {Key: conf.FTPPublicHost, Value: "127.0.0.1", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPPasvPortMap, Value: "", Type: conf.TypeText, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPProxyUserAgent, Value: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + @@ -205,6 +206,18 @@ func InitialSettings() []model.SettingItem { {Key: conf.FTPImplicitTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPTLSPrivateKeyPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPTLSPublicCertPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + + // traffic settings + {Key: conf.TaskOfflineDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Download.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskOfflineDownloadTransferThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Transfer.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Upload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskCopyThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Copy.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskDecompressDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Decompress.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskDecompressUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.DecompressUpload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.StreamMaxClientDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.StreamMaxClientUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.StreamMaxServerDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.StreamMaxServerUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, } initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { diff --git a/internal/bootstrap/stream_limit.go b/internal/bootstrap/stream_limit.go new file mode 100644 index 00000000000..5ece71e4beb --- /dev/null +++ b/internal/bootstrap/stream_limit.go @@ -0,0 +1,53 @@ +package bootstrap + +import ( + "context" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/stream" + "golang.org/x/time/rate" +) + +type blockBurstLimiter struct { + *rate.Limiter +} + +func (l blockBurstLimiter) WaitN(ctx context.Context, total int) error { + for total > 0 { + n := l.Burst() + if l.Limiter.Limit() == rate.Inf || n > total { + n = total + } + err := l.Limiter.WaitN(ctx, n) + if err != nil { + return err + } + total -= n + } + return nil +} + +func streamFilterNegative(limit int) (rate.Limit, int) { + if limit < 0 { + return rate.Inf, 0 + } + return rate.Limit(limit) * 1024.0, limit * 1024 +} + +func initLimiter(limiter *stream.Limiter, s string) { + clientDownLimit, burst := streamFilterNegative(setting.GetInt(s, -1)) + *limiter = blockBurstLimiter{Limiter: rate.NewLimiter(clientDownLimit, burst)} + op.RegisterSettingChangingCallback(func() { + newLimit, newBurst := streamFilterNegative(setting.GetInt(s, -1)) + (*limiter).SetLimit(newLimit) + (*limiter).SetBurst(newBurst) + }) +} + +func InitStreamLimit() { + initLimiter(&stream.ClientDownloadLimit, conf.StreamMaxClientDownloadSpeed) + initLimiter(&stream.ClientUploadLimit, conf.StreamMaxClientUploadSpeed) + initLimiter(&stream.ServerDownloadLimit, conf.StreamMaxServerDownloadSpeed) + initLimiter(&stream.ServerUploadLimit, conf.StreamMaxServerUploadSpeed) +} diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go index 9c30c3926b5..c67e3029b61 100644 --- a/internal/bootstrap/task.go +++ b/internal/bootstrap/task.go @@ -5,17 +5,44 @@ import ( "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" "github.com/xhofe/tache" ) +func taskFilterNegative(num int) int64 { + if num < 0 { + num = 0 + } + return int64(num) +} + func InitTaskManager() { - fs.UploadTaskManager = tache.NewManager[*fs.UploadTask](tache.WithWorks(conf.Conf.Tasks.Upload.Workers), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //upload will not support persist - fs.CopyTaskManager = tache.NewManager[*fs.CopyTask](tache.WithWorks(conf.Conf.Tasks.Copy.Workers), tache.WithPersistFunction(db.GetTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant), db.UpdateTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Copy.MaxRetry)) - tool.DownloadTaskManager = tache.NewManager[*tool.DownloadTask](tache.WithWorks(conf.Conf.Tasks.Download.Workers), tache.WithPersistFunction(db.GetTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant), db.UpdateTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Download.MaxRetry)) - tool.TransferTaskManager = tache.NewManager[*tool.TransferTask](tache.WithWorks(conf.Conf.Tasks.Transfer.Workers), tache.WithPersistFunction(db.GetTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant), db.UpdateTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Transfer.MaxRetry)) + fs.UploadTaskManager = tache.NewManager[*fs.UploadTask](tache.WithWorks(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //upload will not support persist + op.RegisterSettingChangingCallback(func() { + fs.UploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers))) + }) + fs.CopyTaskManager = tache.NewManager[*fs.CopyTask](tache.WithWorks(setting.GetInt(conf.TaskCopyThreadsNum, conf.Conf.Tasks.Copy.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant), db.UpdateTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Copy.MaxRetry)) + op.RegisterSettingChangingCallback(func() { + fs.CopyTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskCopyThreadsNum, conf.Conf.Tasks.Copy.Workers))) + }) + tool.DownloadTaskManager = tache.NewManager[*tool.DownloadTask](tache.WithWorks(setting.GetInt(conf.TaskOfflineDownloadThreadsNum, conf.Conf.Tasks.Download.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant), db.UpdateTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Download.MaxRetry)) + op.RegisterSettingChangingCallback(func() { + tool.DownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskOfflineDownloadThreadsNum, conf.Conf.Tasks.Download.Workers))) + }) + tool.TransferTaskManager = tache.NewManager[*tool.TransferTask](tache.WithWorks(setting.GetInt(conf.TaskOfflineDownloadTransferThreadsNum, conf.Conf.Tasks.Transfer.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant), db.UpdateTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Transfer.MaxRetry)) + op.RegisterSettingChangingCallback(func() { + tool.TransferTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskOfflineDownloadTransferThreadsNum, conf.Conf.Tasks.Transfer.Workers))) + }) if len(tool.TransferTaskManager.GetAll()) == 0 { //prevent offline downloaded files from being deleted CleanTempDir() } - fs.ArchiveDownloadTaskManager = tache.NewManager[*fs.ArchiveDownloadTask](tache.WithWorks(conf.Conf.Tasks.Decompress.Workers), tache.WithPersistFunction(db.GetTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant), db.UpdateTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Decompress.MaxRetry)) - fs.ArchiveContentUploadTaskManager.Manager = tache.NewManager[*fs.ArchiveContentUploadTask](tache.WithWorks(conf.Conf.Tasks.DecompressUpload.Workers), tache.WithMaxRetry(conf.Conf.Tasks.DecompressUpload.MaxRetry)) //decompress upload will not support persist + fs.ArchiveDownloadTaskManager = tache.NewManager[*fs.ArchiveDownloadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant), db.UpdateTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Decompress.MaxRetry)) + op.RegisterSettingChangingCallback(func() { + fs.ArchiveDownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers))) + }) + fs.ArchiveContentUploadTaskManager.Manager = tache.NewManager[*fs.ArchiveContentUploadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.DecompressUpload.MaxRetry)) //decompress upload will not support persist + op.RegisterSettingChangingCallback(func() { + fs.ArchiveContentUploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers))) + }) } diff --git a/internal/conf/const.go b/internal/conf/const.go index 0e534350de3..fa286e46474 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -115,6 +115,18 @@ const ( FTPImplicitTLS = "ftp_implicit_tls" FTPTLSPrivateKeyPath = "ftp_tls_private_key_path" FTPTLSPublicCertPath = "ftp_tls_public_cert_path" + + // traffic + TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" + TaskOfflineDownloadTransferThreadsNum = "offline_download_transfer_task_threads_num" + TaskUploadThreadsNum = "upload_task_threads_num" + TaskCopyThreadsNum = "copy_task_threads_num" + TaskDecompressDownloadThreadsNum = "decompress_download_task_threads_num" + TaskDecompressUploadThreadsNum = "decompress_upload_task_threads_num" + StreamMaxClientDownloadSpeed = "max_client_download_speed" + StreamMaxClientUploadSpeed = "max_client_upload_speed" + StreamMaxServerDownloadSpeed = "max_server_download_speed" + StreamMaxServerUploadSpeed = "max_server_upload_speed" ) const ( diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 292f8e6a480..05f0fe24576 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -77,6 +77,29 @@ type Remove interface { } type Put interface { + // Put a file (provided as a FileStreamer) into the driver + // Besides the most basic upload functionality, the following features also need to be implemented: + // 1. Canceling (when `<-ctx.Done()` returns), by the following methods: + // (1) Use request methods that carry context, such as the following: + // a. http.NewRequestWithContext + // b. resty.Request.SetContext + // c. s3manager.Uploader.UploadWithContext + // d. utils.CopyWithCtx + // (2) Use a `driver.ReaderWithCtx` or a `driver.NewLimitedUploadStream` + // (3) Use `utils.IsCanceled` to check if the upload has been canceled during the upload process, + // this is typically applicable to chunked uploads. + // 2. Submit upload progress (via `up`) in real-time. There are three recommended ways as follows: + // (1) Use `utils.CopyWithCtx` + // (2) Use `driver.ReaderUpdatingProgress` + // (3) Use `driver.Progress` with `io.TeeReader` + // 3. Slow down upload speed (via `stream.ServerUploadLimit`). It requires you to wrap the read stream + // in a `driver.RateLimitReader` or a `driver.RateLimitFile` after calculating the file's hash and + // before uploading the file or file chunks. Or you can directly call `driver.ServerUploadLimitWaitN` + // if your file chunks are sufficiently small (less than about 50KB). + // NOTE that the network speed may be significantly slower than the stream's read speed. Therefore, if + // you use a `errgroup.Group` to upload each chunk in parallel, you should consider using a recursive + // mutex like `semaphore.Weighted` to limit the maximum number of upload threads, preventing excessive + // memory usage caused by buffering too many file chunks awaiting upload. Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up UpdateProgress) error } @@ -113,6 +136,29 @@ type CopyResult interface { } type PutResult interface { + // Put a file (provided as a FileStreamer) into the driver and return the put obj + // Besides the most basic upload functionality, the following features also need to be implemented: + // 1. Canceling (when `<-ctx.Done()` returns), which can be supported by the following methods: + // (1) Use request methods that carry context, such as the following: + // a. http.NewRequestWithContext + // b. resty.Request.SetContext + // c. s3manager.Uploader.UploadWithContext + // d. utils.CopyWithCtx + // (2) Use a `driver.ReaderWithCtx` or `driver.NewLimitedUploadStream` + // (3) Use `utils.IsCanceled` to check if the upload has been canceled during the upload process, + // this is typically applicable to chunked uploads. + // 2. Submit upload progress (via `up`) in real-time. There are three recommended ways as follows: + // (1) Use `utils.CopyWithCtx` + // (2) Use `driver.ReaderUpdatingProgress` + // (3) Use `driver.Progress` with `io.TeeReader` + // 3. Slow down upload speed (via `stream.ServerUploadLimit`). It requires you to wrap the read stream + // in a `driver.RateLimitReader` or a `driver.RateLimitFile` after calculating the file's hash and + // before uploading the file or file chunks. Or you can directly call `driver.ServerUploadLimitWaitN` + // if your file chunks are sufficiently small (less than about 50KB). + // NOTE that the network speed may be significantly slower than the stream's read speed. Therefore, if + // you use a `errgroup.Group` to upload each chunk in parallel, you should consider using a recursive + // mutex like `semaphore.Weighted` to limit the maximum number of upload threads, preventing excessive + // memory usage caused by buffering too many file chunks awaiting upload. Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up UpdateProgress) (model.Obj, error) } @@ -159,28 +205,6 @@ type ArchiveDecompressResult interface { ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) } -type UpdateProgress = model.UpdateProgress - -type Progress struct { - Total int64 - Done int64 - up UpdateProgress -} - -func (p *Progress) Write(b []byte) (n int, err error) { - n = len(b) - p.Done += int64(n) - p.up(float64(p.Done) / float64(p.Total) * 100) - return -} - -func NewProgress(total int64, up UpdateProgress) *Progress { - return &Progress{ - Total: total, - up: up, - } -} - type Reference interface { InitReference(storage Driver) error } diff --git a/internal/driver/utils.go b/internal/driver/utils.go new file mode 100644 index 00000000000..2af850ecb8d --- /dev/null +++ b/internal/driver/utils.go @@ -0,0 +1,62 @@ +package driver + +import ( + "context" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "io" +) + +type UpdateProgress = model.UpdateProgress + +type Progress struct { + Total int64 + Done int64 + up UpdateProgress +} + +func (p *Progress) Write(b []byte) (n int, err error) { + n = len(b) + p.Done += int64(n) + p.up(float64(p.Done) / float64(p.Total) * 100) + return +} + +func NewProgress(total int64, up UpdateProgress) *Progress { + return &Progress{ + Total: total, + up: up, + } +} + +type RateLimitReader = stream.RateLimitReader + +type RateLimitWriter = stream.RateLimitWriter + +type RateLimitFile = stream.RateLimitFile + +func NewLimitedUploadStream(ctx context.Context, r io.Reader) *RateLimitReader { + return &RateLimitReader{ + Reader: r, + Limiter: stream.ServerUploadLimit, + Ctx: ctx, + } +} + +func NewLimitedUploadFile(ctx context.Context, f model.File) *RateLimitFile { + return &RateLimitFile{ + File: f, + Limiter: stream.ServerUploadLimit, + Ctx: ctx, + } +} + +func ServerUploadLimitWaitN(ctx context.Context, n int) error { + return stream.ServerUploadLimit.WaitN(ctx, n) +} + +type ReaderWithCtx = stream.ReaderWithCtx + +type ReaderUpdatingProgress = stream.ReaderUpdatingProgress + +type SimpleReaderWithSize = stream.SimpleReaderWithSize diff --git a/internal/model/setting.go b/internal/model/setting.go index 9b60d98a76e..93b81fe5941 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -12,6 +12,7 @@ const ( LDAP S3 FTP + TRAFFIC ) const ( diff --git a/internal/net/serve.go b/internal/net/serve.go index 6216cd217e9..c75e611f8d5 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -3,6 +3,7 @@ package net import ( "compress/gzip" "context" + "crypto/tls" "fmt" "io" "mime" @@ -14,7 +15,6 @@ import ( "sync" "time" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/http_range" @@ -264,7 +264,7 @@ var httpClient *http.Client func HttpClient() *http.Client { once.Do(func() { - httpClient = base.NewHttpClient() + httpClient = NewHttpClient() httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { if len(via) >= 10 { return errors.New("stopped after 10 redirects") @@ -275,3 +275,13 @@ func HttpClient() *http.Client { }) return httpClient } + +func NewHttpClient() *http.Client { + return &http.Client{ + Timeout: time.Hour * 48, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + }, + } +} diff --git a/internal/op/setting.go b/internal/op/setting.go index 50eba3f744e..36a792b0a6a 100644 --- a/internal/op/setting.go +++ b/internal/op/setting.go @@ -26,9 +26,18 @@ var settingGroupCacheF = func(key string, item []model.SettingItem) { settingGroupCache.Set(key, item, cache.WithEx[[]model.SettingItem](time.Hour)) } +var settingChangingCallbacks = make([]func(), 0) + +func RegisterSettingChangingCallback(f func()) { + settingChangingCallbacks = append(settingChangingCallbacks, f) +} + func SettingCacheUpdate() { settingCache.Clear() settingGroupCache.Clear() + for _, cb := range settingChangingCallbacks { + cb() + } } func GetPublicSettingsMap() map[string]string { diff --git a/internal/stream/limit.go b/internal/stream/limit.go new file mode 100644 index 00000000000..3b32a55ff6d --- /dev/null +++ b/internal/stream/limit.go @@ -0,0 +1,152 @@ +package stream + +import ( + "context" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "golang.org/x/time/rate" + "io" + "time" +) + +type Limiter interface { + Limit() rate.Limit + Burst() int + TokensAt(time.Time) float64 + Tokens() float64 + Allow() bool + AllowN(time.Time, int) bool + Reserve() *rate.Reservation + ReserveN(time.Time, int) *rate.Reservation + Wait(context.Context) error + WaitN(context.Context, int) error + SetLimit(rate.Limit) + SetLimitAt(time.Time, rate.Limit) + SetBurst(int) + SetBurstAt(time.Time, int) +} + +var ( + ClientDownloadLimit Limiter + ClientUploadLimit Limiter + ServerDownloadLimit Limiter + ServerUploadLimit Limiter +) + +type RateLimitReader struct { + io.Reader + Limiter Limiter + Ctx context.Context +} + +func (r *RateLimitReader) Read(p []byte) (n int, err error) { + if r.Ctx != nil && utils.IsCanceled(r.Ctx) { + return 0, r.Ctx.Err() + } + n, err = r.Reader.Read(p) + if err != nil { + return + } + if r.Limiter != nil { + if r.Ctx == nil { + r.Ctx = context.Background() + } + err = r.Limiter.WaitN(r.Ctx, n) + } + return +} + +func (r *RateLimitReader) Close() error { + if c, ok := r.Reader.(io.Closer); ok { + return c.Close() + } + return nil +} + +type RateLimitWriter struct { + io.Writer + Limiter Limiter + Ctx context.Context +} + +func (w *RateLimitWriter) Write(p []byte) (n int, err error) { + if w.Ctx != nil && utils.IsCanceled(w.Ctx) { + return 0, w.Ctx.Err() + } + n, err = w.Writer.Write(p) + if err != nil { + return + } + if w.Limiter != nil { + if w.Ctx == nil { + w.Ctx = context.Background() + } + err = w.Limiter.WaitN(w.Ctx, n) + } + return +} + +func (w *RateLimitWriter) Close() error { + if c, ok := w.Writer.(io.Closer); ok { + return c.Close() + } + return nil +} + +type RateLimitFile struct { + model.File + Limiter Limiter + Ctx context.Context +} + +func (r *RateLimitFile) Read(p []byte) (n int, err error) { + if r.Ctx != nil && utils.IsCanceled(r.Ctx) { + return 0, r.Ctx.Err() + } + n, err = r.File.Read(p) + if err != nil { + return + } + if r.Limiter != nil { + if r.Ctx == nil { + r.Ctx = context.Background() + } + err = r.Limiter.WaitN(r.Ctx, n) + } + return +} + +func (r *RateLimitFile) ReadAt(p []byte, off int64) (n int, err error) { + if r.Ctx != nil && utils.IsCanceled(r.Ctx) { + return 0, r.Ctx.Err() + } + n, err = r.File.ReadAt(p, off) + if err != nil { + return + } + if r.Limiter != nil { + if r.Ctx == nil { + r.Ctx = context.Background() + } + err = r.Limiter.WaitN(r.Ctx, n) + } + return +} + +type RateLimitRangeReadCloser struct { + model.RangeReadCloserIF + Limiter Limiter +} + +func (rrc RateLimitRangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + rc, err := rrc.RangeReadCloserIF.RangeRead(ctx, httpRange) + if err != nil { + return nil, err + } + return &RateLimitReader{ + Reader: rc, + Limiter: rrc.Limiter, + Ctx: ctx, + }, nil +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 74646bfbccd..5eb6bdc7a4c 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -182,14 +182,24 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) } if ss.Link != nil { if ss.Link.MFile != nil { - ss.mFile = ss.Link.MFile - ss.Reader = ss.Link.MFile - ss.Closers.Add(ss.Link.MFile) + mFile := ss.Link.MFile + if _, ok := mFile.(*os.File); !ok { + mFile = &RateLimitFile{ + File: mFile, + Limiter: ServerDownloadLimit, + Ctx: fs.Ctx, + } + } + ss.mFile = mFile + ss.Reader = mFile + ss.Closers.Add(mFile) return &ss, nil } - if ss.Link.RangeReadCloser != nil { - ss.rangeReadCloser = ss.Link.RangeReadCloser + ss.rangeReadCloser = RateLimitRangeReadCloser{ + RangeReadCloserIF: ss.Link.RangeReadCloser, + Limiter: ServerDownloadLimit, + } ss.Add(ss.rangeReadCloser) return &ss, nil } @@ -198,6 +208,10 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) if err != nil { return nil, err } + rrc = RateLimitRangeReadCloser{ + RangeReadCloserIF: rrc, + Limiter: ServerDownloadLimit, + } ss.rangeReadCloser = rrc ss.Add(rrc) return &ss, nil @@ -259,7 +273,7 @@ func (ss *SeekableStream) CacheFullInTempFile() (model.File, error) { if ss.tmpFile != nil { return ss.tmpFile, nil } - if ss.mFile != nil { + if _, ok := ss.mFile.(*os.File); ok { return ss.mFile, nil } tmpF, err := utils.CreateTempFile(ss, ss.GetSize()) @@ -276,7 +290,7 @@ func (ss *SeekableStream) CacheFullInTempFileAndUpdateProgress(up model.UpdatePr if ss.tmpFile != nil { return ss.tmpFile, nil } - if ss.mFile != nil { + if _, ok := ss.mFile.(*os.File); ok { return ss.mFile, nil } tmpF, err := utils.CreateTempFile(&ReaderUpdatingProgress{ @@ -293,12 +307,13 @@ func (ss *SeekableStream) CacheFullInTempFileAndUpdateProgress(up model.UpdatePr } func (f *FileStream) SetTmpFile(r *os.File) { - f.Reader = r + f.Add(r) f.tmpFile = r + f.Reader = r } type ReaderWithSize interface { - io.Reader + io.ReadCloser GetSize() int64 } @@ -311,6 +326,13 @@ func (r *SimpleReaderWithSize) GetSize() int64 { return r.Size } +func (r *SimpleReaderWithSize) Close() error { + if c, ok := r.Reader.(io.Closer); ok { + return c.Close() + } + return nil +} + type ReaderUpdatingProgress struct { Reader ReaderWithSize model.UpdateProgress @@ -324,6 +346,10 @@ func (r *ReaderUpdatingProgress) Read(p []byte) (n int, err error) { return n, err } +func (r *ReaderUpdatingProgress) Close() error { + return r.Reader.Close() +} + type SStreamReadAtSeeker interface { model.File GetRawStream() *SeekableStream @@ -534,7 +560,7 @@ func (r *RangeReadReadAtSeeker) Read(p []byte) (n int, err error) { func (r *RangeReadReadAtSeeker) Close() error { if r.headCache != nil { - r.headCache.close() + _ = r.headCache.close() } return r.ss.Close() } @@ -562,17 +588,3 @@ func (f *FileReadAtSeeker) Seek(offset int64, whence int) (int64, error) { func (f *FileReadAtSeeker) Close() error { return f.ss.Close() } - -type ReaderWithCtx struct { - io.Reader - Ctx context.Context -} - -func (r *ReaderWithCtx) Read(p []byte) (n int, err error) { - select { - case <-r.Ctx.Done(): - return 0, r.Ctx.Err() - default: - return r.Reader.Read(p) - } -} diff --git a/internal/stream/util.go b/internal/stream/util.go index 16854c38c80..bb5019e0df2 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -3,6 +3,7 @@ package stream import ( "context" "fmt" + "github.com/alist-org/alist/v3/pkg/utils" "io" "net/http" @@ -76,3 +77,22 @@ func checkContentRange(header *http.Header, offset int64) bool { } return false } + +type ReaderWithCtx struct { + io.Reader + Ctx context.Context +} + +func (r *ReaderWithCtx) Read(p []byte) (n int, err error) { + if utils.IsCanceled(r.Ctx) { + return 0, r.Ctx.Err() + } + return r.Reader.Read(p) +} + +func (r *ReaderWithCtx) Close() error { + if c, ok := r.Reader.(io.Closer); ok { + return c.Close() + } + return nil +} diff --git a/server/common/proxy.go b/server/common/proxy.go index 2d828efdfcc..66854976b08 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/url" + "os" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" @@ -23,11 +24,22 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. if contentType != "" { w.Header().Set("Content-Type", contentType) } - http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) + mFile := link.MFile + if _, ok := mFile.(*os.File); !ok { + mFile = &stream.RateLimitFile{ + File: mFile, + Limiter: stream.ServerDownloadLimit, + Ctx: r.Context(), + } + } + http.ServeContent(w, r, file.GetName(), file.ModTime(), mFile) return nil } else if link.RangeReadCloser != nil { attachFileName(w, file) - net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), link.RangeReadCloser) + net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ + RangeReadCloserIF: link.RangeReadCloser, + Limiter: stream.ServerDownloadLimit, + }) return nil } else if link.Concurrency != 0 || link.PartSize != 0 { attachFileName(w, file) @@ -47,7 +59,10 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. rc, err := down.Download(ctx, req) return rc, err } - net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &model.RangeReadCloser{RangeReader: rangeReader}) + net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ + RangeReadCloserIF: &model.RangeReadCloser{RangeReader: rangeReader}, + Limiter: stream.ServerDownloadLimit, + }) return nil } else { //transparent proxy @@ -65,7 +80,11 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. if r.Method == http.MethodHead { return nil } - _, err = utils.CopyWithBuffer(w, res.Body) + _, err = utils.CopyWithBuffer(w, &stream.RateLimitReader{ + Reader: res.Body, + Limiter: stream.ServerDownloadLimit, + Ctx: r.Context(), + }) if err != nil { return err } diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index f7e018e0f5f..c051a19db21 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -60,7 +60,12 @@ func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownl } func (f *FileDownloadProxy) Read(p []byte) (n int, err error) { - return f.reader.Read(p) + n, err = f.reader.Read(p) + if err != nil { + return + } + err = stream.ClientDownloadLimit.WaitN(f.reader.GetRawStream().Ctx, n) + return } func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index 4d626d0efcb..ee38b1bfb07 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -59,7 +59,12 @@ func (f *FileUploadProxy) Read(p []byte) (n int, err error) { } func (f *FileUploadProxy) Write(p []byte) (n int, err error) { - return f.buffer.Write(p) + n, err = f.buffer.Write(p) + if err != nil { + return + } + err = stream.ClientUploadLimit.WaitN(f.ctx, n) + return } func (f *FileUploadProxy) Seek(offset int64, whence int) (int64, error) { @@ -96,7 +101,6 @@ func (f *FileUploadProxy) Close() error { WebPutAsTask: true, } s.SetTmpFile(f.buffer) - s.Closers.Add(f.buffer) _, err = fs.PutAsTask(f.ctx, dir, s) return err } @@ -127,7 +131,7 @@ func (f *FileUploadWithLengthProxy) Read(p []byte) (n int, err error) { return 0, errs.NotSupport } -func (f *FileUploadWithLengthProxy) Write(p []byte) (n int, err error) { +func (f *FileUploadWithLengthProxy) write(p []byte) (n int, err error) { if f.pipeWriter != nil { select { case e := <-f.errChan: @@ -174,6 +178,15 @@ func (f *FileUploadWithLengthProxy) Write(p []byte) (n int, err error) { } } +func (f *FileUploadWithLengthProxy) Write(p []byte) (n int, err error) { + n, err = f.write(p) + if err != nil { + return + } + err = stream.ClientUploadLimit.WaitN(f.ctx, n) + return +} + func (f *FileUploadWithLengthProxy) Seek(offset int64, whence int) (int64, error) { return 0, errs.NotSupport } diff --git a/server/middlewares/limit.go b/server/middlewares/limit.go index 44c079b37e0..2ccee950c80 100644 --- a/server/middlewares/limit.go +++ b/server/middlewares/limit.go @@ -1,7 +1,9 @@ package middlewares import ( + "github.com/alist-org/alist/v3/internal/stream" "github.com/gin-gonic/gin" + "io" ) func MaxAllowed(n int) gin.HandlerFunc { @@ -14,3 +16,37 @@ func MaxAllowed(n int) gin.HandlerFunc { c.Next() } } + +func UploadRateLimiter(limiter stream.Limiter) gin.HandlerFunc { + return func(c *gin.Context) { + c.Request.Body = &stream.RateLimitReader{ + Reader: c.Request.Body, + Limiter: limiter, + Ctx: c, + } + c.Next() + } +} + +type ResponseWriterWrapper struct { + gin.ResponseWriter + WrapWriter io.Writer +} + +func (w *ResponseWriterWrapper) Write(p []byte) (n int, err error) { + return w.WrapWriter.Write(p) +} + +func DownloadRateLimiter(limiter stream.Limiter) gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer = &ResponseWriterWrapper{ + ResponseWriter: c.Writer, + WrapWriter: &stream.RateLimitWriter{ + Writer: c.Writer, + Limiter: limiter, + Ctx: c, + }, + } + c.Next() + } +} diff --git a/server/router.go b/server/router.go index 63bad60f03f..830051d8f51 100644 --- a/server/router.go +++ b/server/router.go @@ -4,6 +4,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/message" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/handles" @@ -38,13 +39,14 @@ func Init(e *gin.Engine) { WebDav(g.Group("/dav")) S3(g.Group("/s3")) - g.GET("/d/*path", middlewares.Down, handles.Down) - g.GET("/p/*path", middlewares.Down, handles.Proxy) + downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) + g.GET("/d/*path", middlewares.Down, downloadLimiter, handles.Down) + g.GET("/p/*path", middlewares.Down, downloadLimiter, handles.Proxy) g.HEAD("/d/*path", middlewares.Down, handles.Down) g.HEAD("/p/*path", middlewares.Down, handles.Proxy) - g.GET("/ad/*path", middlewares.Down, handles.ArchiveDown) - g.GET("/ap/*path", middlewares.Down, handles.ArchiveProxy) - g.GET("/ae/*path", middlewares.Down, handles.ArchiveInternalExtract) + g.GET("/ad/*path", middlewares.Down, downloadLimiter, handles.ArchiveDown) + g.GET("/ap/*path", middlewares.Down, downloadLimiter, handles.ArchiveProxy) + g.GET("/ae/*path", middlewares.Down, downloadLimiter, handles.ArchiveInternalExtract) g.HEAD("/ad/*path", middlewares.Down, handles.ArchiveDown) g.HEAD("/ap/*path", middlewares.Down, handles.ArchiveProxy) g.HEAD("/ae/*path", middlewares.Down, handles.ArchiveInternalExtract) @@ -173,8 +175,9 @@ func _fs(g *gin.RouterGroup) { g.POST("/copy", handles.FsCopy) g.POST("/remove", handles.FsRemove) g.POST("/remove_empty_directory", handles.FsRemoveEmptyDirectory) - g.PUT("/put", middlewares.FsUp, handles.FsStream) - g.PUT("/form", middlewares.FsUp, handles.FsForm) + uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit) + g.PUT("/put", middlewares.FsUp, uploadLimiter, handles.FsStream) + g.PUT("/form", middlewares.FsUp, uploadLimiter, handles.FsForm) g.POST("/link", middlewares.AuthAdmin, handles.Link) // g.POST("/add_aria2", handles.AddOfflineDownload) // g.POST("/add_qbit", handles.AddQbittorrent) diff --git a/server/webdav.go b/server/webdav.go index cdfdce7d9d3..a735e285527 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -3,6 +3,8 @@ package server import ( "context" "crypto/subtle" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/middlewares" "net/http" "path" "strings" @@ -27,8 +29,10 @@ func WebDav(dav *gin.RouterGroup) { }, } dav.Use(WebDAVAuth) - dav.Any("/*path", ServeWebDAV) - dav.Any("", ServeWebDAV) + uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit) + downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) + dav.Any("/*path", uploadLimiter, downloadLimiter, ServeWebDAV) + dav.Any("", uploadLimiter, downloadLimiter, ServeWebDAV) dav.Handle("PROPFIND", "/*path", ServeWebDAV) dav.Handle("PROPFIND", "", ServeWebDAV) dav.Handle("MKCOL", "/*path", ServeWebDAV) From 30d8c2075630e92fc93c4057bb189b4172219d73 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Sun, 16 Feb 2025 12:24:10 +0800 Subject: [PATCH 451/659] feat(archive): support deprioritize previewing (#7984) --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index de3b8af91a3..026a89e17ad 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -140,6 +140,7 @@ func InitialSettings() []model.SettingItem { {Key: "audio_cover", Value: "https://jsd.nn.ci/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW}, {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, diff --git a/internal/conf/const.go b/internal/conf/const.go index fa286e46474..2234e9bc5c5 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -22,14 +22,15 @@ const ( MainColor = "main_color" // preview - TextTypes = "text_types" - AudioTypes = "audio_types" - VideoTypes = "video_types" - ImageTypes = "image_types" - ProxyTypes = "proxy_types" - ProxyIgnoreHeaders = "proxy_ignore_headers" - AudioAutoplay = "audio_autoplay" - VideoAutoplay = "video_autoplay" + TextTypes = "text_types" + AudioTypes = "audio_types" + VideoTypes = "video_types" + ImageTypes = "image_types" + ProxyTypes = "proxy_types" + ProxyIgnoreHeaders = "proxy_ignore_headers" + AudioAutoplay = "audio_autoplay" + VideoAutoplay = "video_autoplay" + PreviewArchivesByDefault = "preview_archives_by_default" // global HideFiles = "hide_files" From c230f24ebedfa96e9cf949df8066914373cd9eef Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Sun, 16 Feb 2025 12:25:01 +0800 Subject: [PATCH 452/659] fix(archive): decode filename when decompressing zips (#7998 close #7988) --- internal/archive/zip/utils.go | 45 +++++++++++++++++++++++++++++++---- internal/archive/zip/zip.go | 2 +- server/handles/archive.go | 2 +- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/internal/archive/zip/utils.go b/internal/archive/zip/utils.go index 81b47782840..aa51b88eb93 100644 --- a/internal/archive/zip/utils.go +++ b/internal/archive/zip/utils.go @@ -59,7 +59,7 @@ func _decompress(file *zip.File, targetPath, password string, up model.UpdatePro return err } defer rc.Close() - f, err := os.OpenFile(stdpath.Join(targetPath, file.FileInfo().Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + f, err := os.OpenFile(stdpath.Join(targetPath, decodeName(file.FileInfo().Name())), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } @@ -87,12 +87,27 @@ func filterPassword(err error) error { func decodeName(name string) string { b := []byte(name) detector := chardet.NewTextDetector() - result, err := detector.DetectBest(b) + results, err := detector.DetectAll(b) if err != nil { return name } - enc := getEncoding(result.Charset) - if enc == nil { + var ce, re, enc encoding.Encoding + for _, r := range results { + if r.Confidence > 30 { + ce = getCommonEncoding(r.Charset) + if ce != nil { + break + } + } + if re == nil { + re = getEncoding(r.Charset) + } + } + if ce != nil { + enc = ce + } else if re != nil { + enc = re + } else { return name } i := bytes.NewReader(b) @@ -101,8 +116,30 @@ func decodeName(name string) string { return string(content) } +func getCommonEncoding(name string) (enc encoding.Encoding) { + switch name { + case "UTF-8": + enc = unicode.UTF8 + case "UTF-16LE": + enc = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) + case "Shift_JIS": + enc = japanese.ShiftJIS + case "GB-18030": + enc = simplifiedchinese.GB18030 + case "EUC-KR": + enc = korean.EUCKR + case "Big5": + enc = traditionalchinese.Big5 + default: + enc = nil + } + return +} + func getEncoding(name string) (enc encoding.Encoding) { switch name { + case "UTF-8": + enc = unicode.UTF8 case "UTF-16BE": enc = unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM) case "UTF-16LE": diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go index e5285518b05..9dc8cc7638f 100644 --- a/internal/archive/zip/zip.go +++ b/internal/archive/zip/zip.go @@ -35,7 +35,6 @@ func (*Zip) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.Ar for _, file := range zipReader.File { if file.IsEncrypted() { encrypted = true - break } name := strings.TrimPrefix(decodeName(file.Name), "/") @@ -70,6 +69,7 @@ func (*Zip) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.Ar dirObj.IsFolder = true dirObj.Name = stdpath.Base(dir) dirObj.Modified = file.ModTime() + dirObj.Children = make([]model.ObjTree, 0) } if isNewFolder { // 将 文件夹 添加到 父文件夹 diff --git a/server/handles/archive.go b/server/handles/archive.go index 6ff13641b44..fab3916e3ca 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -39,7 +39,7 @@ type ArchiveMetaResp struct { type ArchiveContentResp struct { ObjResp - Children []ArchiveContentResp `json:"children,omitempty"` + Children []ArchiveContentResp `json:"children"` } func toObjsRespWithoutSignAndThumb(obj model.Obj) ObjResp { From 79bef0be9ee14b1087c4e639015674665c947001 Mon Sep 17 00:00:00 2001 From: KirCute_ECT <951206789@qq.com> Date: Sun, 16 Feb 2025 15:11:48 +0800 Subject: [PATCH 453/659] chore: fix build failed (#8005) --- drivers/ilanzou/driver.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 697d85b1b3b..39a311ddbc0 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -13,8 +13,6 @@ import ( "strings" "time" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" From cdc41595bcc1094c8ff2f7a2b7b40763dfb87492 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 24 Feb 2025 23:12:23 +0800 Subject: [PATCH 454/659] feat(github): support GPG verification (#7996 close #7986) * feat(github): support GPG verification * chore --- drivers/github/driver.go | 134 ++++++++++++++++++++++++--------------- drivers/github/meta.go | 32 +++++----- drivers/github/types.go | 7 +- drivers/github/util.go | 72 ++++++++++++++++++++- go.mod | 2 + go.sum | 15 +++++ 6 files changed, 193 insertions(+), 69 deletions(-) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index d1cfd9fbc02..dedd4945bdc 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -3,7 +3,6 @@ package github import ( "context" "encoding/base64" - "errors" "fmt" "io" "net/http" @@ -12,12 +11,14 @@ import ( "sync" "text/template" + "github.com/ProtonMail/go-crypto/openpgp" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -33,6 +34,7 @@ type Github struct { moveMsgTmpl *template.Template isOnBranch bool commitMutex sync.Mutex + pgpEntity *openpgp.Entity } func (d *Github) Config() driver.Config { @@ -102,6 +104,26 @@ func (d *Github) Init(ctx context.Context) error { _, err = d.getBranchHead() d.isOnBranch = err == nil } + if d.GPGPrivateKey != "" { + if d.CommitterName == "" || d.AuthorName == "" { + user, e := d.getAuthenticatedUser() + if e != nil { + return e + } + if d.CommitterName == "" { + d.CommitterName = user.Name + d.CommitterEmail = user.Email + } + if d.AuthorName == "" { + d.AuthorName = user.Name + d.AuthorEmail = user.Email + } + } + d.pgpEntity, err = loadPrivateKey(d.GPGPrivateKey, d.GPGKeyPassphrase) + if err != nil { + return err + } + } return nil } @@ -174,10 +196,39 @@ func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin if parent.Entries == nil { return errs.NotFolder } - // if parent folder contains .gitkeep only, mark it and delete .gitkeep later - gitKeepSha := "" + subDirSha, err := d.newTree("", []interface{}{ + map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }, + }) + if err != nil { + return err + } + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, TreeObjReq{ + Path: dirName, + Mode: "040000", + Type: "tree", + Sha: subDirSha, + }) if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { - gitKeepSha = parent.Entries[0].Sha + newTree = append(newTree, TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + }) + } + newSha, err := d.newTree(parent.Sha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(parentDir.GetPath(), parent.Sha, newSha, "/") + if err != nil { + return err } commitMessage, err := getMessage(d.mkdirMsgTmpl, &MessageTemplateVars{ @@ -190,13 +241,7 @@ func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin if err != nil { return err } - if err = d.createGitKeep(stdpath.Join(parentDir.GetPath(), dirName), commitMessage); err != nil { - return err - } - if gitKeepSha != "" { - err = d.delete(stdpath.Join(parentDir.GetPath(), ".gitkeep"), gitKeepSha, commitMessage) - } - return err + return d.commit(commitMessage, rootSha) } func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -639,24 +684,6 @@ func (d *Github) get(path string) (*Object, error) { return &resp, err } -func (d *Github) createGitKeep(path, message string) error { - body := map[string]interface{}{ - "message": message, - "content": "", - "branch": d.Ref, - } - d.addCommitterAndAuthor(&body) - - res, err := d.client.R().SetBody(body).Put(d.getContentApiUrl(stdpath.Join(path, ".gitkeep"))) - if err != nil { - return err - } - if res.StatusCode() != 200 && res.StatusCode() != 201 { - return toErr(res) - } - return nil -} - func (d *Github) putBlob(ctx context.Context, s model.FileStreamer, up driver.UpdateProgress) (string, error) { beforeContent := "{\"encoding\":\"base64\",\"content\":\"" afterContent := "\"}" @@ -717,23 +744,6 @@ func (d *Github) putBlob(ctx context.Context, s model.FileStreamer, up driver.Up return resp.Sha, nil } -func (d *Github) delete(path, sha, message string) error { - body := map[string]interface{}{ - "message": message, - "sha": sha, - "branch": d.Ref, - } - d.addCommitterAndAuthor(&body) - res, err := d.client.R().SetBody(body).Delete(d.getContentApiUrl(path)) - if err != nil { - return err - } - if res.StatusCode() != 200 { - return toErr(res) - } - return nil -} - func (d *Github) renewParentTrees(path, prevSha, curSha, until string) (string, error) { for path != until { path = stdpath.Dir(path) @@ -795,11 +805,11 @@ func (d *Github) getTreeDirectly(path string) (*TreeResp, string, error) { } func (d *Github) newTree(baseSha string, tree []interface{}) (string, error) { - res, err := d.client.R(). - SetBody(&TreeReq{ - BaseTree: baseSha, - Trees: tree, - }). + body := &TreeReq{Trees: tree} + if baseSha != "" { + body.BaseTree = baseSha + } + res, err := d.client.R().SetBody(body). Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees", d.Owner, d.Repo)) if err != nil { return "", err @@ -822,6 +832,13 @@ func (d *Github) commit(message, treeSha string) error { "parents": []string{oldCommit}, } d.addCommitterAndAuthor(&body) + if d.pgpEntity != nil { + signature, e := signCommit(&body, d.pgpEntity) + if e != nil { + return e + } + body["signature"] = signature + } res, err := d.client.R().SetBody(body).Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/commits", d.Owner, d.Repo)) if err != nil { return err @@ -925,6 +942,21 @@ func (d *Github) getRepo() (*RepoResp, error) { return &resp, nil } +func (d *Github) getAuthenticatedUser() (*UserResp, error) { + res, err := d.client.R().Get("https://api.github.com/user") + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + resp := &UserResp{} + if err = utils.Json.Unmarshal(res.Body(), resp); err != nil { + return nil, err + } + return resp, nil +} + func (d *Github) addCommitterAndAuthor(m *map[string]interface{}) { if d.CommitterName != "" { committer := map[string]string{ diff --git a/drivers/github/meta.go b/drivers/github/meta.go index 05e704be8e0..7de8d73c391 100644 --- a/drivers/github/meta.go +++ b/drivers/github/meta.go @@ -7,21 +7,23 @@ import ( type Addition struct { driver.RootPath - Token string `json:"token" type:"string"` - Owner string `json:"owner" type:"string" required:"true"` - Repo string `json:"repo" type:"string" required:"true"` - Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` - GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` - CommitterName string `json:"committer_name" type:"string"` - CommitterEmail string `json:"committer_email" type:"string"` - AuthorName string `json:"author_name" type:"string"` - AuthorEmail string `json:"author_email" type:"string"` - MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` - DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` - PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` - RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` - CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` - MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` + Token string `json:"token" type:"string" required:"true"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` + GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` + GPGPrivateKey string `json:"gpg_private_key" type:"text"` + GPGKeyPassphrase string `json:"gpg_key_passphrase" type:"string"` + CommitterName string `json:"committer_name" type:"string"` + CommitterEmail string `json:"committer_email" type:"string"` + AuthorName string `json:"author_name" type:"string"` + AuthorEmail string `json:"author_email" type:"string"` + MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` + DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` + PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` + RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` + CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` + MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` } var config = driver.Config{ diff --git a/drivers/github/types.go b/drivers/github/types.go index 425f89795a7..b057385cdf5 100644 --- a/drivers/github/types.go +++ b/drivers/github/types.go @@ -79,7 +79,7 @@ type TreeResp struct { } type TreeReq struct { - BaseTree string `json:"base_tree"` + BaseTree interface{} `json:"base_tree,omitempty"` Trees []interface{} `json:"tree"` } @@ -100,3 +100,8 @@ type UpdateRefReq struct { type RepoResp struct { DefaultBranch string `json:"default_branch"` } + +type UserResp struct { + Name string `json:"name"` + Email string `json:"email"` +} diff --git a/drivers/github/util.go b/drivers/github/util.go index 85bc3cb9078..03318784f72 100644 --- a/drivers/github/util.go +++ b/drivers/github/util.go @@ -1,14 +1,20 @@ package github import ( + "bytes" "context" "errors" "fmt" + "io" + "strings" + "text/template" + "time" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - "strings" - "text/template" ) type MessageTemplateVars struct { @@ -97,3 +103,65 @@ func getUsername(ctx context.Context) string { } return user.Username } + +func loadPrivateKey(key, passphrase string) (*openpgp.Entity, error) { + entityList, err := openpgp.ReadArmoredKeyRing(strings.NewReader(key)) + if err != nil { + return nil, err + } + if len(entityList) < 1 { + return nil, fmt.Errorf("no keys found in key ring") + } + entity := entityList[0] + + pass := []byte(passphrase) + if entity.PrivateKey != nil && entity.PrivateKey.Encrypted { + if err = entity.PrivateKey.Decrypt(pass); err != nil { + return nil, fmt.Errorf("password incorrect: %+v", err) + } + } + for _, subKey := range entity.Subkeys { + if subKey.PrivateKey != nil && subKey.PrivateKey.Encrypted { + if err = subKey.PrivateKey.Decrypt(pass); err != nil { + return nil, fmt.Errorf("password incorrect: %+v", err) + } + } + } + return entity, nil +} + +func signCommit(m *map[string]interface{}, entity *openpgp.Entity) (string, error) { + var commit strings.Builder + commit.WriteString(fmt.Sprintf("tree %s\n", (*m)["tree"].(string))) + parents := (*m)["parents"].([]string) + for _, p := range parents { + commit.WriteString(fmt.Sprintf("parent %s\n", p)) + } + now := time.Now() + _, offset := now.Zone() + hour := offset / 3600 + author := (*m)["author"].(map[string]string) + commit.WriteString(fmt.Sprintf("author %s <%s> %d %+03d00\n", author["name"], author["email"], now.Unix(), hour)) + author["date"] = now.Format(time.RFC3339) + committer := (*m)["committer"].(map[string]string) + commit.WriteString(fmt.Sprintf("committer %s <%s> %d %+03d00\n", committer["name"], committer["email"], now.Unix(), hour)) + committer["date"] = now.Format(time.RFC3339) + commit.WriteString(fmt.Sprintf("\n%s", (*m)["message"].(string))) + data := commit.String() + + var sigBuffer bytes.Buffer + err := openpgp.DetachSign(&sigBuffer, entity, strings.NewReader(data), nil) + if err != nil { + return "", fmt.Errorf("signing failed: %v", err) + } + var armoredSig bytes.Buffer + armorWriter, err := armor.Encode(&armoredSig, "PGP SIGNATURE", nil) + if err != nil { + return "", err + } + if _, err = io.Copy(armorWriter, &sigBuffer); err != nil { + return "", err + } + _ = armorWriter.Close() + return armoredSig.String(), nil +} diff --git a/go.mod b/go.mod index 7bf8a4bb846..fad155016a0 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.23.1 require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 + github.com/ProtonMail/go-crypto v1.0.0 github.com/SheltonZhu/115driver v1.0.34 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 @@ -90,6 +91,7 @@ require ( github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/charmbracelet/x/ansi v0.2.3 // indirect github.com/charmbracelet/x/term v0.2.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect diff --git a/go.sum b/go.sum index a51e0c6a28c..4237df784c3 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,8 @@ github.com/KirCute/sftpd-alist v0.0.12 h1:GNVM5QLbQLAfXP4wGUlXFA2IO6fVek0n0IsGnO github.com/KirCute/sftpd-alist v0.0.12/go.mod h1:2wNK7yyW2XfjyJq10OY6xB4COLac64hOwfV6clDJn6s= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= @@ -118,6 +120,7 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= @@ -147,6 +150,9 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -643,6 +649,8 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= @@ -706,8 +714,10 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= @@ -764,6 +774,8 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -779,7 +791,9 @@ golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXct golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= @@ -797,6 +811,7 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= From 646c7bcd21d8c00b75f8e12ecc275fcdf5689dc9 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 1 Mar 2025 18:34:33 +0800 Subject: [PATCH 455/659] fix(archive): use another sign for extraction (#7982) --- internal/sign/archive.go | 41 ++++++++++++++++++++++++++++++++++++++ server/debug.go | 3 ++- server/handles/archive.go | 2 +- server/middlewares/down.go | 41 +++++++++++++++++++------------------- server/router.go | 23 +++++++++++---------- 5 files changed, 78 insertions(+), 32 deletions(-) create mode 100644 internal/sign/archive.go diff --git a/internal/sign/archive.go b/internal/sign/archive.go new file mode 100644 index 00000000000..26a2c208d56 --- /dev/null +++ b/internal/sign/archive.go @@ -0,0 +1,41 @@ +package sign + +import ( + "sync" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/sign" +) + +var onceArchive sync.Once +var instanceArchive sign.Sign + +func SignArchive(data string) string { + expire := setting.GetInt(conf.LinkExpiration, 0) + if expire == 0 { + return NotExpiredArchive(data) + } else { + return WithDurationArchive(data, time.Duration(expire)*time.Hour) + } +} + +func WithDurationArchive(data string, d time.Duration) string { + onceArchive.Do(InstanceArchive) + return instanceArchive.Sign(data, time.Now().Add(d).Unix()) +} + +func NotExpiredArchive(data string) string { + onceArchive.Do(InstanceArchive) + return instanceArchive.Sign(data, 0) +} + +func VerifyArchive(data string, sign string) error { + onceArchive.Do(InstanceArchive) + return instanceArchive.Verify(data, sign) +} + +func InstanceArchive() { + instanceArchive = sign.NewHMACSign([]byte(setting.GetStr(conf.Token) + "-archive")) +} diff --git a/server/debug.go b/server/debug.go index 081ef8c3381..a4242abdb2b 100644 --- a/server/debug.go +++ b/server/debug.go @@ -5,6 +5,7 @@ import ( _ "net/http/pprof" "runtime" + "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/middlewares" "github.com/gin-gonic/gin" @@ -15,7 +16,7 @@ func _pprof(g *gin.RouterGroup) { } func debug(g *gin.RouterGroup) { - g.GET("/path/*path", middlewares.Down, func(ctx *gin.Context) { + g.GET("/path/*path", middlewares.Down(sign.Verify), func(ctx *gin.Context) { rawPath := ctx.MustGet("path").(string) ctx.JSON(200, gin.H{ "path": rawPath, diff --git a/server/handles/archive.go b/server/handles/archive.go index fab3916e3ca..4ec933e17b0 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -120,7 +120,7 @@ func FsArchiveMeta(c *gin.Context) { } s := "" if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { - s = sign.Sign(reqPath) + s = sign.SignArchive(reqPath) } api := "/ae" if ret.DriverProviding { diff --git a/server/middlewares/down.go b/server/middlewares/down.go index 05e9dc856d8..d015672de80 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -9,35 +9,36 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) -func Down(c *gin.Context) { - rawPath := parsePath(c.Param("path")) - c.Set("path", rawPath) - meta, err := op.GetNearestMeta(rawPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } - } - c.Set("meta", meta) - // verify sign - if needSign(meta, rawPath) { - s := c.Query("sign") - err = sign.Verify(rawPath, strings.TrimSuffix(s, "/")) +func Down(verifyFunc func(string, string) error) func(c *gin.Context) { + return func(c *gin.Context) { + rawPath := parsePath(c.Param("path")) + c.Set("path", rawPath) + meta, err := op.GetNearestMeta(rawPath) if err != nil { - common.ErrorResp(c, err, 401) - c.Abort() - return + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("meta", meta) + // verify sign + if needSign(meta, rawPath) { + s := c.Query("sign") + err = verifyFunc(rawPath, strings.TrimSuffix(s, "/")) + if err != nil { + common.ErrorResp(c, err, 401) + c.Abort() + return + } } + c.Next() } - c.Next() } // TODO: implement diff --git a/server/router.go b/server/router.go index 830051d8f51..2dd6ee88601 100644 --- a/server/router.go +++ b/server/router.go @@ -4,6 +4,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/message" + "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -40,16 +41,18 @@ func Init(e *gin.Engine) { S3(g.Group("/s3")) downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) - g.GET("/d/*path", middlewares.Down, downloadLimiter, handles.Down) - g.GET("/p/*path", middlewares.Down, downloadLimiter, handles.Proxy) - g.HEAD("/d/*path", middlewares.Down, handles.Down) - g.HEAD("/p/*path", middlewares.Down, handles.Proxy) - g.GET("/ad/*path", middlewares.Down, downloadLimiter, handles.ArchiveDown) - g.GET("/ap/*path", middlewares.Down, downloadLimiter, handles.ArchiveProxy) - g.GET("/ae/*path", middlewares.Down, downloadLimiter, handles.ArchiveInternalExtract) - g.HEAD("/ad/*path", middlewares.Down, handles.ArchiveDown) - g.HEAD("/ap/*path", middlewares.Down, handles.ArchiveProxy) - g.HEAD("/ae/*path", middlewares.Down, handles.ArchiveInternalExtract) + signCheck := middlewares.Down(sign.Verify) + g.GET("/d/*path", signCheck, downloadLimiter, handles.Down) + g.GET("/p/*path", signCheck, downloadLimiter, handles.Proxy) + g.HEAD("/d/*path", signCheck, handles.Down) + g.HEAD("/p/*path", signCheck, handles.Proxy) + archiveSignCheck := middlewares.Down(sign.VerifyArchive) + g.GET("/ad/*path", archiveSignCheck, downloadLimiter, handles.ArchiveDown) + g.GET("/ap/*path", archiveSignCheck, downloadLimiter, handles.ArchiveProxy) + g.GET("/ae/*path", archiveSignCheck, downloadLimiter, handles.ArchiveInternalExtract) + g.HEAD("/ad/*path", archiveSignCheck, handles.ArchiveDown) + g.HEAD("/ap/*path", archiveSignCheck, handles.ArchiveProxy) + g.HEAD("/ae/*path", archiveSignCheck, handles.ArchiveInternalExtract) api := g.Group("/api") auth := api.Group("", middlewares.Auth) From 4145734c1883f8c2b29618ababc69f4527419cd0 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sat, 1 Mar 2025 18:35:34 +0800 Subject: [PATCH 456/659] refactor(net): pass request header (#8031 close #8008) * refactor(net): pass request header * feat(proxy): add `Etag` to response header * refactor --- drivers/alias/util.go | 1 + drivers/crypt/driver.go | 8 +------- drivers/quark_uc/util.go | 2 +- drivers/quqi/util.go | 4 ---- internal/net/request.go | 3 +++ internal/net/serve.go | 2 +- internal/net/util.go | 15 +++++++++++++-- internal/stream/stream.go | 4 ++-- internal/stream/util.go | 14 +++++++++++--- pkg/utils/io.go | 6 ------ server/common/proxy.go | 24 +++++++++++++++++++----- server/s3/backend.go | 6 +----- server/webdav/prop.go | 4 ++-- server/webdav/webdav.go | 7 +------ 14 files changed, 56 insertions(+), 44 deletions(-) diff --git a/drivers/alias/util.go b/drivers/alias/util.go index ee17b622e13..2157a43d7eb 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -63,6 +63,7 @@ func (d *Alias) get(ctx context.Context, path string, dst, sub string) (model.Ob Size: obj.GetSize(), Modified: obj.ModTime(), IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), }, nil } diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index e6f253d187a..59b25806ecf 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -263,12 +263,7 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } rrc := remoteLink.RangeReadCloser if len(remoteLink.URL) > 0 { - - rangedRemoteLink := &model.Link{ - URL: remoteLink.URL, - Header: remoteLink.Header, - } - var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, rangedRemoteLink) + var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, remoteLink) if err != nil { return nil, err } @@ -304,7 +299,6 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: remoteClosers} resultLink := &model.Link{ - Header: remoteLink.Header, RangeReadCloser: resultRangeReadCloser, Expiration: remoteLink.Expiration, } diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index 9a3bdc1c00a..c5845cc6823 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -170,7 +170,7 @@ x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit if res.StatusCode() != 200 { return "", fmt.Errorf("up status: %d, error: %s", res.StatusCode(), res.String()) } - return res.Header().Get("ETag"), nil + return res.Header().Get("Etag"), nil } func (d *QuarkOrUC) upCommit(pre UpPreResp, md5s []string) error { diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go index c57e641bf1e..5ad43c4b97b 100644 --- a/drivers/quqi/util.go +++ b/drivers/quqi/util.go @@ -304,10 +304,6 @@ func (d *Quqi) linkFromCDN(id string) (*model.Link, error) { } return &model.Link{ - Header: http.Header{ - "Origin": []string{"https://quqi.com"}, - "Cookie": []string{d.Cookie}, - }, RangeReadCloser: &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: remoteClosers}, Expiration: &expiration, }, nil diff --git a/internal/net/request.go b/internal/net/request.go index d2f3028fac9..c9ef363f2e0 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -382,6 +382,9 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int if resp == nil { return 0, err } + if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { + return 0, err + } if ch.id == 0 { //第1个任务 有限的重试，超过重试就会结束请求 switch resp.StatusCode { default: diff --git a/internal/net/serve.go b/internal/net/serve.go index c75e611f8d5..8b6b3d1d234 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -114,7 +114,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time // 使用请求的Context // 不然从sendContent读不到数据，即使请求断开CopyBuffer也会一直堵塞 - ctx := r.Context() + ctx := context.WithValue(r.Context(), "request_header", &r.Header) switch { case len(ranges) == 0: reader, err := RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) diff --git a/internal/net/util.go b/internal/net/util.go index 45301dde9f2..5b335a7f718 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -71,6 +71,7 @@ func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { if im == "" { return condNone } + r.Header.Del("If-Match") for { im = textproto.TrimString(im) if len(im) == 0 { @@ -98,7 +99,11 @@ func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { ius := r.Header.Get("If-Unmodified-Since") - if ius == "" || isZeroTime(modtime) { + if ius == "" { + return condNone + } + r.Header.Del("If-Unmodified-Since") + if isZeroTime(modtime) { return condNone } t, err := http.ParseTime(ius) @@ -120,6 +125,7 @@ func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult { if inm == "" { return condNone } + r.Header.Del("If-None-Match") buf := inm for { buf = textproto.TrimString(buf) @@ -150,7 +156,11 @@ func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { return condNone } ims := r.Header.Get("If-Modified-Since") - if ims == "" || isZeroTime(modtime) { + if ims == "" { + return condNone + } + r.Header.Del("If-Modified-Since") + if isZeroTime(modtime) { return condNone } t, err := http.ParseTime(ims) @@ -174,6 +184,7 @@ func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) con if ir == "" { return condNone } + r.Header.Del("If-Range") etag, _ := scanETag(ir) if etag != "" { if etagStrongMatch(etag, w.Header().Get("Etag")) { diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 5eb6bdc7a4c..1c94715f95a 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -384,7 +384,7 @@ func (c *headCache) read(p []byte) (n int, err error) { n, err = lr.Read(buf[off:]) off += n c.cur += int64(n) - if err == io.EOF && n == int(bufL) { + if err == io.EOF && off == int(bufL) { err = nil } if err != nil { @@ -468,7 +468,7 @@ func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (*readerCur, error) } } if rc != nil && off-rc.cur <= utils.MB { - n, err := utils.CopyWithBufferN(utils.NullWriter{}, rc.reader, off-rc.cur) + n, err := utils.CopyWithBufferN(io.Discard, rc.reader, off-rc.cur) rc.cur += n if err == io.EOF && rc.cur == off { err = nil diff --git a/internal/stream/util.go b/internal/stream/util.go index bb5019e0df2..b2c76754040 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -3,13 +3,13 @@ package stream import ( "context" "fmt" - "github.com/alist-org/alist/v3/pkg/utils" "io" "net/http" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" log "github.com/sirupsen/logrus" ) @@ -19,7 +19,11 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl } rangeReaderFunc := func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { if link.Concurrency != 0 || link.PartSize != 0 { - header := net.ProcessHeader(http.Header{}, link.Header) + requestHeader := ctx.Value("request_header") + if requestHeader == nil { + requestHeader = &http.Header{} + } + header := net.ProcessHeader(*(requestHeader.(*http.Header)), link.Header) down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize @@ -60,7 +64,11 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl } func RequestRangedHttp(ctx context.Context, link *model.Link, offset, length int64) (*http.Response, error) { - header := net.ProcessHeader(http.Header{}, link.Header) + requestHeader := ctx.Value("request_header") + if requestHeader == nil { + requestHeader = &http.Header{} + } + header := net.ProcessHeader(*(requestHeader.(*http.Header)), link.Header) header = http_range.ApplyRangeToHttpHeader(http_range.Range{Start: offset, Length: length}, header) return net.RequestHttp(ctx, "GET", header, link.URL) diff --git a/pkg/utils/io.go b/pkg/utils/io.go index c314307d51d..e06fb235b8b 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -233,9 +233,3 @@ func CopyWithBufferN(dst io.Writer, src io.Reader, n int64) (written int64, err } return } - -type NullWriter struct{} - -func (NullWriter) Write(p []byte) (n int, err error) { - return len(p), nil -} diff --git a/server/common/proxy.go b/server/common/proxy.go index 66854976b08..8519ed53c44 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -19,7 +19,7 @@ import ( func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { if link.MFile != nil { defer link.MFile.Close() - attachFileName(w, file) + attachHeader(w, file) contentType := link.Header.Get("Content-Type") if contentType != "" { w.Header().Set("Content-Type", contentType) @@ -35,17 +35,21 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. http.ServeContent(w, r, file.GetName(), file.ModTime(), mFile) return nil } else if link.RangeReadCloser != nil { - attachFileName(w, file) + attachHeader(w, file) net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ RangeReadCloserIF: link.RangeReadCloser, Limiter: stream.ServerDownloadLimit, }) return nil } else if link.Concurrency != 0 || link.PartSize != 0 { - attachFileName(w, file) + attachHeader(w, file) size := file.GetSize() - header := net.ProcessHeader(r.Header, link.Header) rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + requestHeader := ctx.Value("request_header") + if requestHeader == nil { + requestHeader = &http.Header{} + } + header := net.ProcessHeader(*(requestHeader.(*http.Header)), link.Header) down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize @@ -91,10 +95,20 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. return nil } } -func attachFileName(w http.ResponseWriter, file model.Obj) { +func attachHeader(w http.ResponseWriter, file model.Obj) { fileName := file.GetName() w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, fileName, url.PathEscape(fileName))) w.Header().Set("Content-Type", utils.GetMimeType(fileName)) + w.Header().Set("Etag", GetEtag(file)) +} +func GetEtag(file model.Obj) string { + for _, v := range file.GetHash().Export() { + if len(v) != 0 { + return fmt.Sprintf(`"%s"`, v) + } + } + // 参考nginx + return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), file.GetSize()) } var NoProxyRange = &model.RangeReadCloser{} diff --git a/server/s3/backend.go b/server/s3/backend.go index bca45008cde..a1e990441be 100644 --- a/server/s3/backend.go +++ b/server/s3/backend.go @@ -195,11 +195,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string } rrc := link.RangeReadCloser if len(link.URL) > 0 { - rangedRemoteLink := &model.Link{ - URL: link.URL, - Header: link.Header, - } - var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, rangedRemoteLink) + var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, link) if err != nil { return nil, err } diff --git a/server/webdav/prop.go b/server/webdav/prop.go index b1474ea3e95..5e053af4b1a 100644 --- a/server/webdav/prop.go +++ b/server/webdav/prop.go @@ -9,7 +9,6 @@ import ( "context" "encoding/xml" "errors" - "fmt" "mime" "net/http" "path" @@ -18,6 +17,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/server/common" ) // Proppatch describes a property update instruction as defined in RFC 4918. @@ -473,7 +473,7 @@ func findETag(ctx context.Context, ls LockSystem, name string, fi model.Obj) (st // The Apache http 2.4 web server by default concatenates the // modification time and size of a file. We replicate the heuristic // with nanosecond granularity. - return fmt.Sprintf(`"%x%x"`, fi.ModTime().UnixNano(), fi.GetSize()), nil + return common.GetEtag(fi), nil } func findSupportedLock(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 6585056bbde..1b7ec6ff72b 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -227,11 +227,6 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta if err != nil { return http.StatusNotFound, err } - etag, err := findETag(ctx, h.LockSystem, reqPath, fi) - if err != nil { - return http.StatusInternalServerError, err - } - w.Header().Set("ETag", etag) if r.Method == http.MethodHead { w.Header().Set("Content-Length", fmt.Sprintf("%d", fi.GetSize())) return http.StatusOK, nil @@ -361,7 +356,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if err != nil { return http.StatusInternalServerError, err } - w.Header().Set("ETag", etag) + w.Header().Set("Etag", etag) return http.StatusCreated, nil } From 2570707a0619bbc377df0b0cd94233d3ea788f5e Mon Sep 17 00:00:00 2001 From: Ljcbaby <46277145+ljcbaby@users.noreply.github.com> Date: Sat, 1 Mar 2025 18:46:05 +0800 Subject: [PATCH 457/659] feat(baidu_netdisk): support dynamical slice size for low bandwith upload case (#7965) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 动态分片尺寸 * 补充严格测试结果 --- drivers/baidu_netdisk/driver.go | 4 +- drivers/baidu_netdisk/meta.go | 21 +++++----- drivers/baidu_netdisk/util.go | 68 ++++++++++++++++++++++++++++----- 3 files changed, 72 insertions(+), 21 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index e0ba98fa9b5..a07ef742bb1 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -189,7 +189,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } streamSize := stream.GetSize() - sliceSize := d.getSliceSize() + sliceSize := d.getSliceSize(streamSize) count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1)) lastBlockSize := streamSize % sliceSize if streamSize > 0 && lastBlockSize == 0 { @@ -197,7 +197,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } //cal md5 for first 256k data - const SliceSize int64 = 256 * 1024 + const SliceSize int64 = 256 * utils.KB // cal md5 blockList := make([]string, 0, count) byteSize := sliceSize diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index bf2aed5a2b5..e9226a0d37a 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -8,16 +8,17 @@ import ( type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` driver.RootPath - OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` - ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` - ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` - CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` - AccessToken string - UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` - UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` - CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` + ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` + ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` + CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` + AccessToken string + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` + LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` } var config = driver.Config{ diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index ca1a6805a04..a4fc13f8514 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -136,7 +136,7 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { return res, nil } -func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model.Link, error) { +func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Link, error) { var resp DownloadResp params := map[string]string{ "method": "filemetas", @@ -164,7 +164,7 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model }, nil } -func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Link, error) { +func (d *BaiduNetdisk) linkCrack(file model.Obj, _ model.LinkArgs) (*model.Link, error) { var resp DownloadResp2 param := map[string]string{ "target": fmt.Sprintf("[\"%s\"]", file.GetPath()), @@ -230,22 +230,72 @@ func joinTime(form map[string]string, ctime, mtime int64) { const ( DefaultSliceSize int64 = 4 * utils.MB - VipSliceSize = 16 * utils.MB - SVipSliceSize = 32 * utils.MB + VipSliceSize int64 = 16 * utils.MB + SVipSliceSize int64 = 32 * utils.MB + + MaxSliceNum = 2048 // 文档写的是 1024/没写 ，但实际测试是 2048 + SliceStep int64 = 1 * utils.MB ) -func (d *BaiduNetdisk) getSliceSize() int64 { +func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { + // 非会员固定为 4MB + if d.vipType == 0 { + if d.CustomUploadPartSize != 0 { + log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") + } + if filesize > MaxSliceNum*DefaultSliceSize { + log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + } + + return DefaultSliceSize + } + if d.CustomUploadPartSize != 0 { + if d.CustomUploadPartSize < DefaultSliceSize { + log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) + return DefaultSliceSize + } + + if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize { + log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) + return VipSliceSize + } + + if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize { + log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) + return SVipSliceSize + } + return d.CustomUploadPartSize } + + maxSliceSize := DefaultSliceSize + switch d.vipType { case 1: - return VipSliceSize + maxSliceSize = VipSliceSize case 2: - return SVipSliceSize - default: - return DefaultSliceSize + maxSliceSize = SVipSliceSize + } + + // upload on low bandwidth + if d.LowBandwithUploadMode { + size := DefaultSliceSize + + for size <= maxSliceSize { + if filesize <= MaxSliceNum*size { + return size + } + + size += SliceStep + } } + + if filesize > MaxSliceNum*maxSliceSize { + log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + } + + return maxSliceSize } // func encodeURIComponent(str string) string { From 370a6c15a96503d5a3688a723e9c7a6735f6b2b0 Mon Sep 17 00:00:00 2001 From: Ljcbaby <46277145+ljcbaby@users.noreply.github.com> Date: Sat, 1 Mar 2025 19:00:36 +0800 Subject: [PATCH 458/659] fix(baidu_netdisk): remove duplicate retry (#7972) --- drivers/baidu_netdisk/driver.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index a07ef742bb1..264f3b0225f 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -20,7 +20,6 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/errgroup" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) @@ -261,10 +260,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } } // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(3), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread) sem := semaphore.NewWeighted(3) for i, partseq := range precreateResp.BlockList { if utils.IsCanceled(upCtx) { From 5dfea714d816e2da04782843f3ab4ee00a53e755 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sat, 15 Mar 2025 00:12:15 +0800 Subject: [PATCH 459/659] fix(cloudreve): use milliseconds timestamp in last_modified (#8133) --- drivers/cloudreve/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 73fc3fea2af..33ef7ddcc0a 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -149,7 +149,7 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File "size": stream.GetSize(), "name": stream.GetName(), "policy_id": r.Policy.Id, - "last_modified": stream.ModTime().Unix(), + "last_modified": stream.ModTime().UnixMilli(), } // 获取上传会话信息 From 7579d44517de5153bf612576be6b9d93dddbac8b Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sat, 15 Mar 2025 00:12:37 +0800 Subject: [PATCH 460/659] fix(onedrive): set req.ContentLength (#8081) * fix(onedrive): set req.ContentLength * fix(onedrive_app): set req.ContentLength * fix(cloudreve): set req.ContentLength --- drivers/cloudreve/util.go | 6 ++++-- drivers/onedrive/util.go | 4 ++-- drivers/onedrive_app/util.go | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 8a90a42fee7..f41b6b84d96 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -208,7 +208,8 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U return err } req = req.WithContext(ctx) - req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Authorization", fmt.Sprint(credential)) finish += byteSize res, err := base.HttpClient.Do(req) @@ -247,7 +248,8 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u return err } req = req.WithContext(ctx) - req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) finish += byteSize res, err := base.HttpClient.Do(req) diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 9350a681cbd..554349679d0 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -8,7 +8,6 @@ import ( "io" "net/http" stdpath "path" - "strconv" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -226,7 +225,8 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil return err } req = req.WithContext(ctx) - req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) finish += byteSize res, err := base.HttpClient.Do(req) diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index a6793520269..1b01324e09a 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -8,7 +8,6 @@ import ( "io" "net/http" stdpath "path" - "strconv" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -176,7 +175,8 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. return err } req = req.WithContext(ctx) - req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) finish += byteSize res, err := base.HttpClient.Do(req) From 0126af4de07d9711e9c2193e7b88583db78cbf46 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:13:30 +0800 Subject: [PATCH 461/659] fix(crypt): premature close of MFile (#8132 close #8119) * fix(crypt): premature close of MFile * refactor --- drivers/crypt/driver.go | 5 +++-- server/common/proxy.go | 9 +++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index 59b25806ecf..2330fb9782b 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -282,8 +282,9 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if err != nil { return nil, err } - // 可以直接返回，读取完也不会调用Close，直到连接断开Close - return remoteLink.MFile, nil + //keep reuse same MFile and close at last. + remoteClosers.Add(remoteLink.MFile) + return io.NopCloser(remoteLink.MFile), nil } return nil, errs.NotSupport diff --git a/server/common/proxy.go b/server/common/proxy.go index 8519ed53c44..23360a346a4 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" "os" + "strings" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" @@ -102,11 +103,15 @@ func attachHeader(w http.ResponseWriter, file model.Obj) { w.Header().Set("Etag", GetEtag(file)) } func GetEtag(file model.Obj) string { + hash := "" for _, v := range file.GetHash().Export() { - if len(v) != 0 { - return fmt.Sprintf(`"%s"`, v) + if strings.Compare(v, hash) > 0 { + hash = v } } + if len(hash) > 0 { + return fmt.Sprintf(`"%s"`, hash) + } // 参考nginx return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), file.GetSize()) } From 28b61a93fdd6b71553bf32b40107822a72b22d91 Mon Sep 17 00:00:00 2001 From: shniubobo Date: Fri, 14 Mar 2025 16:21:07 +0000 Subject: [PATCH 462/659] feat(webdav): support `oc:checksums` (#8064 close #7472) Ref: #7472 --- pkg/utils/hash.go | 11 +++++++++++ server/webdav/prop.go | 15 ++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index fa06bcc24c2..a281dd4e6e5 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -10,6 +10,7 @@ import ( "errors" "hash" "io" + "iter" "github.com/alist-org/alist/v3/internal/errs" log "github.com/sirupsen/logrus" @@ -226,3 +227,13 @@ func (hi HashInfo) GetHash(ht *HashType) string { func (hi HashInfo) Export() map[*HashType]string { return hi.h } + +func (hi HashInfo) All() iter.Seq2[*HashType, string] { + return func(yield func(*HashType, string) bool) { + for hashType, hashValue := range hi.h { + if !yield(hashType, hashValue) { + return + } + } + } +} diff --git a/server/webdav/prop.go b/server/webdav/prop.go index 5e053af4b1a..a81f31b05c7 100644 --- a/server/webdav/prop.go +++ b/server/webdav/prop.go @@ -9,6 +9,7 @@ import ( "context" "encoding/xml" "errors" + "fmt" "mime" "net/http" "path" @@ -101,7 +102,7 @@ type DeadPropsHolder interface { Patch([]Proppatch) ([]Propstat, error) } -// liveProps contains all supported, protected DAV: properties. +// liveProps contains all supported properties. var liveProps = map[xml.Name]struct { // findFn implements the propfind function of this property. If nil, // it indicates a hidden property. @@ -160,6 +161,10 @@ var liveProps = map[xml.Name]struct { findFn: findSupportedLock, dir: true, }, + {Space: "http://owncloud.org/ns", Local: "checksums"}: { + findFn: findChecksums, + dir: false, + }, } // TODO(nigeltao) merge props and allprop? @@ -483,3 +488,11 @@ func findSupportedLock(ctx context.Context, ls LockSystem, name string, fi model `` + ``, nil } + +func findChecksums(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { + checksums := "" + for hashType, hashValue := range fi.GetHash().All() { + checksums += fmt.Sprintf("%s:%s", hashType.Name, hashValue) + } + return checksums, nil +} From 04f5525f207e3f2324df41c7bb17f4d17df4fa5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=8A=98=E7=BA=B8=E9=A3=9E=E6=9C=BA?= Date: Fri, 14 Mar 2025 17:21:24 +0100 Subject: [PATCH 463/659] fix(s3): incorrectly added slash before the Bucket name (#8083 close #8001) --- drivers/s3/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s3/util.go b/drivers/s3/util.go index 99f271aa071..e02945a07d2 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -199,7 +199,7 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { dstKey := getKey(dst, false) input := &s3.CopyObjectInput{ Bucket: &d.Bucket, - CopySource: aws.String(url.PathEscape("/" + d.Bucket + "/" + srcKey)), + CopySource: aws.String(url.PathEscape(d.Bucket + "/" + srcKey)), Key: &dstKey, } _, err := d.client.CopyObject(input) From c82e632ee16c150a2844665ddd89defa405d0b29 Mon Sep 17 00:00:00 2001 From: hshpy Date: Sat, 15 Mar 2025 23:28:40 +0800 Subject: [PATCH 464/659] fix: potential XSS vulnerabilities (#7923) * fix: potential XSS vulnerabilities * feat: support filter and render for readme.md * chore: set ReadMeAutoRender to true * fix attachFileName undefined --------- Co-authored-by: Andy Hsu --- go.mod | 4 ++ go.sum | 8 ++++ internal/bootstrap/data/setting.go | 5 ++- internal/conf/const.go | 3 +- server/common/proxy.go | 64 ++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index fad155016a0..f07db3fe321 100644 --- a/go.mod +++ b/go.mod @@ -83,6 +83,7 @@ require ( require ( github.com/STARRY-S/zip v0.2.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/blevesearch/go-faiss v1.0.20 // indirect github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/bodgit/plumbing v1.3.0 // indirect @@ -97,6 +98,7 @@ require ( github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/fclairamb/go-log v0.5.0 // indirect + github.com/gorilla/css v1.0.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect @@ -105,11 +107,13 @@ require ( github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/text v0.2.0 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect + github.com/microcosm-cc/bluemonday v1.0.27 github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect + github.com/yuin/goldmark v1.7.8 go4.org v0.0.0-20230225012048-214862532bf5 // indirect ) diff --git a/go.sum b/go.sum index 4237df784c3..e6a8574b57d 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiE github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -303,6 +305,8 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -424,6 +428,8 @@ github.com/meilisearch/meilisearch-go v0.27.2 h1:3G21dJ5i208shnLPDsIEZ0L0Geg/5oe github.com/meilisearch/meilisearch-go v0.27.2/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= @@ -613,6 +619,8 @@ github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9/go.mod h1:9BnoKCcgJ/+SLhf github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= +github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 h1:X+lHsNTlbatQ1cErXIbtyrh+3MTWxqQFS+sBP/wpFXo= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 026a89e17ad..407a5c64e17 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -1,6 +1,8 @@ package data import ( + "strconv" + "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" @@ -11,7 +13,6 @@ import ( "github.com/alist-org/alist/v3/pkg/utils/random" "github.com/pkg/errors" "gorm.io/gorm" - "strconv" ) var initialSettingItems []model.SettingItem @@ -141,6 +142,8 @@ func InitialSettings() []model.SettingItem { {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 2234e9bc5c5..5cb8d850bf0 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -31,7 +31,8 @@ const ( AudioAutoplay = "audio_autoplay" VideoAutoplay = "video_autoplay" PreviewArchivesByDefault = "preview_archives_by_default" - + ReadMeAutoRender = "readme_autorender" + FilterReadMeScripts = "filter_readme_scripts" // global HideFiles = "hide_files" CustomizeHead = "customize_head" diff --git a/server/common/proxy.go b/server/common/proxy.go index 23360a346a4..1d61e5fa3b8 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -1,23 +1,87 @@ package common import ( + "bytes" "context" "fmt" "io" "net/http" "net/url" "os" + "strconv" "strings" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" + "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" + "github.com/yuin/goldmark" ) +func processMarkdown(content []byte) ([]byte, error) { + var buf bytes.Buffer + if err := goldmark.New().Convert(content, &buf); err != nil { + return nil, fmt.Errorf("markdown conversion failed: %w", err) + } + return bluemonday.UGCPolicy().SanitizeBytes(buf.Bytes()), nil +} + func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { + + //优先处理md文件 + if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) { + var markdownContent []byte + var err error + + if link.MFile != nil { + defer link.MFile.Close() + attachHeader(w, file) + markdownContent, err = io.ReadAll(link.MFile) + if err != nil { + return fmt.Errorf("failed to read markdown content: %w", err) + } + + } else { + header := net.ProcessHeader(r.Header, link.Header) + res, err := net.RequestHttp(r.Context(), r.Method, header, link.URL) + if err != nil { + return err + } + defer res.Body.Close() + for h, v := range res.Header { + w.Header()[h] = v + } + w.WriteHeader(res.StatusCode) + if r.Method == http.MethodHead { + return nil + } + markdownContent, err = io.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("failed to read markdown content: %w", err) + } + + } + + safeHTML, err := processMarkdown(markdownContent) + if err != nil { + return err + } + + safeHTMLReader := bytes.NewReader(safeHTML) + w.Header().Set("Content-Length", strconv.FormatInt(int64(len(safeHTML)), 10)) + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, err = utils.CopyWithBuffer(w, safeHTMLReader) + if err != nil { + return err + } + return nil + } + if link.MFile != nil { defer link.MFile.Close() attachHeader(w, file) From d16ba65f4224bb9ace40655c458f6f5e5f9fedad Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 16 Mar 2025 16:37:33 +0800 Subject: [PATCH 465/659] fix(lang): initialize configuration in LangCmd before generating language JSON file --- cmd/lang.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/lang.go b/cmd/lang.go index 56ef037ba24..5d8ce837878 100644 --- a/cmd/lang.go +++ b/cmd/lang.go @@ -12,6 +12,7 @@ import ( "strings" _ "github.com/alist-org/alist/v3/drivers" + "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/bootstrap/data" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/op" @@ -137,6 +138,7 @@ var LangCmd = &cobra.Command{ Use: "lang", Short: "Generate language json file", Run: func(cmd *cobra.Command, args []string) { + bootstrap.InitConfig() err := os.MkdirAll("lang", 0777) if err != nil { utils.Log.Fatalf("failed create folder: %s", err.Error()) From d20f41d687827c4faab615d05dea4e7938c5be25 Mon Sep 17 00:00:00 2001 From: hshpy Date: Sun, 16 Mar 2025 22:14:44 +0800 Subject: [PATCH 466/659] fix: missing handling of RangeReadCloser (#8146) --- server/common/proxy.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/server/common/proxy.go b/server/common/proxy.go index 1d61e5fa3b8..00fee4b2153 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -45,7 +45,17 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. if err != nil { return fmt.Errorf("failed to read markdown content: %w", err) } - + } else if link.RangeReadCloser != nil { + attachHeader(w, file) + rrc, err := link.RangeReadCloser.RangeRead(r.Context(), http_range.Range{Start: 0, Length: -1}) + if err != nil { + return err + } + defer rrc.Close() + markdownContent, err = io.ReadAll(rrc) + if err != nil { + return fmt.Errorf("failed to read markdown content: %w", err) + } } else { header := net.ProcessHeader(r.Header, link.Header) res, err := net.RequestHttp(r.Context(), r.Method, header, link.URL) From 3499c4db8712564c9bdbd8c97d3fb7a7739d42a2 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Mon, 17 Mar 2025 00:52:09 +0800 Subject: [PATCH 467/659] feat: 115 open driver (#8139) * wip: 115 open * chore(go.mod): update 115-sdk-go dependency version * feat(115_open): implement directory management and file operations * chore(go.mod): update 115-sdk-go dependency to v0.1.1 and adjust callback handling in driver * chore: rename driver --- drivers/115_open/driver.go | 308 +++++++++++++++++++++++++++++++++++++ drivers/115_open/meta.go | 36 +++++ drivers/115_open/types.go | 59 +++++++ drivers/115_open/util.go | 3 + drivers/all.go | 1 + go.mod | 24 +-- go.sum | 28 ++-- 7 files changed, 436 insertions(+), 23 deletions(-) create mode 100644 drivers/115_open/driver.go create mode 100644 drivers/115_open/meta.go create mode 100644 drivers/115_open/types.go create mode 100644 drivers/115_open/util.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go new file mode 100644 index 00000000000..67c17608cf3 --- /dev/null +++ b/drivers/115_open/driver.go @@ -0,0 +1,308 @@ +package _115_open + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/cmd/flags" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + sdk "github.com/xhofe/115-sdk-go" +) + +type Open115 struct { + model.Storage + Addition + client *sdk.Client +} + +func (d *Open115) Config() driver.Config { + return config +} + +func (d *Open115) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Open115) Init(ctx context.Context) error { + d.client = sdk.New(sdk.WithRefreshToken(d.Addition.RefreshToken), + sdk.WithAccessToken(d.Addition.AccessToken), + sdk.WithOnRefreshToken(func(s1, s2 string) { + d.Addition.AccessToken = s1 + d.Addition.RefreshToken = s2 + op.MustSaveDriverStorage(d) + })) + if flags.Debug || flags.Dev { + d.client.SetDebug(true) + } + _, err := d.client.UserInfo(ctx) + if err != nil { + return err + } + return nil +} + +func (d *Open115) Drop(ctx context.Context) error { + return nil +} + +func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var res []model.Obj + pageSize := int64(200) + offset := int64(0) + for { + resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ + CID: dir.GetID(), + Limit: pageSize, + Offset: offset, + ASC: d.Addition.OrderDirection == "asc", + O: d.Addition.OrderBy, + // Cur: 1, + ShowDir: true, + }) + if err != nil { + return nil, err + } + res = append(res, utils.MustSliceConvert(resp.Data, func(src sdk.GetFilesResp_File) model.Obj { + obj := Obj(src) + return &obj + })...) + if len(res) >= int(resp.Count) { + break + } + offset += pageSize + } + return res, nil +} + +func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var ua string + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + obj, ok := file.(*Obj) + if !ok { + return nil, fmt.Errorf("can't convert obj") + } + pc := obj.Pc + resp, err := d.client.DownURL(ctx, pc, ua) + if err != nil { + return nil, err + } + u, ok := resp[obj.GetID()] + if !ok { + return nil, fmt.Errorf("can't get link") + } + return &model.Link{ + URL: u.URL.URL, + Header: http.Header{ + "User-Agent": []string{ua}, + }, + }, nil +} + +func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + resp, err := d.client.Mkdir(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + return &Obj{ + Fid: resp.FileID, + Pid: parentDir.GetID(), + Fn: dirName, + Fc: "0", + Upt: time.Now().Unix(), + Uet: time.Now().Unix(), + UpPt: time.Now().Unix(), + }, nil +} + +func (d *Open115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + _, err := d.client.Move(ctx, &sdk.MoveReq{ + FileIDs: srcObj.GetID(), + ToCid: dstDir.GetID(), + }) + if err != nil { + return nil, err + } + return srcObj, nil +} + +func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ + FileID: srcObj.GetID(), + FileNma: newName, + }) + if err != nil { + return nil, err + } + return srcObj, nil +} + +func (d *Open115) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + _, err := d.client.Copy(ctx, &sdk.CopyReq{ + PID: dstDir.GetID(), + FileID: srcObj.GetID(), + NoDupli: "1", + }) + if err != nil { + return nil, err + } + return srcObj, nil +} + +func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { + _obj, ok := obj.(*Obj) + if !ok { + return fmt.Errorf("can't convert obj") + } + _, err := d.client.DelFile(ctx, &sdk.DelFileReq{ + FileIDs: _obj.GetID(), + ParentID: _obj.Pid, + }) + if err != nil { + return err + } + return nil +} + +func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + tempF, err := file.CacheFullInTempFile() + if err != nil { + return err + } + // cal full sha1 + sha1, err := utils.HashReader(utils.SHA1, tempF) + if err != nil { + return err + } + _, err = tempF.Seek(0, io.SeekStart) + if err != nil { + return err + } + // pre 128k sha1 + sha1128k, err := utils.HashReader(utils.SHA1, io.LimitReader(tempF, 128*1024)) + if err != nil { + return err + } + _, err = tempF.Seek(0, io.SeekStart) + if err != nil { + return err + } + // 1. Init + resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + }) + if err != nil { + return err + } + if resp.Status == 2 { + return nil + } + // 2. two way verify + if utils.SliceContains([]int{6, 7, 8}, resp.Status) { + signCheck := strings.Split(resp.SignCheck, "-") //"sign_check": "2392148-2392298" 取2392148-2392298之间的内容(包含2392148、2392298)的sha1 + start, err := strconv.ParseInt(signCheck[0], 10, 64) + if err != nil { + return err + } + end, err := strconv.ParseInt(signCheck[1], 10, 64) + if err != nil { + return err + } + _, err = tempF.Seek(start, io.SeekStart) + if err != nil { + return err + } + signVal, err := utils.HashReader(utils.SHA1, io.LimitReader(tempF, end-start+1)) + if err != nil { + return err + } + _, err = tempF.Seek(0, io.SeekStart) + if err != nil { + return err + } + resp, err = d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + SignKey: resp.SignKey, + SignVal: strings.ToUpper(signVal), + }) + if err != nil { + return err + } + if resp.Status == 2 { + return nil + } + } + // 3. get upload token + tokenResp, err := d.client.UploadGetToken(ctx) + if err != nil { + return err + } + // 4. upload + ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) + if err != nil { + return err + } + bucket, err := ossClient.Bucket(resp.Bucket) + if err != nil { + return err + } + err = bucket.PutObject(resp.Object, tempF, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(resp.Callback.Value.Callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(resp.Callback.Value.CallbackVar))), + ) + if err != nil { + return err + } + return nil +} + +// func (d *Open115) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { +// // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional +// return nil, errs.NotImplement +// } + +// func (d *Open115) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { +// // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional +// return nil, errs.NotImplement +// } + +// func (d *Open115) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { +// // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional +// return nil, errs.NotImplement +// } + +// func (d *Open115) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { +// // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional +// // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir +// // return errs.NotImplement to use an internal archive tool +// return nil, errs.NotImplement +// } + +//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Open115)(nil) diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go new file mode 100644 index 00000000000..7e26e0ddbc2 --- /dev/null +++ b/drivers/115_open/meta.go @@ -0,0 +1,36 @@ +package _115_open + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + RefreshToken string `json:"refresh_token" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + AccessToken string +} + +var config = driver.Config{ + Name: "115 Open", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Open115{} + }) +} diff --git a/drivers/115_open/types.go b/drivers/115_open/types.go new file mode 100644 index 00000000000..491a368edf2 --- /dev/null +++ b/drivers/115_open/types.go @@ -0,0 +1,59 @@ +package _115_open + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + sdk "github.com/xhofe/115-sdk-go" +) + +type Obj sdk.GetFilesResp_File + +// Thumb implements model.Thumb. +func (o *Obj) Thumb() string { + return o.Thumbnail +} + +// CreateTime implements model.Obj. +func (o *Obj) CreateTime() time.Time { + return time.Unix(o.UpPt, 0) +} + +// GetHash implements model.Obj. +func (o *Obj) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.SHA1, o.Sha1) +} + +// GetID implements model.Obj. +func (o *Obj) GetID() string { + return o.Fid +} + +// GetName implements model.Obj. +func (o *Obj) GetName() string { + return o.Fn +} + +// GetPath implements model.Obj. +func (o *Obj) GetPath() string { + return "" +} + +// GetSize implements model.Obj. +func (o *Obj) GetSize() int64 { + return o.FS +} + +// IsDir implements model.Obj. +func (o *Obj) IsDir() bool { + return o.Fc == "0" +} + +// ModTime implements model.Obj. +func (o *Obj) ModTime() time.Time { + return time.Unix(o.Upt, 0) +} + +var _ model.Obj = (*Obj)(nil) +var _ model.Thumb = (*Obj)(nil) diff --git a/drivers/115_open/util.go b/drivers/115_open/util.go new file mode 100644 index 00000000000..ee0216597e4 --- /dev/null +++ b/drivers/115_open/util.go @@ -0,0 +1,3 @@ +package _115_open + +// do others that not defined in Driver interface diff --git a/drivers/all.go b/drivers/all.go index 2746e1bf7cb..963f0c4453c 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -2,6 +2,7 @@ package drivers import ( _ "github.com/alist-org/alist/v3/drivers/115" + _ "github.com/alist-org/alist/v3/drivers/115_open" _ "github.com/alist-org/alist/v3/drivers/115_share" _ "github.com/alist-org/alist/v3/drivers/123" _ "github.com/alist-org/alist/v3/drivers/123_link" diff --git a/go.mod b/go.mod index f07db3fe321..557c16c2ab7 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/alist-org/alist/v3 -go 1.23 - -toolchain go1.23.1 +go 1.23.4 require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 @@ -67,10 +65,10 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.31.0 + golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 - golang.org/x/net v0.28.0 + golang.org/x/net v0.37.0 golang.org/x/oauth2 v0.22.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 @@ -107,14 +105,16 @@ require ( github.com/klauspost/pgzip v1.2.6 // indirect github.com/kr/text v0.2.0 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect - github.com/microcosm-cc/bluemonday v1.0.27 + github.com/microcosm-cc/bluemonday v1.0.27 github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/yuin/goldmark v1.7.8 + github.com/xhofe/115-sdk-go v0.1.1 + github.com/yuin/goldmark v1.7.8 go4.org v0.0.0-20230225012048-214862532bf5 // indirect + resty.dev/v3 v3.0.0-beta.2 // indirect ) require ( @@ -246,10 +246,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.10.0 - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect - golang.org/x/text v0.21.0 + golang.org/x/sync v0.12.0 + golang.org/x/sys v0.31.0 // indirect + golang.org/x/term v0.30.0 // indirect + golang.org/x/text v0.23.0 golang.org/x/tools v0.24.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect @@ -261,3 +261,5 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) + +// replace github.com/xhofe/115-sdk-go => ../../xhofe/115-sdk-go diff --git a/go.sum b/go.sum index e6a8574b57d..1b5f46f2289 100644 --- a/go.sum +++ b/go.sum @@ -606,6 +606,8 @@ github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXo github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xhofe/115-sdk-go v0.1.1 h1:eMQIuCyhWZHQApqdCIt7bTA3S5MYQnANeLJbWYSDv6A= +github.com/xhofe/115-sdk-go v0.1.1/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= @@ -663,8 +665,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -731,8 +733,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -752,8 +754,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -793,8 +795,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -807,8 +809,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -825,8 +827,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -953,6 +955,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= +resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= From b4e6ab12d9b093ab328f45879cae298b89278d93 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Tue, 18 Mar 2025 22:02:33 +0800 Subject: [PATCH 468/659] refactor: FilterReadMeScripts (#8154 close #8150) * refactor: FilterReadMeScripts * . --- drivers/quqi/driver.go | 4 +- drivers/quqi/util.go | 11 ------ drivers/vtencent/util.go | 4 +- internal/net/request.go | 5 +-- pkg/utils/path.go | 2 +- server/common/proxy.go | 80 ++-------------------------------------- server/handles/down.go | 43 ++++++++++++++++++++- 7 files changed, 51 insertions(+), 98 deletions(-) diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go index 0fa64041d26..36758bd1d14 100644 --- a/drivers/quqi/driver.go +++ b/drivers/quqi/driver.go @@ -316,7 +316,7 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea // if the file already exists in Quqi server, there is no need to actually upload it if uploadInitResp.Data.Exist { // the file name returned by Quqi does not include the extension name - nodeName, nodeExt := uploadInitResp.Data.NodeName, rawExt(stream.GetName()) + nodeName, nodeExt := uploadInitResp.Data.NodeName, utils.Ext(stream.GetName()) if nodeExt != "" { nodeName = nodeName + "." + nodeExt } @@ -432,7 +432,7 @@ func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea return nil, err } // the file name returned by Quqi does not include the extension name - nodeName, nodeExt := uploadFinishResp.Data.NodeName, rawExt(stream.GetName()) + nodeName, nodeExt := uploadFinishResp.Data.NodeName, utils.Ext(stream.GetName()) if nodeExt != "" { nodeName = nodeName + "." + nodeExt } diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go index 5ad43c4b97b..aa184d70fc4 100644 --- a/drivers/quqi/util.go +++ b/drivers/quqi/util.go @@ -9,7 +9,6 @@ import ( "io" "net/http" "net/url" - stdpath "path" "strings" "time" @@ -115,16 +114,6 @@ func (d *Quqi) checkLogin() bool { return true } -// rawExt 保留扩展名大小写 -func rawExt(name string) string { - ext := stdpath.Ext(name) - if strings.HasPrefix(ext, ".") { - ext = ext[1:] - } - - return ext -} - // decryptKey 获取密码 func decryptKey(encodeKey string) []byte { // 移除非法字符 diff --git a/drivers/vtencent/util.go b/drivers/vtencent/util.go index 91db54b7ad5..4ba72d1b2e3 100644 --- a/drivers/vtencent/util.go +++ b/drivers/vtencent/util.go @@ -8,9 +8,7 @@ import ( "fmt" "io" "net/http" - "path" "strconv" - "strings" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -151,7 +149,7 @@ func (d *Vtencent) ApplyUploadUGC(signature string, stream model.FileStreamer) ( form := base.Json{ "signature": signature, "videoName": stream.GetName(), - "videoType": strings.ReplaceAll(path.Ext(stream.GetName()), ".", ""), + "videoType": utils.Ext(stream.GetName()), "videoSize": stream.GetSize(), } var resps RspApplyUploadUGC diff --git a/internal/net/request.go b/internal/net/request.go index c9ef363f2e0..d4f9321c585 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -619,10 +619,9 @@ type Buf struct { // NewBuf is a buffer that can have 1 read & 1 write at the same time. // when read is faster write, immediately feed data to read after written func NewBuf(ctx context.Context, maxSize int) *Buf { - d := make([]byte, 0, maxSize) return &Buf{ ctx: ctx, - buffer: bytes.NewBuffer(d), + buffer: bytes.NewBuffer(make([]byte, 0, maxSize)), size: maxSize, } } @@ -677,5 +676,5 @@ func (br *Buf) Write(p []byte) (n int, err error) { } func (br *Buf) Close() { - br.buffer.Reset() + br.buffer = nil } diff --git a/pkg/utils/path.go b/pkg/utils/path.go index c0793a3ec0f..135f8e4ebca 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -45,7 +45,7 @@ func IsSubPath(path string, subPath string) bool { func Ext(path string) string { ext := stdpath.Ext(path) - if strings.HasPrefix(ext, ".") { + if len(ext) > 0 && ext[0] == '.' { ext = ext[1:] } return strings.ToLower(ext) diff --git a/server/common/proxy.go b/server/common/proxy.go index 00fee4b2153..c14af6fad5e 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -1,97 +1,25 @@ package common import ( - "bytes" "context" "fmt" "io" "net/http" "net/url" "os" - "strconv" "strings" - "github.com/alist-org/alist/v3/internal/conf" + "maps" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" - "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" - "github.com/yuin/goldmark" ) -func processMarkdown(content []byte) ([]byte, error) { - var buf bytes.Buffer - if err := goldmark.New().Convert(content, &buf); err != nil { - return nil, fmt.Errorf("markdown conversion failed: %w", err) - } - return bluemonday.UGCPolicy().SanitizeBytes(buf.Bytes()), nil -} - func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { - - //优先处理md文件 - if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) { - var markdownContent []byte - var err error - - if link.MFile != nil { - defer link.MFile.Close() - attachHeader(w, file) - markdownContent, err = io.ReadAll(link.MFile) - if err != nil { - return fmt.Errorf("failed to read markdown content: %w", err) - } - } else if link.RangeReadCloser != nil { - attachHeader(w, file) - rrc, err := link.RangeReadCloser.RangeRead(r.Context(), http_range.Range{Start: 0, Length: -1}) - if err != nil { - return err - } - defer rrc.Close() - markdownContent, err = io.ReadAll(rrc) - if err != nil { - return fmt.Errorf("failed to read markdown content: %w", err) - } - } else { - header := net.ProcessHeader(r.Header, link.Header) - res, err := net.RequestHttp(r.Context(), r.Method, header, link.URL) - if err != nil { - return err - } - defer res.Body.Close() - for h, v := range res.Header { - w.Header()[h] = v - } - w.WriteHeader(res.StatusCode) - if r.Method == http.MethodHead { - return nil - } - markdownContent, err = io.ReadAll(res.Body) - if err != nil { - return fmt.Errorf("failed to read markdown content: %w", err) - } - - } - - safeHTML, err := processMarkdown(markdownContent) - if err != nil { - return err - } - - safeHTMLReader := bytes.NewReader(safeHTML) - w.Header().Set("Content-Length", strconv.FormatInt(int64(len(safeHTML)), 10)) - w.Header().Set("Content-Type", "text/html; charset=utf-8") - _, err = utils.CopyWithBuffer(w, safeHTMLReader) - if err != nil { - return err - } - return nil - } - if link.MFile != nil { defer link.MFile.Close() attachHeader(w, file) @@ -152,9 +80,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. } defer res.Body.Close() - for h, v := range res.Header { - w.Header()[h] = v - } + maps.Copy(w.Header(), res.Header) w.WriteHeader(res.StatusCode) if r.Method == http.MethodHead { return nil diff --git a/server/handles/down.go b/server/handles/down.go index b2f9a21bc63..1153881fdda 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -1,9 +1,12 @@ package handles import ( + "bytes" "fmt" "io" + "net/http" stdpath "path" + "strconv" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -15,7 +18,9 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" + "github.com/yuin/goldmark" ) func Down(c *gin.Context) { @@ -124,7 +129,34 @@ func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange boo if proxyRange { common.ProxyRange(link, file.GetSize()) } - err = common.Proxy(c.Writer, c.Request, link, file) + + //优先处理md文件 + if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) { + w := c.Writer + buf := bytes.NewBuffer(make([]byte, 0, file.GetSize())) + err = common.Proxy(&proxyResponseWriter{ResponseWriter: w, Writer: buf}, c.Request, link, file) + if err == nil && buf.Len() > 0 { + if w.Status() < 200 || w.Status() > 300 { + w.Write(buf.Bytes()) + return + } + + var html bytes.Buffer + if err = goldmark.Convert(buf.Bytes(), &html); err != nil { + err = fmt.Errorf("markdown conversion failed: %w", err) + } else { + buf.Reset() + err = bluemonday.UGCPolicy().SanitizeReaderToWriter(&html, buf) + if err == nil { + w.Header().Set("Content-Length", strconv.FormatInt(int64(buf.Len()), 10)) + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, err = utils.CopyWithBuffer(c.Writer, buf) + } + } + } + } else { + err = common.Proxy(c.Writer, c.Request, link, file) + } if err != nil { common.ErrorResp(c, err, 500, true) return @@ -150,3 +182,12 @@ func canProxy(storage driver.Driver, filename string) bool { } return false } + +type proxyResponseWriter struct { + http.ResponseWriter + io.Writer +} + +func (pw *proxyResponseWriter) Write(p []byte) (int, error) { + return pw.Writer.Write(p) +} From 35d6f3b8fc818a6ad5c9c39088170c37619327b0 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 18 Mar 2025 22:21:50 +0800 Subject: [PATCH 469/659] fix(115_open): upgrade sdk (close #8151) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 557c16c2ab7..b506254c95c 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/xhofe/115-sdk-go v0.1.1 + github.com/xhofe/115-sdk-go v0.1.2 github.com/yuin/goldmark v1.7.8 go4.org v0.0.0-20230225012048-214862532bf5 // indirect resty.dev/v3 v3.0.0-beta.2 // indirect diff --git a/go.sum b/go.sum index 1b5f46f2289..245010d445f 100644 --- a/go.sum +++ b/go.sum @@ -606,8 +606,8 @@ github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXo github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xhofe/115-sdk-go v0.1.1 h1:eMQIuCyhWZHQApqdCIt7bTA3S5MYQnANeLJbWYSDv6A= -github.com/xhofe/115-sdk-go v0.1.1/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= +github.com/xhofe/115-sdk-go v0.1.2 h1:Y58Zg+pz9D5FDCgwdg7T/F+6/t07/F1Ni/5bRa7yJNA= +github.com/xhofe/115-sdk-go v0.1.2/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= From 4563aea47e5039248b80854420f90bfe83da1d2d Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Tue, 18 Mar 2025 22:25:04 +0800 Subject: [PATCH 470/659] fix(115_open): rename delay to take effect (close #8156) --- drivers/115_open/driver.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 67c17608cf3..00337c0b920 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -149,6 +149,10 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) if err != nil { return nil, err } + obj, ok := srcObj.(*Obj) + if ok { + obj.Fn = newName + } return srcObj, nil } From 758554a40fe4f5a3b91b6b89cdad5150ffe44d65 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Wed, 19 Mar 2025 21:47:42 +0800 Subject: [PATCH 471/659] fix(115_open): upgrade 115-sdk-go dependency to v0.1.3 (close #8169) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b506254c95c..bdbb1c8edc9 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/xhofe/115-sdk-go v0.1.2 + github.com/xhofe/115-sdk-go v0.1.3 github.com/yuin/goldmark v1.7.8 go4.org v0.0.0-20230225012048-214862532bf5 // indirect resty.dev/v3 v3.0.0-beta.2 // indirect diff --git a/go.sum b/go.sum index 245010d445f..2fbc48da74b 100644 --- a/go.sum +++ b/go.sum @@ -606,8 +606,8 @@ github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXo github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xhofe/115-sdk-go v0.1.2 h1:Y58Zg+pz9D5FDCgwdg7T/F+6/t07/F1Ni/5bRa7yJNA= -github.com/xhofe/115-sdk-go v0.1.2/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= +github.com/xhofe/115-sdk-go v0.1.3 h1:n/00JkEwNOZUb7+U8BgrftotMbPf8yQKpm1bYc+WBoE= +github.com/xhofe/115-sdk-go v0.1.3/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= From 32890da29f72e380ffd6e64df4a693e5cd4f3baa Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 21 Mar 2025 19:06:09 +0800 Subject: [PATCH 472/659] fix(115_open): upgrade 115-sdk-go dependency to v0.1.4 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bdbb1c8edc9..a06c62ba8d8 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/xhofe/115-sdk-go v0.1.3 + github.com/xhofe/115-sdk-go v0.1.4 github.com/yuin/goldmark v1.7.8 go4.org v0.0.0-20230225012048-214862532bf5 // indirect resty.dev/v3 v3.0.0-beta.2 // indirect diff --git a/go.sum b/go.sum index 2fbc48da74b..bf98a8cd784 100644 --- a/go.sum +++ b/go.sum @@ -606,8 +606,8 @@ github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXo github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xhofe/115-sdk-go v0.1.3 h1:n/00JkEwNOZUb7+U8BgrftotMbPf8yQKpm1bYc+WBoE= -github.com/xhofe/115-sdk-go v0.1.3/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= +github.com/xhofe/115-sdk-go v0.1.4 h1:erIWuWH+kZQOEHM+YZK8Y6sWQ2s/SFJIFh/WeCtjiiY= +github.com/xhofe/115-sdk-go v0.1.4/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= From 6e13923225afdc6c95b09996636919b67c96d62b Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Thu, 27 Mar 2025 23:14:36 +0800 Subject: [PATCH 473/659] fix(sftp-server): postgre cannot store control characters (#8188 close #8186) --- internal/op/sshkey.go | 1 - server/handles/sshkey.go | 3 ++- server/sftp.go | 12 ++++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/internal/op/sshkey.go b/internal/op/sshkey.go index 6ed55658abf..139698e6ca5 100644 --- a/internal/op/sshkey.go +++ b/internal/op/sshkey.go @@ -17,7 +17,6 @@ func CreateSSHPublicKey(k *model.SSHPublicKey) (error, bool) { if err != nil { return err, false } - k.KeyStr = string(pubKey.Marshal()) k.Fingerprint = ssh.FingerprintSHA256(pubKey) k.AddedTime = time.Now() k.LastUsedTime = k.AddedTime diff --git a/server/handles/sshkey.go b/server/handles/sshkey.go index c53b46f2932..6f8d46b4969 100644 --- a/server/handles/sshkey.go +++ b/server/handles/sshkey.go @@ -6,6 +6,7 @@ import ( "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "strconv" + "strings" ) type SSHKeyAddReq struct { @@ -30,7 +31,7 @@ func AddMyPublicKey(c *gin.Context) { } key := &model.SSHPublicKey{ Title: req.Title, - KeyStr: req.Key, + KeyStr: strings.TrimSpace(req.Key), UserId: userObj.ID, } err, parsed := op.CreateSSHPublicKey(key) diff --git a/server/sftp.go b/server/sftp.go index 0455c96230f..42c676e8c17 100644 --- a/server/sftp.go +++ b/server/sftp.go @@ -113,11 +113,15 @@ func (d *SftpDriver) PublicKeyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*s } marshal := string(key.Marshal()) for _, sk := range keys { - if marshal == sk.KeyStr { - sk.LastUsedTime = time.Now() - _ = op.UpdateSSHPublicKey(&sk) - return nil, nil + if marshal != sk.KeyStr { + pubKey, _, _, _, e := ssh.ParseAuthorizedKey([]byte(sk.KeyStr)) + if e != nil || marshal != string(pubKey.Marshal()) { + continue + } } + sk.LastUsedTime = time.Now() + _ = op.UpdateSSHPublicKey(&sk) + return nil, nil } return nil, errors.New("public key refused") } From 10a76c701dc5fd1a2557f6c2367b88b490aa4a33 Mon Sep 17 00:00:00 2001 From: Ljcbaby <46277145+ljcbaby@users.noreply.github.com> Date: Thu, 27 Mar 2025 23:15:04 +0800 Subject: [PATCH 474/659] fix(db): support postgres trust/peer mode (#8198 close #8066) --- internal/bootstrap/db.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index 39b659b78f1..5f5f6fcef3e 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -68,8 +68,13 @@ func InitDB() { { dsn := database.DSN if dsn == "" { - dsn = fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", - database.Host, database.User, database.Password, database.Name, database.Port, database.SSLMode) + if database.Password != "" { + dsn = fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", + database.Host, database.User, database.Password, database.Name, database.Port, database.SSLMode) + } else { + dsn = fmt.Sprintf("host=%s user=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", + database.Host, database.User, database.Name, database.Port, database.SSLMode) + } } dB, err = gorm.Open(postgres.Open(dsn), gormConfig) } From 4fcc3a187e19de022116a41d884b02e74d2da70e Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Thu, 27 Mar 2025 23:15:47 +0800 Subject: [PATCH 475/659] fix(traffic): duplicate semaphore release when uploading (#8211 close #8180) --- drivers/189pc/utils.go | 6 +++--- drivers/baidu_netdisk/driver.go | 6 +++--- drivers/baidu_photo/driver.go | 6 +++--- drivers/mopan/driver.go | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 290d2e568aa..fb1a183ab38 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -520,9 +520,6 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo if utils.IsCanceled(upCtx) { break } - if err = sem.Acquire(ctx, 1); err != nil { - break - } byteData := make([]byte, sliceSize) if i == count { byteData = byteData[:lastPartSize] @@ -541,6 +538,9 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo partInfo := fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) threadG.Go(func(ctx context.Context) error { + if err = sem.Acquire(ctx, 1); err != nil { + return err + } defer sem.Release(1) uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo) if err != nil { diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 264f3b0225f..6ea62197b70 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -266,15 +266,15 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if utils.IsCanceled(upCtx) { break } - if err = sem.Acquire(ctx, 1); err != nil { - break - } i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize if partseq+1 == count { byteSize = lastBlockSize } threadG.Go(func(ctx context.Context) error { + if err = sem.Acquire(ctx, 1); err != nil { + return err + } defer sem.Release(1) params := map[string]string{ "method": "upload", diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index 9ee0a7ae860..eeee746f71d 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -321,9 +321,6 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if utils.IsCanceled(upCtx) { break } - if err = sem.Acquire(ctx, 1); err != nil { - break - } i, partseq, offset, byteSize := i, partseq, int64(partseq)*DEFAULT, DEFAULT if partseq+1 == count { @@ -331,6 +328,9 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil } threadG.Go(func(ctx context.Context) error { + if err = sem.Acquire(ctx, 1); err != nil { + return err + } defer sem.Release(1) uploadParams := map[string]string{ "method": "upload", diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index 2cbabe46b36..736d612a96b 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -315,9 +315,6 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if utils.IsCanceled(upCtx) { break } - if err = sem.Acquire(ctx, 1); err != nil { - break - } i, part, byteSize := i, part, initUpdload.PartSize if part.PartNumber == uploadPartData.PartTotal { byteSize = initUpdload.LastPartSize @@ -325,6 +322,9 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre // step.4 threadG.Go(func(ctx context.Context) error { + if err = sem.Acquire(ctx, 1); err != nil { + return err + } defer sem.Release(1) reader := io.NewSectionReader(file, int64(part.PartNumber-1)*initUpdload.PartSize, byteSize) req, err := part.NewRequest(ctx, driver.NewLimitedUploadStream(ctx, reader)) From 9a9aee9ac6e968fe18645b14d69830eb00e33182 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Thu, 27 Mar 2025 23:17:45 +0800 Subject: [PATCH 476/659] feat(alias): support writing to non-ambiguous paths (#8216) * feat(alias): support writing to non-ambiguous paths * feat(alias): support extract concurrency * fix(alias): extract url no pass query --- drivers/alias/driver.go | 174 +++++++++++++++++++++++++- drivers/alias/meta.go | 3 +- drivers/alias/util.go | 71 ++++++++++- internal/fs/fs.go | 23 +++- internal/offline_download/tool/add.go | 41 +++--- internal/op/archive.go | 2 +- 6 files changed, 285 insertions(+), 29 deletions(-) diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 16215c8e784..e292a62816f 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -3,6 +3,7 @@ package alias import ( "context" "errors" + stdpath "path" "strings" "github.com/alist-org/alist/v3/internal/driver" @@ -126,8 +127,46 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( return nil, errs.ObjectNotFound } +func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, parentDir, true) + if err == nil { + return fs.MakeDir(ctx, stdpath.Join(*reqPath, dirName)) + } + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot make sub-dir") + } + return err +} + +func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.Writable { + return errs.PermissionDenied + } + srcPath, err := d.getReqPath(ctx, srcObj, false) + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be moved") + } + if err != nil { + return err + } + dstPath, err := d.getReqPath(ctx, dstDir, true) + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot be moved to") + } + if err != nil { + return err + } + return fs.Move(ctx, *srcPath, *dstPath) +} + func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - reqPath, err := d.getReqPath(ctx, srcObj) + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, srcObj, false) if err == nil { return fs.Rename(ctx, *reqPath, newName) } @@ -137,8 +176,33 @@ func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) er return err } +func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.Writable { + return errs.PermissionDenied + } + srcPath, err := d.getReqPath(ctx, srcObj, false) + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be copied") + } + if err != nil { + return err + } + dstPath, err := d.getReqPath(ctx, dstDir, true) + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot be copied to") + } + if err != nil { + return err + } + _, err = fs.Copy(ctx, *srcPath, *dstPath) + return err +} + func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { - reqPath, err := d.getReqPath(ctx, obj) + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, obj, false) if err == nil { return fs.Remove(ctx, *reqPath) } @@ -148,4 +212,110 @@ func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { return err } +func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, dstDir, true) + if err == nil { + return fs.PutDirectly(ctx, *reqPath, s) + } + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot be Put") + } + return err +} + +func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, dstDir, true) + if err == nil { + return fs.PutURL(ctx, *reqPath, name, url) + } + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot offline download") + } + return err +} + +func (d *Alias) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + root, sub := d.getRootAndPath(obj.GetPath()) + dsts, ok := d.pathMap[root] + if !ok { + return nil, errs.ObjectNotFound + } + for _, dst := range dsts { + meta, err := d.getArchiveMeta(ctx, dst, sub, args) + if err == nil { + return meta, nil + } + } + return nil, errs.NotImplement +} + +func (d *Alias) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + root, sub := d.getRootAndPath(obj.GetPath()) + dsts, ok := d.pathMap[root] + if !ok { + return nil, errs.ObjectNotFound + } + for _, dst := range dsts { + l, err := d.listArchive(ctx, dst, sub, args) + if err == nil { + return l, nil + } + } + return nil, errs.NotImplement +} + +func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // alias的两个驱动，一个支持驱动提取，一个不支持，如何兼容？ + // 如果访问的是不支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回errs.NotImplement，提取URL前缀就会是/ae，Extract就不会被调用 + // 如果访问的是支持驱动提取的驱动内的压缩文件，GetArchiveMeta就会返回有效值，提取URL前缀就会是/ad，Extract就会被调用 + root, sub := d.getRootAndPath(obj.GetPath()) + dsts, ok := d.pathMap[root] + if !ok { + return nil, errs.ObjectNotFound + } + for _, dst := range dsts { + link, err := d.extract(ctx, dst, sub, args) + if err == nil { + if !args.Redirect && len(link.URL) > 0 { + if d.DownloadConcurrency > 0 { + link.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + link.PartSize = d.DownloadPartSize * utils.KB + } + } + return link, nil + } + } + return nil, errs.NotImplement +} + +func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { + if !d.Writable { + return errs.PermissionDenied + } + srcPath, err := d.getReqPath(ctx, srcObj, false) + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be decompressed") + } + if err != nil { + return err + } + dstPath, err := d.getReqPath(ctx, dstDir, true) + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot be decompressed to") + } + if err != nil { + return err + } + _, err = fs.ArchiveDecompress(ctx, *srcPath, *dstPath, args) + return err +} + var _ driver.Driver = (*Alias)(nil) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index ed657a5d21b..70dc59f0b73 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -13,13 +13,14 @@ type Addition struct { ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"` DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` + Writable bool `json:"writable" type:"bool" default:"false"` } var config = driver.Config{ Name: "Alias", LocalSort: true, NoCache: true, - NoUpload: true, + NoUpload: false, DefaultRoot: "/", ProxyRangeOption: true, } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 2157a43d7eb..ffb0b84f605 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -3,9 +3,11 @@ package alias import ( "context" "fmt" + "net/url" stdpath "path" "strings" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" @@ -125,9 +127,9 @@ func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) return link, err } -func (d *Alias) getReqPath(ctx context.Context, obj model.Obj) (*string, error) { +func (d *Alias) getReqPath(ctx context.Context, obj model.Obj, isParent bool) (*string, error) { root, sub := d.getRootAndPath(obj.GetPath()) - if sub == "" { + if sub == "" && !isParent { return nil, errs.NotSupport } dsts, ok := d.pathMap[root] @@ -156,3 +158,68 @@ func (d *Alias) getReqPath(ctx context.Context, obj model.Obj) (*string, error) } return reqPath, nil } + +func (d *Alias) getArchiveMeta(ctx context.Context, dst, sub string, args model.ArchiveArgs) (model.ArchiveMeta, error) { + reqPath := stdpath.Join(dst, sub) + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + return nil, err + } + if _, ok := storage.(driver.ArchiveReader); ok { + return op.GetArchiveMeta(ctx, storage, reqActualPath, model.ArchiveMetaArgs{ + ArchiveArgs: args, + Refresh: true, + }) + } + return nil, errs.NotImplement +} + +func (d *Alias) listArchive(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) ([]model.Obj, error) { + reqPath := stdpath.Join(dst, sub) + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + return nil, err + } + if _, ok := storage.(driver.ArchiveReader); ok { + return op.ListArchive(ctx, storage, reqActualPath, model.ArchiveListArgs{ + ArchiveInnerArgs: args, + Refresh: true, + }) + } + return nil, errs.NotImplement +} + +func (d *Alias) extract(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) (*model.Link, error) { + reqPath := stdpath.Join(dst, sub) + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + return nil, err + } + if _, ok := storage.(driver.ArchiveReader); ok { + if _, ok := storage.(*Alias); !ok && !args.Redirect { + link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) + return link, err + } + _, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err + } + if common.ShouldProxy(storage, stdpath.Base(sub)) { + link := &model.Link{ + URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", + common.GetApiUrl(args.HttpReq), + utils.EncodePath(reqPath, true), + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + sign.SignArchive(reqPath)), + } + if args.HttpReq != nil && d.ProxyRange { + link.RangeReadCloser = common.NoProxyRange + } + return link, nil + } + link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) + return link, err + } + return nil, errs.NotImplement +} diff --git a/internal/fs/fs.go b/internal/fs/fs.go index a873f917301..01818e5fd71 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -2,12 +2,15 @@ package fs import ( "context" + log "github.com/sirupsen/logrus" + "io" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/task" - log "github.com/sirupsen/logrus" - "io" + "github.com/pkg/errors" ) // the param named path of functions in this package is a mount path @@ -168,3 +171,19 @@ func Other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { } return res, err } + +func PutURL(ctx context.Context, path, dstName, urlStr string) error { + storage, dstDirActualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + if storage.Config().NoUpload { + return errors.WithStack(errs.UploadNotSupported) + } + _, ok := storage.(driver.PutURL) + _, okResult := storage.(driver.PutURLResult) + if !ok && !okResult { + return errs.NotImplement + } + return op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 884e166bab7..d64e43e8615 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -2,20 +2,20 @@ package tool import ( "context" - _115 "github.com/alist-org/alist/v3/drivers/115" - "github.com/alist-org/alist/v3/drivers/pikpak" - "github.com/alist-org/alist/v3/drivers/thunder" - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/internal/task" "net/url" - "path" + stdpath "path" "path/filepath" + _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/task" "github.com/google/uuid" "github.com/pkg/errors" ) @@ -59,8 +59,11 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } } // try putting url - if args.Tool == "SimpleHttp" && tryPutUrl(ctx, storage, dstDirActualPath, args.URL) { - return nil, nil + if args.Tool == "SimpleHttp" { + err = tryPutUrl(ctx, args.DstDirPath, args.URL) + if err == nil || !errors.Is(err, errs.NotImplement) { + return nil, err + } } // get tool @@ -118,17 +121,13 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro return t, nil } -func tryPutUrl(ctx context.Context, storage driver.Driver, dstDirActualPath, urlStr string) bool { - _, ok := storage.(driver.PutURL) - _, okResult := storage.(driver.PutURLResult) - if !ok && !okResult { - return false - } +func tryPutUrl(ctx context.Context, path, urlStr string) error { + var dstName string u, err := url.Parse(urlStr) - if err != nil { - return false + if err == nil { + dstName = stdpath.Base(u.Path) + } else { + dstName = "UnnamedURL" } - dstName := path.Base(u.Path) - err = op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) - return err == nil + return fs.PutURL(ctx, path, dstName, urlStr) } diff --git a/internal/op/archive.go b/internal/op/archive.go index a241838c9c4..4015e299cb6 100644 --- a/internal/op/archive.go +++ b/internal/op/archive.go @@ -84,7 +84,7 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg meta, err := storageAr.GetArchiveMeta(ctx, obj, args.ArchiveArgs) if !errors.Is(err, errs.NotImplement) { archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: true} - if meta.GetTree() != nil { + if meta != nil && meta.GetTree() != nil { archiveMetaProvider.Sort = &storage.GetStorage().Sort } if !storage.Config().NoCache { From 44cc71d354f6a21130010a9ae836510a429180fa Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 27 Mar 2025 23:18:15 +0800 Subject: [PATCH 477/659] fix(cloudreve): enable SetContentLength for uploading to local policy (#8228 close #8174) * fix(cloudreve): upload failure to return error msg instead of deletion success * fix(cloudreve): enable SetContentLength for uploading to local policy * refactor(cloudreve): move local policy upload logic to utils for better error handling * refactor(cloudreve): unified upload code style * refactor(cloudreve): improve user agent handling --- drivers/cloudreve/driver.go | 33 ++------------------- drivers/cloudreve/util.go | 58 ++++++++++++++++++++++++++++++------- 2 files changed, 50 insertions(+), 41 deletions(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 33ef7ddcc0a..d0ab30b6c11 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -1,13 +1,10 @@ package cloudreve import ( - "bytes" "context" - "errors" "io" "net/http" "path" - "strconv" "strings" "github.com/alist-org/alist/v3/drivers/base" @@ -168,39 +165,13 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File case "remote": // 从机存储 err = d.upRemote(ctx, stream, u, up) case "local": // 本机存储 - var chunkSize = u.ChunkSize - var buf []byte - var chunk int - for { - var n int - buf = make([]byte, chunkSize) - n, err = io.ReadAtLeast(stream, buf, chunkSize) - if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { - if err == io.EOF { - return nil - } - return err - } - if n == 0 { - break - } - buf = buf[:n] - err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { - req.SetHeader("Content-Type", "application/octet-stream") - req.SetHeader("Content-Length", strconv.Itoa(n)) - req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(buf))) - }, nil) - if err != nil { - break - } - chunk++ - } + err = d.upLocal(ctx, stream, u, up) default: err = errs.NotImplement } if err != nil { // 删除失败的会话 - err = d.request(http.MethodDelete, "/file/upload/"+u.SessionID, nil, nil) + _ = d.request(http.MethodDelete, "/file/upload/"+u.SessionID, nil, nil) return err } return nil diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index f41b6b84d96..cffa7988bb3 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -27,17 +27,20 @@ import ( const loginPath = "/user/session" +func (d *Cloudreve) getUA() string { + if d.CustomUA != "" { + return d.CustomUA + } + return base.UserAgent +} + func (d *Cloudreve) request(method string, path string, callback base.ReqCallback, out interface{}) error { u := d.Address + "/api/v3" + path - ua := d.CustomUA - if ua == "" { - ua = base.UserAgent - } req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": "cloudreve-session=" + d.Cookie, "Accept": "application/json, text/plain, */*", - "User-Agent": ua, + "User-Agent": d.getUA(), }) var r Resp @@ -161,15 +164,11 @@ func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { if !d.Addition.EnableThumbAndFolderSize { return model.Thumbnail{}, nil } - ua := d.CustomUA - if ua == "" { - ua = base.UserAgent - } req := base.NoRedirectClient.R() req.SetHeaders(map[string]string{ "Cookie": "cloudreve-session=" + d.Cookie, "Accept": "image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", - "User-Agent": ua, + "User-Agent": d.getUA(), }) resp, err := req.Execute(http.MethodGet, d.Address+"/api/v3/file/thumb/"+file.Id) if err != nil { @@ -180,6 +179,43 @@ func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { }, nil } +func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + for finish < stream.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + utils.Log.Debugf("[Cloudreve-Local] upload: %d", finish) + var byteSize = DEFAULT + left := stream.GetSize() - finish + if left < DEFAULT { + byteSize = left + } + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { + req.SetHeader("Content-Type", "application/octet-stream") + req.SetContentLength(true) + req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) + req.SetHeader("User-Agent", d.getUA()) + req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) + }, nil) + if err != nil { + break + } + finish += byteSize + up(float64(finish) * 100 / float64(stream.GetSize())) + chunk++ + } + return nil +} + func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { uploadUrl := u.UploadURLs[0] credential := u.Credential @@ -211,6 +247,7 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U req.ContentLength = byteSize // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) finish += byteSize res, err := base.HttpClient.Do(req) if err != nil { @@ -251,6 +288,7 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u req.ContentLength = byteSize // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) + req.Header.Set("User-Agent", d.getUA()) finish += byteSize res, err := base.HttpClient.Do(req) if err != nil { From 704d3854df234b59b0635d32aa111aa5a811bfa1 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Thu, 27 Mar 2025 23:18:34 +0800 Subject: [PATCH 478/659] feat(alist_v3): support forward archive requests (#8230) * feat(alist_v3): support forward archive requests * fix: encode all inner path --- drivers/alist_v3/driver.go | 145 ++++++++++++++++++++++++++++++++++--- drivers/alist_v3/meta.go | 13 ++-- drivers/alist_v3/types.go | 87 ++++++++++++++++++++++ drivers/alist_v3/util.go | 18 +++-- 4 files changed, 239 insertions(+), 24 deletions(-) diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index 5a299ea0aec..ac7e16a1d16 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -5,12 +5,14 @@ import ( "fmt" "io" "net/http" + "net/url" "path" "strings" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -34,7 +36,7 @@ func (d *AListV3) GetAddition() driver.Additional { func (d *AListV3) Init(ctx context.Context) error { d.Addition.Address = strings.TrimSuffix(d.Addition.Address, "/") var resp common.Resp[MeResp] - _, err := d.request("/me", http.MethodGet, func(req *resty.Request) { + _, _, err := d.request("/me", http.MethodGet, func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { @@ -48,15 +50,15 @@ func (d *AListV3) Init(ctx context.Context) error { } } // re-get the user info - _, err = d.request("/me", http.MethodGet, func(req *resty.Request) { + _, _, err = d.request("/me", http.MethodGet, func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { return err } if resp.Data.Role == model.GUEST { - url := d.Address + "/api/public/settings" - res, err := base.RestyClient.R().Get(url) + u := d.Address + "/api/public/settings" + res, err := base.RestyClient.R().Get(u) if err != nil { return err } @@ -74,7 +76,7 @@ func (d *AListV3) Drop(ctx context.Context) error { func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var resp common.Resp[FsListResp] - _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ListReq{ PageReq: model.PageReq{ Page: 1, @@ -116,7 +118,7 @@ func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) userAgent = base.UserAgent } } - _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(FsGetReq{ Path: file.GetPath(), Password: d.MetaPassword, @@ -131,7 +133,7 @@ func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *AListV3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.request("/fs/mkdir", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/mkdir", http.MethodPost, func(req *resty.Request) { req.SetBody(MkdirOrLinkReq{ Path: path.Join(parentDir.GetPath(), dirName), }) @@ -140,7 +142,7 @@ func (d *AListV3) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri } func (d *AListV3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("/fs/move", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/move", http.MethodPost, func(req *resty.Request) { req.SetBody(MoveCopyReq{ SrcDir: path.Dir(srcObj.GetPath()), DstDir: dstDir.GetPath(), @@ -151,7 +153,7 @@ func (d *AListV3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *AListV3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err := d.request("/fs/rename", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/rename", http.MethodPost, func(req *resty.Request) { req.SetBody(RenameReq{ Path: srcObj.GetPath(), Name: newName, @@ -161,7 +163,7 @@ func (d *AListV3) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *AListV3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("/fs/copy", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(MoveCopyReq{ SrcDir: path.Dir(srcObj.GetPath()), DstDir: dstDir.GetPath(), @@ -172,7 +174,7 @@ func (d *AListV3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { - _, err := d.request("/fs/remove", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/remove", http.MethodPost, func(req *resty.Request) { req.SetBody(RemoveReq{ Dir: path.Dir(obj.GetPath()), Names: []string{obj.GetName()}, @@ -232,6 +234,127 @@ func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame return nil } +func (d *AListV3) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotImplement + } + var resp common.Resp[ArchiveMetaResp] + _, code, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }) + }) + if code == 202 { + return nil, errs.WrongArchivePassword + } + if err != nil { + return nil, err + } + var tree []model.ObjTree + if resp.Data.Content != nil { + tree = make([]model.ObjTree, 0, len(resp.Data.Content)) + for _, content := range resp.Data.Content { + tree = append(tree, &content) + } + } + return &model.ArchiveMetaInfo{ + Comment: resp.Data.Comment, + Encrypted: resp.Data.Encrypted, + Tree: tree, + }, nil +} + +func (d *AListV3) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotImplement + } + var resp common.Resp[ArchiveListResp] + _, code, err := d.request("/fs/archive/list", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveListReq{ + ArchiveMetaReq: ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }, + PageReq: model.PageReq{ + Page: 1, + PerPage: 0, + }, + InnerPath: args.InnerPath, + }) + }) + if code == 202 { + return nil, errs.WrongArchivePassword + } + if err != nil { + return nil, err + } + var files []model.Obj + for _, f := range resp.Data.Content { + file := model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.Modified, + Ctime: f.Created, + Size: f.Size, + IsFolder: f.IsDir, + HashInfo: utils.FromString(f.HashInfo), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, + } + files = append(files, &file) + } + return files, nil +} + +func (d *AListV3) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotSupport + } + var resp common.Resp[ArchiveMetaResp] + _, _, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }) + }) + if err != nil { + return nil, err + } + return &model.Link{ + URL: fmt.Sprintf("%s?inner=%s&pass=%s&sign=%s", + resp.Data.RawURL, + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + resp.Data.Sign), + }, nil +} + +func (d *AListV3) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { + if !d.ForwardArchiveReq { + return errs.NotImplement + } + dir, name := path.Split(srcObj.GetPath()) + _, _, err := d.request("/fs/archive/decompress", http.MethodPost, func(req *resty.Request) { + req.SetBody(DecompressReq{ + ArchivePass: args.Password, + CacheFull: args.CacheFull, + DstDir: dstDir.GetPath(), + InnerPath: args.InnerPath, + Name: []string{name}, + PutIntoNewDir: args.PutIntoNewDir, + SrcDir: dir, + }) + }) + return err +} + //func (d *AList) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/alist_v3/meta.go b/drivers/alist_v3/meta.go index cc5f2189395..1e8b3c53c0a 100644 --- a/drivers/alist_v3/meta.go +++ b/drivers/alist_v3/meta.go @@ -7,12 +7,13 @@ import ( type Addition struct { driver.RootPath - Address string `json:"url" required:"true"` - MetaPassword string `json:"meta_password"` - Username string `json:"username"` - Password string `json:"password"` - Token string `json:"token"` - PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` + Address string `json:"url" required:"true"` + MetaPassword string `json:"meta_password"` + Username string `json:"username"` + Password string `json:"password"` + Token string `json:"token"` + PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` + ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` } var config = driver.Config{ diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go index e517307f3ef..1ae7926e078 100644 --- a/drivers/alist_v3/types.go +++ b/drivers/alist_v3/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" ) type ListReq struct { @@ -81,3 +82,89 @@ type MeResp struct { SsoId string `json:"sso_id"` Otp bool `json:"otp"` } + +type ArchiveMetaReq struct { + ArchivePass string `json:"archive_pass"` + Password string `json:"password"` + Path string `json:"path"` + Refresh bool `json:"refresh"` +} + +type TreeResp struct { + ObjResp + Children []TreeResp `json:"children"` + hashCache *utils.HashInfo +} + +func (t *TreeResp) GetSize() int64 { + return t.Size +} + +func (t *TreeResp) GetName() string { + return t.Name +} + +func (t *TreeResp) ModTime() time.Time { + return t.Modified +} + +func (t *TreeResp) CreateTime() time.Time { + return t.Created +} + +func (t *TreeResp) IsDir() bool { + return t.ObjResp.IsDir +} + +func (t *TreeResp) GetHash() utils.HashInfo { + return utils.FromString(t.HashInfo) +} + +func (t *TreeResp) GetID() string { + return "" +} + +func (t *TreeResp) GetPath() string { + return "" +} + +func (t *TreeResp) GetChildren() []model.ObjTree { + ret := make([]model.ObjTree, 0, len(t.Children)) + for _, child := range t.Children { + ret = append(ret, &child) + } + return ret +} + +func (t *TreeResp) Thumb() string { + return t.ObjResp.Thumb +} + +type ArchiveMetaResp struct { + Comment string `json:"comment"` + Encrypted bool `json:"encrypted"` + Content []TreeResp `json:"content"` + RawURL string `json:"raw_url"` + Sign string `json:"sign"` +} + +type ArchiveListReq struct { + model.PageReq + ArchiveMetaReq + InnerPath string `json:"inner_path"` +} + +type ArchiveListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` +} + +type DecompressReq struct { + ArchivePass string `json:"archive_pass"` + CacheFull bool `json:"cache_full"` + DstDir string `json:"dst_dir"` + InnerPath string `json:"inner_path"` + Name []string `json:"name"` + PutIntoNewDir bool `json:"put_into_new_dir"` + SrcDir string `json:"src_dir"` +} diff --git a/drivers/alist_v3/util.go b/drivers/alist_v3/util.go index 5ede285af5b..50c20250313 100644 --- a/drivers/alist_v3/util.go +++ b/drivers/alist_v3/util.go @@ -17,7 +17,7 @@ func (d *AListV3) login() error { return nil } var resp common.Resp[LoginResp] - _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(base.Json{ "username": d.Username, "password": d.Password, @@ -31,7 +31,7 @@ func (d *AListV3) login() error { return nil } -func (d *AListV3) request(api, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { +func (d *AListV3) request(api, method string, callback base.ReqCallback, retry ...bool) ([]byte, int, error) { url := d.Address + "/api" + api req := base.RestyClient.R() req.SetHeader("Authorization", d.Token) @@ -40,22 +40,26 @@ func (d *AListV3) request(api, method string, callback base.ReqCallback, retry . } res, err := req.Execute(method, url) if err != nil { - return nil, err + code := 0 + if res != nil { + code = res.StatusCode() + } + return nil, code, err } log.Debugf("[alist_v3] response body: %s", res.String()) if res.StatusCode() >= 400 { - return nil, fmt.Errorf("request failed, status: %s", res.Status()) + return nil, res.StatusCode(), fmt.Errorf("request failed, status: %s", res.Status()) } code := utils.Json.Get(res.Body(), "code").ToInt() if code != 200 { if (code == 401 || code == 403) && !utils.IsBool(retry...) { err = d.login() if err != nil { - return nil, err + return nil, code, err } return d.request(api, method, callback, true) } - return nil, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) + return nil, code, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) } - return res.Body(), nil + return res.Body(), 200, nil } From 1335f803622308b4c3dabefdc84ab7b19fd7b4ec Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Thu, 27 Mar 2025 23:20:44 +0800 Subject: [PATCH 479/659] feat(archive): support multipart archives (#8184 close #8015) * feat(archive): multipart support & sevenzip tool * feat(archive): rardecode tool * feat(archive): support decompress multi-selected * fix(archive): decompress response filter internal * feat(archive): support multipart zip * fix: more applicable AcceptedMultipartExtensions interface --- go.mod | 6 +- internal/archive/all.go | 2 + internal/archive/archives/archives.go | 26 +-- internal/archive/iso9660/iso9660.go | 22 ++- internal/archive/rardecode/rardecode.go | 140 +++++++++++++++ internal/archive/rardecode/utils.go | 225 ++++++++++++++++++++++++ internal/archive/sevenzip/sevenzip.go | 72 ++++++++ internal/archive/sevenzip/utils.go | 61 +++++++ internal/archive/tool/base.go | 14 +- internal/archive/tool/helper.go | 201 +++++++++++++++++++++ internal/archive/tool/utils.go | 17 +- internal/archive/zip/utils.go | 102 +++++------ internal/archive/zip/zip.go | 162 +++-------------- internal/driver/driver.go | 4 +- internal/fs/archive.go | 67 ++++--- internal/op/archive.go | 113 +++++++++--- internal/stream/limit.go | 2 +- internal/stream/stream.go | 29 ++- server/handles/archive.go | 93 ++++++---- 19 files changed, 1040 insertions(+), 318 deletions(-) create mode 100644 internal/archive/rardecode/rardecode.go create mode 100644 internal/archive/rardecode/utils.go create mode 100644 internal/archive/sevenzip/sevenzip.go create mode 100644 internal/archive/sevenzip/utils.go create mode 100644 internal/archive/tool/helper.go diff --git a/go.mod b/go.mod index a06c62ba8d8..5ed8a27baa5 100644 --- a/go.mod +++ b/go.mod @@ -85,7 +85,7 @@ require ( github.com/blevesearch/go-faiss v1.0.20 // indirect github.com/blevesearch/zapx/v16 v16.1.5 // indirect github.com/bodgit/plumbing v1.3.0 // indirect - github.com/bodgit/sevenzip v1.6.0 // indirect + github.com/bodgit/sevenzip v1.6.0 github.com/bodgit/windows v1.0.1 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/charmbracelet/x/ansi v0.2.3 // indirect @@ -106,14 +106,14 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 - github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 // indirect + github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 github.com/sorairolake/lzip-go v0.3.5 // indirect github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect github.com/xhofe/115-sdk-go v0.1.4 github.com/yuin/goldmark v1.7.8 - go4.org v0.0.0-20230225012048-214862532bf5 // indirect + go4.org v0.0.0-20230225012048-214862532bf5 resty.dev/v3 v3.0.0-beta.2 // indirect ) diff --git a/internal/archive/all.go b/internal/archive/all.go index 18167933b1c..63206cb89ed 100644 --- a/internal/archive/all.go +++ b/internal/archive/all.go @@ -3,5 +3,7 @@ package archive import ( _ "github.com/alist-org/alist/v3/internal/archive/archives" _ "github.com/alist-org/alist/v3/internal/archive/iso9660" + _ "github.com/alist-org/alist/v3/internal/archive/rardecode" + _ "github.com/alist-org/alist/v3/internal/archive/sevenzip" _ "github.com/alist-org/alist/v3/internal/archive/zip" ) diff --git a/internal/archive/archives/archives.go b/internal/archive/archives/archives.go index 6d48624fa2e..0a42cd0c512 100644 --- a/internal/archive/archives/archives.go +++ b/internal/archive/archives/archives.go @@ -16,14 +16,18 @@ import ( type Archives struct { } -func (*Archives) AcceptedExtensions() []string { +func (Archives) AcceptedExtensions() []string { return []string{ - ".br", ".bz2", ".gz", ".lz4", ".lz", ".sz", ".s2", ".xz", ".zz", ".zst", ".tar", ".rar", ".7z", + ".br", ".bz2", ".gz", ".lz4", ".lz", ".sz", ".s2", ".xz", ".zz", ".zst", ".tar", } } -func (*Archives) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { - fsys, err := getFs(ss, args) +func (Archives) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{} +} + +func (Archives) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + fsys, err := getFs(ss[0], args) if err != nil { return nil, err } @@ -47,8 +51,8 @@ func (*Archives) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (mod }, nil } -func (*Archives) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { - fsys, err := getFs(ss, args.ArchiveArgs) +func (Archives) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + fsys, err := getFs(ss[0], args.ArchiveArgs) if err != nil { return nil, err } @@ -69,8 +73,8 @@ func (*Archives) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([ }) } -func (*Archives) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { - fsys, err := getFs(ss, args.ArchiveArgs) +func (Archives) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + fsys, err := getFs(ss[0], args.ArchiveArgs) if err != nil { return nil, 0, err } @@ -85,8 +89,8 @@ func (*Archives) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) return file, stat.Size(), nil } -func (*Archives) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { - fsys, err := getFs(ss, args.ArchiveArgs) +func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + fsys, err := getFs(ss[0], args.ArchiveArgs) if err != nil { return err } @@ -133,5 +137,5 @@ func (*Archives) Decompress(ss *stream.SeekableStream, outputPath string, args m var _ tool.Tool = (*Archives)(nil) func init() { - tool.RegisterTool(&Archives{}) + tool.RegisterTool(Archives{}) } diff --git a/internal/archive/iso9660/iso9660.go b/internal/archive/iso9660/iso9660.go index e9cb3f538ec..be107d7b4c4 100644 --- a/internal/archive/iso9660/iso9660.go +++ b/internal/archive/iso9660/iso9660.go @@ -14,19 +14,23 @@ import ( type ISO9660 struct { } -func (t *ISO9660) AcceptedExtensions() []string { +func (ISO9660) AcceptedExtensions() []string { return []string{".iso"} } -func (t *ISO9660) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { +func (ISO9660) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{} +} + +func (ISO9660) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { return &model.ArchiveMetaInfo{ Comment: "", Encrypted: false, }, nil } -func (t *ISO9660) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { - img, err := getImage(ss) +func (ISO9660) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + img, err := getImage(ss[0]) if err != nil { return nil, err } @@ -48,8 +52,8 @@ func (t *ISO9660) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ( return ret, nil } -func (t *ISO9660) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { - img, err := getImage(ss) +func (ISO9660) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + img, err := getImage(ss[0]) if err != nil { return nil, 0, err } @@ -63,8 +67,8 @@ func (t *ISO9660) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs return io.NopCloser(obj.Reader()), obj.Size(), nil } -func (t *ISO9660) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { - img, err := getImage(ss) +func (ISO9660) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + img, err := getImage(ss[0]) if err != nil { return err } @@ -92,5 +96,5 @@ func (t *ISO9660) Decompress(ss *stream.SeekableStream, outputPath string, args var _ tool.Tool = (*ISO9660)(nil) func init() { - tool.RegisterTool(&ISO9660{}) + tool.RegisterTool(ISO9660{}) } diff --git a/internal/archive/rardecode/rardecode.go b/internal/archive/rardecode/rardecode.go new file mode 100644 index 00000000000..cd31d1a40e0 --- /dev/null +++ b/internal/archive/rardecode/rardecode.go @@ -0,0 +1,140 @@ +package rardecode + +import ( + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/nwaples/rardecode/v2" + "io" + "os" + stdpath "path" + "strings" +) + +type RarDecoder struct{} + +func (RarDecoder) AcceptedExtensions() []string { + return []string{".rar"} +} + +func (RarDecoder) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{ + ".part1.rar": {".part%d.rar", 2}, + } +} + +func (RarDecoder) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + l, err := list(ss, args.Password) + if err != nil { + return nil, err + } + _, tree := tool.GenerateMetaTreeFromFolderTraversal(l) + return &model.ArchiveMetaInfo{ + Comment: "", + Encrypted: false, + Tree: tree, + }, nil +} + +func (RarDecoder) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +func (RarDecoder) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + reader, err := getReader(ss, args.Password) + if err != nil { + return nil, 0, err + } + innerPath := strings.TrimPrefix(args.InnerPath, "/") + for { + var header *rardecode.FileHeader + header, err = reader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, 0, err + } + if header.Name == innerPath { + if header.IsDir { + break + } + return io.NopCloser(reader), header.UnPackedSize, nil + } + } + return nil, 0, errs.ObjectNotFound +} + +func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + reader, err := getReader(ss, args.Password) + if err != nil { + return err + } + if args.InnerPath == "/" { + for { + var header *rardecode.FileHeader + header, err = reader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + name := header.Name + if header.IsDir { + name = name + "/" + } + err = decompress(reader, header, name, outputPath) + if err != nil { + return err + } + } + } else { + innerPath := strings.TrimPrefix(args.InnerPath, "/") + innerBase := stdpath.Base(innerPath) + createdBaseDir := false + for { + var header *rardecode.FileHeader + header, err = reader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + name := header.Name + if header.IsDir { + name = name + "/" + } + if name == innerPath { + err = _decompress(reader, header, outputPath, up) + if err != nil { + return err + } + break + } else if strings.HasPrefix(name, innerPath+"/") { + targetPath := stdpath.Join(outputPath, innerBase) + if !createdBaseDir { + err = os.Mkdir(targetPath, 0700) + if err != nil { + return err + } + createdBaseDir = true + } + restPath := strings.TrimPrefix(name, innerPath+"/") + err = decompress(reader, header, restPath, targetPath) + if err != nil { + return err + } + } + } + } + return nil +} + +var _ tool.Tool = (*RarDecoder)(nil) + +func init() { + tool.RegisterTool(RarDecoder{}) +} diff --git a/internal/archive/rardecode/utils.go b/internal/archive/rardecode/utils.go new file mode 100644 index 00000000000..5790ec58a22 --- /dev/null +++ b/internal/archive/rardecode/utils.go @@ -0,0 +1,225 @@ +package rardecode + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/nwaples/rardecode/v2" + "io" + "io/fs" + "os" + stdpath "path" + "sort" + "strings" + "time" +) + +type VolumeFile struct { + stream.SStreamReadAtSeeker + name string +} + +func (v *VolumeFile) Name() string { + return v.name +} + +func (v *VolumeFile) Size() int64 { + return v.SStreamReadAtSeeker.GetRawStream().GetSize() +} + +func (v *VolumeFile) Mode() fs.FileMode { + return 0644 +} + +func (v *VolumeFile) ModTime() time.Time { + return v.SStreamReadAtSeeker.GetRawStream().ModTime() +} + +func (v *VolumeFile) IsDir() bool { + return false +} + +func (v *VolumeFile) Sys() any { + return nil +} + +func (v *VolumeFile) Stat() (fs.FileInfo, error) { + return v, nil +} + +func (v *VolumeFile) Close() error { + return nil +} + +type VolumeFs struct { + parts map[string]*VolumeFile +} + +func (v *VolumeFs) Open(name string) (fs.File, error) { + file, ok := v.parts[name] + if !ok { + return nil, fs.ErrNotExist + } + return file, nil +} + +func makeOpts(ss []*stream.SeekableStream) (string, rardecode.Option, error) { + if len(ss) == 1 { + reader, err := stream.NewReadAtSeeker(ss[0], 0) + if err != nil { + return "", nil, err + } + fileName := "file.rar" + fsys := &VolumeFs{parts: map[string]*VolumeFile{ + fileName: {SStreamReadAtSeeker: reader, name: fileName}, + }} + return fileName, rardecode.FileSystem(fsys), nil + } else { + parts := make(map[string]*VolumeFile, len(ss)) + for i, s := range ss { + reader, err := stream.NewReadAtSeeker(s, 0) + if err != nil { + return "", nil, err + } + fileName := fmt.Sprintf("file.part%d.rar", i+1) + parts[fileName] = &VolumeFile{SStreamReadAtSeeker: reader, name: fileName} + } + return "file.part1.rar", rardecode.FileSystem(&VolumeFs{parts: parts}), nil + } +} + +type WrapReader struct { + files []*rardecode.File +} + +func (r *WrapReader) Files() []tool.SubFile { + ret := make([]tool.SubFile, 0, len(r.files)) + for _, f := range r.files { + ret = append(ret, &WrapFile{File: f}) + } + return ret +} + +type WrapFile struct { + *rardecode.File +} + +func (f *WrapFile) Name() string { + if f.File.IsDir { + return f.File.Name + "/" + } + return f.File.Name +} + +func (f *WrapFile) FileInfo() fs.FileInfo { + return &WrapFileInfo{File: f.File} +} + +type WrapFileInfo struct { + *rardecode.File +} + +func (f *WrapFileInfo) Name() string { + return stdpath.Base(f.File.Name) +} + +func (f *WrapFileInfo) Size() int64 { + return f.File.UnPackedSize +} + +func (f *WrapFileInfo) ModTime() time.Time { + return f.File.ModificationTime +} + +func (f *WrapFileInfo) IsDir() bool { + return f.File.IsDir +} + +func (f *WrapFileInfo) Sys() any { + return nil +} + +func list(ss []*stream.SeekableStream, password string) (*WrapReader, error) { + fileName, fsOpt, err := makeOpts(ss) + if err != nil { + return nil, err + } + opts := []rardecode.Option{fsOpt} + if password != "" { + opts = append(opts, rardecode.Password(password)) + } + files, err := rardecode.List(fileName, opts...) + // rardecode输出文件列表的顺序不一定是父目录在前，子目录在后 + // 父路径的长度一定比子路径短，排序后的files可保证父路径在前 + sort.Slice(files, func(i, j int) bool { + return len(files[i].Name) < len(files[j].Name) + }) + if err != nil { + return nil, filterPassword(err) + } + return &WrapReader{files: files}, nil +} + +func getReader(ss []*stream.SeekableStream, password string) (*rardecode.Reader, error) { + fileName, fsOpt, err := makeOpts(ss) + if err != nil { + return nil, err + } + opts := []rardecode.Option{fsOpt} + if password != "" { + opts = append(opts, rardecode.Password(password)) + } + rc, err := rardecode.OpenReader(fileName, opts...) + if err != nil { + return nil, filterPassword(err) + } + ss[0].Closers.Add(rc) + return &rc.Reader, nil +} + +func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, filePath, outputPath string) error { + targetPath := outputPath + dir, base := stdpath.Split(filePath) + if dir != "" { + targetPath = stdpath.Join(targetPath, dir) + err := os.MkdirAll(targetPath, 0700) + if err != nil { + return err + } + } + if base != "" { + err := _decompress(reader, header, targetPath, func(_ float64) {}) + if err != nil { + return err + } + } + return nil +} + +func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, targetPath string, up model.UpdateProgress) error { + f, err := os.OpenFile(stdpath.Join(targetPath, stdpath.Base(header.Name)), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: reader, + Size: header.UnPackedSize, + }, + UpdateProgress: up, + }) + if err != nil { + return err + } + return nil +} + +func filterPassword(err error) error { + if err != nil && strings.Contains(err.Error(), "password") { + return errs.WrongArchivePassword + } + return err +} diff --git a/internal/archive/sevenzip/sevenzip.go b/internal/archive/sevenzip/sevenzip.go new file mode 100644 index 00000000000..281699664f8 --- /dev/null +++ b/internal/archive/sevenzip/sevenzip.go @@ -0,0 +1,72 @@ +package sevenzip + +import ( + "io" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" +) + +type SevenZip struct{} + +func (SevenZip) AcceptedExtensions() []string { + return []string{".7z"} +} + +func (SevenZip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{ + ".7z.001": {".7z.%.3d", 2}, + } +} + +func (SevenZip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + reader, err := getReader(ss, args.Password) + if err != nil { + return nil, err + } + _, tree := tool.GenerateMetaTreeFromFolderTraversal(&WrapReader{Reader: reader}) + return &model.ArchiveMetaInfo{ + Comment: "", + Encrypted: args.Password != "", + Tree: tree, + }, nil +} + +func (SevenZip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +func (SevenZip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + reader, err := getReader(ss, args.Password) + if err != nil { + return nil, 0, err + } + innerPath := strings.TrimPrefix(args.InnerPath, "/") + for _, file := range reader.File { + if file.Name == innerPath { + r, e := file.Open() + if e != nil { + return nil, 0, e + } + return r, file.FileInfo().Size(), nil + } + } + return nil, 0, errs.ObjectNotFound +} + +func (SevenZip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + reader, err := getReader(ss, args.Password) + if err != nil { + return err + } + return tool.DecompressFromFolderTraversal(&WrapReader{Reader: reader}, outputPath, args, up) +} + +var _ tool.Tool = (*SevenZip)(nil) + +func init() { + tool.RegisterTool(SevenZip{}) +} diff --git a/internal/archive/sevenzip/utils.go b/internal/archive/sevenzip/utils.go new file mode 100644 index 00000000000..624ba1879c8 --- /dev/null +++ b/internal/archive/sevenzip/utils.go @@ -0,0 +1,61 @@ +package sevenzip + +import ( + "errors" + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/bodgit/sevenzip" + "io" + "io/fs" +) + +type WrapReader struct { + Reader *sevenzip.Reader +} + +func (r *WrapReader) Files() []tool.SubFile { + ret := make([]tool.SubFile, 0, len(r.Reader.File)) + for _, f := range r.Reader.File { + ret = append(ret, &WrapFile{f: f}) + } + return ret +} + +type WrapFile struct { + f *sevenzip.File +} + +func (f *WrapFile) Name() string { + return f.f.Name +} + +func (f *WrapFile) FileInfo() fs.FileInfo { + return f.f.FileInfo() +} + +func (f *WrapFile) Open() (io.ReadCloser, error) { + return f.f.Open() +} + +func getReader(ss []*stream.SeekableStream, password string) (*sevenzip.Reader, error) { + readerAt, err := stream.NewMultiReaderAt(ss) + if err != nil { + return nil, err + } + sr, err := sevenzip.NewReaderWithPassword(readerAt, readerAt.Size(), password) + if err != nil { + return nil, filterPassword(err) + } + return sr, nil +} + +func filterPassword(err error) error { + if err != nil { + var e *sevenzip.ReadError + if errors.As(err, &e) && e.Encrypted { + return errs.WrongArchivePassword + } + } + return err +} diff --git a/internal/archive/tool/base.go b/internal/archive/tool/base.go index 08e96614f51..8f5b10d96b7 100644 --- a/internal/archive/tool/base.go +++ b/internal/archive/tool/base.go @@ -6,10 +6,16 @@ import ( "io" ) +type MultipartExtension struct { + PartFileFormat string + SecondPartIndex int +} + type Tool interface { AcceptedExtensions() []string - GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) - List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) - Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) - Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error + AcceptedMultipartExtensions() map[string]MultipartExtension + GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) + List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) + Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) + Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error } diff --git a/internal/archive/tool/helper.go b/internal/archive/tool/helper.go new file mode 100644 index 00000000000..8f71900ac56 --- /dev/null +++ b/internal/archive/tool/helper.go @@ -0,0 +1,201 @@ +package tool + +import ( + "io" + "io/fs" + "os" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" +) + +type SubFile interface { + Name() string + FileInfo() fs.FileInfo + Open() (io.ReadCloser, error) +} + +type CanEncryptSubFile interface { + IsEncrypted() bool + SetPassword(password string) +} + +type ArchiveReader interface { + Files() []SubFile +} + +func GenerateMetaTreeFromFolderTraversal(r ArchiveReader) (bool, []model.ObjTree) { + encrypted := false + dirMap := make(map[string]*model.ObjectTree) + dirMap["."] = &model.ObjectTree{} + for _, file := range r.Files() { + if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { + encrypted = true + } + + name := strings.TrimPrefix(file.Name(), "/") + var dir string + var dirObj *model.ObjectTree + isNewFolder := false + if !file.FileInfo().IsDir() { + // 先将 文件 添加到 所在的文件夹 + dir = stdpath.Dir(name) + dirObj = dirMap[dir] + if dirObj == nil { + isNewFolder = true + dirObj = &model.ObjectTree{} + dirObj.IsFolder = true + dirObj.Name = stdpath.Base(dir) + dirObj.Modified = file.FileInfo().ModTime() + dirMap[dir] = dirObj + } + dirObj.Children = append( + dirObj.Children, &model.ObjectTree{ + Object: *MakeModelObj(file.FileInfo()), + }, + ) + } else { + dir = strings.TrimSuffix(name, "/") + dirObj = dirMap[dir] + if dirObj == nil { + isNewFolder = true + dirObj = &model.ObjectTree{} + dirMap[dir] = dirObj + } + dirObj.IsFolder = true + dirObj.Name = stdpath.Base(dir) + dirObj.Modified = file.FileInfo().ModTime() + dirObj.Children = make([]model.ObjTree, 0) + } + if isNewFolder { + // 将 文件夹 添加到 父文件夹 + dir = stdpath.Dir(dir) + pDirObj := dirMap[dir] + if pDirObj != nil { + pDirObj.Children = append(pDirObj.Children, dirObj) + continue + } + + for { + // 考虑压缩包仅记录文件的路径，不记录文件夹 + pDirObj = &model.ObjectTree{} + pDirObj.IsFolder = true + pDirObj.Name = stdpath.Base(dir) + pDirObj.Modified = file.FileInfo().ModTime() + dirMap[dir] = pDirObj + pDirObj.Children = append(pDirObj.Children, dirObj) + dir = stdpath.Dir(dir) + if dirMap[dir] != nil { + break + } + dirObj = pDirObj + } + } + } + return encrypted, dirMap["."].GetChildren() +} + +func MakeModelObj(file os.FileInfo) *model.Object { + return &model.Object{ + Name: file.Name(), + Size: file.Size(), + Modified: file.ModTime(), + IsFolder: file.IsDir(), + } +} + +type WrapFileInfo struct { + model.Obj +} + +func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + var err error + files := r.Files() + if args.InnerPath == "/" { + for i, file := range files { + name := file.Name() + err = decompress(file, name, outputPath, args.Password) + if err != nil { + return err + } + up(float64(i+1) * 100.0 / float64(len(files))) + } + } else { + innerPath := strings.TrimPrefix(args.InnerPath, "/") + innerBase := stdpath.Base(innerPath) + createdBaseDir := false + for _, file := range files { + name := file.Name() + if name == innerPath { + err = _decompress(file, outputPath, args.Password, up) + if err != nil { + return err + } + break + } else if strings.HasPrefix(name, innerPath+"/") { + targetPath := stdpath.Join(outputPath, innerBase) + if !createdBaseDir { + err = os.Mkdir(targetPath, 0700) + if err != nil { + return err + } + createdBaseDir = true + } + restPath := strings.TrimPrefix(name, innerPath+"/") + err = decompress(file, restPath, targetPath, args.Password) + if err != nil { + return err + } + } + } + } + return nil +} + +func decompress(file SubFile, filePath, outputPath, password string) error { + targetPath := outputPath + dir, base := stdpath.Split(filePath) + if dir != "" { + targetPath = stdpath.Join(targetPath, dir) + err := os.MkdirAll(targetPath, 0700) + if err != nil { + return err + } + } + if base != "" { + err := _decompress(file, targetPath, password, func(_ float64) {}) + if err != nil { + return err + } + } + return nil +} + +func _decompress(file SubFile, targetPath, password string, up model.UpdateProgress) error { + if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { + encrypt.SetPassword(password) + } + rc, err := file.Open() + if err != nil { + return err + } + defer func() { _ = rc.Close() }() + f, err := os.OpenFile(stdpath.Join(targetPath, file.FileInfo().Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: rc, + Size: file.FileInfo().Size(), + }, + UpdateProgress: up, + }) + if err != nil { + return err + } + return nil +} diff --git a/internal/archive/tool/utils.go b/internal/archive/tool/utils.go index 822ee894fd1..aa92cb1d792 100644 --- a/internal/archive/tool/utils.go +++ b/internal/archive/tool/utils.go @@ -5,19 +5,28 @@ import ( ) var ( - Tools = make(map[string]Tool) + Tools = make(map[string]Tool) + MultipartExtensions = make(map[string]MultipartExtension) ) func RegisterTool(tool Tool) { for _, ext := range tool.AcceptedExtensions() { Tools[ext] = tool } + for mainFile, ext := range tool.AcceptedMultipartExtensions() { + MultipartExtensions[mainFile] = ext + Tools[mainFile] = tool + } } -func GetArchiveTool(ext string) (Tool, error) { +func GetArchiveTool(ext string) (*MultipartExtension, Tool, error) { t, ok := Tools[ext] if !ok { - return nil, errs.UnknownArchiveFormat + return nil, nil, errs.UnknownArchiveFormat + } + partExt, ok := MultipartExtensions[ext] + if !ok { + return nil, t, nil } - return t, nil + return &partExt, t, nil } diff --git a/internal/archive/zip/utils.go b/internal/archive/zip/utils.go index aa51b88eb93..59f4ed51378 100644 --- a/internal/archive/zip/utils.go +++ b/internal/archive/zip/utils.go @@ -2,8 +2,13 @@ package zip import ( "bytes" + "io" + "io/fs" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" "github.com/saintfish/chardet" "github.com/yeka/zip" @@ -16,65 +21,62 @@ import ( "golang.org/x/text/encoding/unicode" "golang.org/x/text/encoding/unicode/utf32" "golang.org/x/text/transform" - "io" - "os" - stdpath "path" - "strings" ) -func toModelObj(file os.FileInfo) *model.Object { - return &model.Object{ - Name: decodeName(file.Name()), - Size: file.Size(), - Modified: file.ModTime(), - IsFolder: file.IsDir(), - } +type WrapReader struct { + Reader *zip.Reader } -func decompress(file *zip.File, filePath, outputPath, password string) error { - targetPath := outputPath - dir, base := stdpath.Split(filePath) - if dir != "" { - targetPath = stdpath.Join(targetPath, dir) - err := os.MkdirAll(targetPath, 0700) - if err != nil { - return err - } - } - if base != "" { - err := _decompress(file, targetPath, password, func(_ float64) {}) - if err != nil { - return err - } +func (r *WrapReader) Files() []tool.SubFile { + ret := make([]tool.SubFile, 0, len(r.Reader.File)) + for _, f := range r.Reader.File { + ret = append(ret, &WrapFile{f: f}) } - return nil + return ret } -func _decompress(file *zip.File, targetPath, password string, up model.UpdateProgress) error { - if file.IsEncrypted() { - file.SetPassword(password) - } - rc, err := file.Open() - if err != nil { - return err - } - defer rc.Close() - f, err := os.OpenFile(stdpath.Join(targetPath, decodeName(file.FileInfo().Name())), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) - if err != nil { - return err +type WrapFileInfo struct { + fs.FileInfo +} + +func (f *WrapFileInfo) Name() string { + return decodeName(f.FileInfo.Name()) +} + +type WrapFile struct { + f *zip.File +} + +func (f *WrapFile) Name() string { + return decodeName(f.f.Name) +} + +func (f *WrapFile) FileInfo() fs.FileInfo { + return &WrapFileInfo{FileInfo: f.f.FileInfo()} +} + +func (f *WrapFile) Open() (io.ReadCloser, error) { + return f.f.Open() +} + +func (f *WrapFile) IsEncrypted() bool { + return f.f.IsEncrypted() +} + +func (f *WrapFile) SetPassword(password string) { + f.f.SetPassword(password) +} + +func getReader(ss []*stream.SeekableStream) (*zip.Reader, error) { + if len(ss) > 1 && stdpath.Ext(ss[1].GetName()) == ".z01" { + // FIXME: Incorrect parsing method for standard multipart zip format + ss = append(ss[1:], ss[0]) } - defer f.Close() - _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ - Reader: &stream.SimpleReaderWithSize{ - Reader: rc, - Size: file.FileInfo().Size(), - }, - UpdateProgress: up, - }) + reader, err := stream.NewMultiReaderAt(ss) if err != nil { - return err + return nil, err } - return nil + return zip.NewReader(reader, reader.Size()) } func filterPassword(err error) error { diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go index 9dc8cc7638f..6e23570c73c 100644 --- a/internal/archive/zip/zip.go +++ b/internal/archive/zip/zip.go @@ -2,7 +2,6 @@ package zip import ( "io" - "os" stdpath "path" "strings" @@ -10,106 +9,37 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" - "github.com/yeka/zip" ) type Zip struct { } -func (*Zip) AcceptedExtensions() []string { - return []string{".zip"} +func (Zip) AcceptedExtensions() []string { + return []string{} } -func (*Zip) GetMeta(ss *stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { - reader, err := stream.NewReadAtSeeker(ss, 0) - if err != nil { - return nil, err +func (Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{ + ".zip": {".z%.2d", 1}, + ".zip.001": {".zip.%.3d", 2}, } - zipReader, err := zip.NewReader(reader, ss.GetSize()) +} + +func (Zip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + zipReader, err := getReader(ss) if err != nil { return nil, err } - encrypted := false - dirMap := make(map[string]*model.ObjectTree) - dirMap["."] = &model.ObjectTree{} - for _, file := range zipReader.File { - if file.IsEncrypted() { - encrypted = true - } - - name := strings.TrimPrefix(decodeName(file.Name), "/") - var dir string - var dirObj *model.ObjectTree - isNewFolder := false - if !file.FileInfo().IsDir() { - // 先将 文件 添加到 所在的文件夹 - dir = stdpath.Dir(name) - dirObj = dirMap[dir] - if dirObj == nil { - isNewFolder = true - dirObj = &model.ObjectTree{} - dirObj.IsFolder = true - dirObj.Name = stdpath.Base(dir) - dirObj.Modified = file.ModTime() - dirMap[dir] = dirObj - } - dirObj.Children = append( - dirObj.Children, &model.ObjectTree{ - Object: *toModelObj(file.FileInfo()), - }, - ) - } else { - dir = strings.TrimSuffix(name, "/") - dirObj = dirMap[dir] - if dirObj == nil { - isNewFolder = true - dirObj = &model.ObjectTree{} - dirMap[dir] = dirObj - } - dirObj.IsFolder = true - dirObj.Name = stdpath.Base(dir) - dirObj.Modified = file.ModTime() - dirObj.Children = make([]model.ObjTree, 0) - } - if isNewFolder { - // 将 文件夹 添加到 父文件夹 - dir = stdpath.Dir(dir) - pDirObj := dirMap[dir] - if pDirObj != nil { - pDirObj.Children = append(pDirObj.Children, dirObj) - continue - } - - for { - // 考虑压缩包仅记录文件的路径，不记录文件夹 - pDirObj = &model.ObjectTree{} - pDirObj.IsFolder = true - pDirObj.Name = stdpath.Base(dir) - pDirObj.Modified = file.ModTime() - dirMap[dir] = pDirObj - pDirObj.Children = append(pDirObj.Children, dirObj) - dir = stdpath.Dir(dir) - if dirMap[dir] != nil { - break - } - dirObj = pDirObj - } - } - } - + encrypted, tree := tool.GenerateMetaTreeFromFolderTraversal(&WrapReader{Reader: zipReader}) return &model.ArchiveMetaInfo{ Comment: zipReader.Comment, Encrypted: encrypted, - Tree: dirMap["."].GetChildren(), + Tree: tree, }, nil } -func (*Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { - reader, err := stream.NewReadAtSeeker(ss, 0) - if err != nil { - return nil, err - } - zipReader, err := zip.NewReader(reader, ss.GetSize()) +func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + zipReader, err := getReader(ss) if err != nil { return nil, err } @@ -134,13 +64,13 @@ func (*Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]mode if dir == nil && len(strs) == 2 { dir = &model.Object{ Name: strs[0], - Modified: ss.ModTime(), + Modified: ss[0].ModTime(), IsFolder: true, } } continue } - ret = append(ret, toModelObj(file.FileInfo())) + ret = append(ret, tool.MakeModelObj(&WrapFileInfo{FileInfo: file.FileInfo()})) } if len(ret) == 0 && dir != nil { ret = append(ret, dir) @@ -157,7 +87,7 @@ func (*Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]mode continue } exist = true - ret = append(ret, toModelObj(file.FileInfo())) + ret = append(ret, tool.MakeModelObj(&WrapFileInfo{file.FileInfo()})) } if !exist { return nil, errs.ObjectNotFound @@ -166,12 +96,8 @@ func (*Zip) List(ss *stream.SeekableStream, args model.ArchiveInnerArgs) ([]mode } } -func (*Zip) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { - reader, err := stream.NewReadAtSeeker(ss, 0) - if err != nil { - return nil, 0, err - } - zipReader, err := zip.NewReader(reader, ss.GetSize()) +func (Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + zipReader, err := getReader(ss) if err != nil { return nil, 0, err } @@ -191,58 +117,16 @@ func (*Zip) Extract(ss *stream.SeekableStream, args model.ArchiveInnerArgs) (io. return nil, 0, errs.ObjectNotFound } -func (*Zip) Decompress(ss *stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { - reader, err := stream.NewReadAtSeeker(ss, 0) - if err != nil { - return err - } - zipReader, err := zip.NewReader(reader, ss.GetSize()) +func (Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + zipReader, err := getReader(ss) if err != nil { return err } - if args.InnerPath == "/" { - for i, file := range zipReader.File { - name := decodeName(file.Name) - err = decompress(file, name, outputPath, args.Password) - if err != nil { - return err - } - up(float64(i+1) * 100.0 / float64(len(zipReader.File))) - } - } else { - innerPath := strings.TrimPrefix(args.InnerPath, "/") - innerBase := stdpath.Base(innerPath) - createdBaseDir := false - for _, file := range zipReader.File { - name := decodeName(file.Name) - if name == innerPath { - err = _decompress(file, outputPath, args.Password, up) - if err != nil { - return err - } - break - } else if strings.HasPrefix(name, innerPath+"/") { - targetPath := stdpath.Join(outputPath, innerBase) - if !createdBaseDir { - err = os.Mkdir(targetPath, 0700) - if err != nil { - return err - } - createdBaseDir = true - } - restPath := strings.TrimPrefix(name, innerPath+"/") - err = decompress(file, restPath, targetPath, args.Password) - if err != nil { - return err - } - } - } - } - return nil + return tool.DecompressFromFolderTraversal(&WrapReader{Reader: zipReader}, outputPath, args, up) } var _ tool.Tool = (*Zip)(nil) func init() { - tool.RegisterTool(&Zip{}) + tool.RegisterTool(Zip{}) } diff --git a/internal/driver/driver.go b/internal/driver/driver.go index 05f0fe24576..9e9440b6700 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -79,13 +79,13 @@ type Remove interface { type Put interface { // Put a file (provided as a FileStreamer) into the driver // Besides the most basic upload functionality, the following features also need to be implemented: - // 1. Canceling (when `<-ctx.Done()` returns), by the following methods: + // 1. Canceling (when `<-ctx.Done()` returns), which can be supported by the following methods: // (1) Use request methods that carry context, such as the following: // a. http.NewRequestWithContext // b. resty.Request.SetContext // c. s3manager.Uploader.UploadWithContext // d. utils.CopyWithCtx - // (2) Use a `driver.ReaderWithCtx` or a `driver.NewLimitedUploadStream` + // (2) Use a `driver.ReaderWithCtx` or `driver.NewLimitedUploadStream` // (3) Use `utils.IsCanceled` to check if the upload has been canceled during the upload process, // this is typically applicable to chunked uploads. // 2. Submit upload progress (via `up`) in real-time. There are three recommended ways as follows: diff --git a/internal/fs/archive.go b/internal/fs/archive.go index 3913182702c..b056decf9a2 100644 --- a/internal/fs/archive.go +++ b/internal/fs/archive.go @@ -4,17 +4,6 @@ import ( "context" stderrors "errors" "fmt" - "github.com/alist-org/alist/v3/internal/archive/tool" - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/internal/task" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/xhofe/tache" "io" "math/rand" "mime" @@ -25,6 +14,17 @@ import ( "strconv" "strings" "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" ) type ArchiveDownloadTask struct { @@ -37,7 +37,6 @@ type ArchiveDownloadTask struct { dstStorage driver.Driver SrcStorageMp string DstStorageMp string - Tool tool.Tool } func (t *ArchiveDownloadTask) GetName() string { @@ -67,33 +66,39 @@ func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadT if t.srcStorage == nil { t.srcStorage, err = op.GetStorageByMountPath(t.SrcStorageMp) } - l, srcObj, err := op.Link(t.Ctx(), t.srcStorage, t.SrcObjPath, model.LinkArgs{ + srcObj, tool, ss, err := op.GetArchiveToolAndStream(t.Ctx(), t.srcStorage, t.SrcObjPath, model.LinkArgs{ Header: http.Header{}, }) if err != nil { return nil, err } - fs := stream.FileStream{ - Obj: srcObj, - Ctx: t.Ctx(), - } - ss, err := stream.NewSeekableStream(fs, l) - if err != nil { - return nil, err - } defer func() { - if err := ss.Close(); err != nil { - log.Errorf("failed to close file streamer, %v", err) + var e error + for _, s := range ss { + e = stderrors.Join(e, s.Close()) + } + if e != nil { + log.Errorf("failed to close file streamer, %v", e) } }() var decompressUp model.UpdateProgress if t.CacheFull { - t.SetTotalBytes(srcObj.GetSize()) + var total, cur int64 = 0, 0 + for _, s := range ss { + total += s.GetSize() + } + t.SetTotalBytes(total) t.status = "getting src object" - _, err = ss.CacheFullInTempFileAndUpdateProgress(t.SetProgress) - if err != nil { - return nil, err + for _, s := range ss { + _, err = s.CacheFullInTempFileAndUpdateProgress(func(p float64) { + t.SetProgress((float64(cur) + float64(s.GetSize())*p/100.0) / float64(total)) + }) + cur += s.GetSize() + if err != nil { + return nil, err + } } + t.SetProgress(100.0) decompressUp = func(_ float64) {} } else { decompressUp = t.SetProgress @@ -103,7 +108,7 @@ func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadT if err != nil { return nil, err } - err = t.Tool.Decompress(ss, dir, t.ArchiveInnerArgs, decompressUp) + err = tool.Decompress(ss, dir, t.ArchiveInnerArgs, decompressUp) if err != nil { return nil, err } @@ -344,11 +349,6 @@ func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args return nil, err } } - ext := stdpath.Ext(srcObjActualPath) - t, err := tool.GetArchiveTool(ext) - if err != nil { - return nil, errors.WithMessagef(err, "failed get [%s] archive tool", ext) - } taskCreator, _ := ctx.Value("user").(*model.User) tsk := &ArchiveDownloadTask{ TaskExtension: task.TaskExtension{ @@ -361,7 +361,6 @@ func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args DstDirPath: dstDirActualPath, SrcStorageMp: srcStorage.GetStorage().MountPath, DstStorageMp: dstStorage.GetStorage().MountPath, - Tool: t, } if ctx.Value(conf.NoTaskKey) != nil { uploadTask, err := tsk.RunWithoutPushUploadTask() diff --git a/internal/op/archive.go b/internal/op/archive.go index 4015e299cb6..38b870c70e3 100644 --- a/internal/op/archive.go +++ b/internal/op/archive.go @@ -3,6 +3,7 @@ package op import ( "context" stderrors "errors" + "fmt" "io" stdpath "path" "strings" @@ -54,21 +55,76 @@ func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg return meta, err } -func getArchiveToolAndStream(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (model.Obj, tool.Tool, *stream.SeekableStream, error) { +func GetArchiveToolAndStream(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (model.Obj, tool.Tool, []*stream.SeekableStream, error) { l, obj, err := Link(ctx, storage, path, args) if err != nil { return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] link", path) } - ext := stdpath.Ext(obj.GetName()) - t, err := tool.GetArchiveTool(ext) + baseName, ext, found := strings.Cut(obj.GetName(), ".") + if !found { + if l.MFile != nil { + _ = l.MFile.Close() + } + if l.RangeReadCloser != nil { + _ = l.RangeReadCloser.Close() + } + return nil, nil, nil, errors.Errorf("failed get archive tool: the obj does not have an extension.") + } + partExt, t, err := tool.GetArchiveTool("." + ext) if err != nil { - return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] archive tool", ext) + var e error + partExt, t, e = tool.GetArchiveTool(stdpath.Ext(obj.GetName())) + if e != nil { + if l.MFile != nil { + _ = l.MFile.Close() + } + if l.RangeReadCloser != nil { + _ = l.RangeReadCloser.Close() + } + return nil, nil, nil, errors.WithMessagef(stderrors.Join(err, e), "failed get archive tool: %s", ext) + } } ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, l) if err != nil { + if l.MFile != nil { + _ = l.MFile.Close() + } + if l.RangeReadCloser != nil { + _ = l.RangeReadCloser.Close() + } return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) } - return obj, t, ss, nil + ret := []*stream.SeekableStream{ss} + if partExt == nil { + return obj, t, ret, nil + } else { + index := partExt.SecondPartIndex + dir := stdpath.Dir(path) + for { + p := stdpath.Join(dir, baseName+fmt.Sprintf(partExt.PartFileFormat, index)) + var o model.Obj + l, o, err = Link(ctx, storage, p, args) + if err != nil { + break + } + ss, err = stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: o}, l) + if err != nil { + if l.MFile != nil { + _ = l.MFile.Close() + } + if l.RangeReadCloser != nil { + _ = l.RangeReadCloser.Close() + } + for _, s := range ret { + _ = s.Close() + } + return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) + } + ret = append(ret, ss) + index++ + } + return obj, t, ret, nil + } } func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (model.Obj, *model.ArchiveMetaProvider, error) { @@ -94,13 +150,17 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg return obj, archiveMetaProvider, err } } - obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + obj, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, nil, err } defer func() { - if err := ss.Close(); err != nil { - log.Errorf("failed to close file streamer, %v", err) + var e error + for _, s := range ss { + e = stderrors.Join(e, s.Close()) + } + if e != nil { + log.Errorf("failed to close file streamer, %v", e) } }() meta, err := t.GetMeta(ss, args.ArchiveArgs) @@ -114,9 +174,9 @@ func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, arg if !storage.Config().NoCache { Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) archiveMetaProvider.Expiration = &Expiration - } else if ss.Link.MFile == nil { + } else if ss[0].Link.MFile == nil { // alias、crypt 驱动 - archiveMetaProvider.Expiration = ss.Link.Expiration + archiveMetaProvider.Expiration = ss[0].Link.Expiration } return obj, archiveMetaProvider, err } @@ -188,13 +248,17 @@ func _listArchive(ctx context.Context, storage driver.Driver, path string, args return obj, files, err } } - obj, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + obj, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, nil, err } defer func() { - if err := ss.Close(); err != nil { - log.Errorf("failed to close file streamer, %v", err) + var e error + for _, s := range ss { + e = stderrors.Join(e, s.Close()) + } + if e != nil { + log.Errorf("failed to close file streamer, %v", e) } }() files, err := t.List(ss, args.ArchiveInnerArgs) @@ -378,8 +442,8 @@ func driverExtract(ctx context.Context, storage driver.Driver, path string, args } type streamWithParent struct { - rc io.ReadCloser - parent *stream.SeekableStream + rc io.ReadCloser + parents []*stream.SeekableStream } func (s *streamWithParent) Read(p []byte) (int, error) { @@ -387,24 +451,31 @@ func (s *streamWithParent) Read(p []byte) (int, error) { } func (s *streamWithParent) Close() error { - err1 := s.rc.Close() - err2 := s.parent.Close() - return stderrors.Join(err1, err2) + err := s.rc.Close() + for _, ss := range s.parents { + err = stderrors.Join(err, ss.Close()) + } + return err } func InternalExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { - _, t, ss, err := getArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + _, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) if err != nil { return nil, 0, err } rc, size, err := t.Extract(ss, args) if err != nil { - if e := ss.Close(); e != nil { + var e error + for _, s := range ss { + e = stderrors.Join(e, s.Close()) + } + if e != nil { log.Errorf("failed to close file streamer, %v", e) + err = stderrors.Join(err, e) } return nil, 0, err } - return &streamWithParent{rc: rc, parent: ss}, size, nil + return &streamWithParent{rc: rc, parents: ss}, size, nil } func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) error { diff --git a/internal/stream/limit.go b/internal/stream/limit.go index 3b32a55ff6d..14d0efd0f3e 100644 --- a/internal/stream/limit.go +++ b/internal/stream/limit.go @@ -139,7 +139,7 @@ type RateLimitRangeReadCloser struct { Limiter Limiter } -func (rrc RateLimitRangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { +func (rrc *RateLimitRangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { rc, err := rrc.RangeReadCloserIF.RangeRead(ctx, httpRange) if err != nil { return nil, err diff --git a/internal/stream/stream.go b/internal/stream/stream.go index 1c94715f95a..f6b045a0238 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -14,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/sirupsen/logrus" + "go4.org/readerutil" ) type FileStream struct { @@ -159,6 +160,10 @@ var _ model.FileStreamer = (*FileStream)(nil) //var _ seekableStream = (*FileStream)(nil) // for most internal stream, which is either RangeReadCloser or MFile +// Any functionality implemented based on SeekableStream should implement a Close method, +// whose only purpose is to close the SeekableStream object. If such functionality has +// additional resources that need to be closed, they should be added to the Closer property of +// the SeekableStream object and be closed together when the SeekableStream object is closed. type SeekableStream struct { FileStream Link *model.Link @@ -196,7 +201,7 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) return &ss, nil } if ss.Link.RangeReadCloser != nil { - ss.rangeReadCloser = RateLimitRangeReadCloser{ + ss.rangeReadCloser = &RateLimitRangeReadCloser{ RangeReadCloserIF: ss.Link.RangeReadCloser, Limiter: ServerDownloadLimit, } @@ -208,7 +213,7 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) if err != nil { return nil, err } - rrc = RateLimitRangeReadCloser{ + rrc = &RateLimitRangeReadCloser{ RangeReadCloserIF: rrc, Limiter: ServerDownloadLimit, } @@ -364,7 +369,7 @@ type RangeReadReadAtSeeker struct { ss *SeekableStream masterOff int64 readers []*readerCur - *headCache + headCache *headCache } type headCache struct { @@ -406,7 +411,7 @@ func (c *headCache) read(p []byte) (n int, err error) { } return } -func (r *headCache) close() error { +func (r *headCache) Close() error { for i := range r.bufs { r.bufs[i] = nil } @@ -419,6 +424,7 @@ func (r *RangeReadReadAtSeeker) InitHeadCache() { reader := r.readers[0] r.readers = r.readers[1:] r.headCache = &headCache{readerCur: reader} + r.ss.Closers.Add(r.headCache) } } @@ -449,6 +455,18 @@ func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (SStr return r, nil } +func NewMultiReaderAt(ss []*SeekableStream) (readerutil.SizeReaderAt, error) { + readers := make([]readerutil.SizeReaderAt, 0, len(ss)) + for _, s := range ss { + ra, err := NewReadAtSeeker(s, 0) + if err != nil { + return nil, err + } + readers = append(readers, io.NewSectionReader(ra, 0, s.GetSize())) + } + return readerutil.NewMultiReaderAt(readers...), nil +} + func (r *RangeReadReadAtSeeker) GetRawStream() *SeekableStream { return r.ss } @@ -559,9 +577,6 @@ func (r *RangeReadReadAtSeeker) Read(p []byte) (n int, err error) { } func (r *RangeReadReadAtSeeker) Close() error { - if r.headCache != nil { - _ = r.headCache.close() - } return r.ss.Close() } diff --git a/server/handles/archive.go b/server/handles/archive.go index 4ec933e17b0..550bc3cec9c 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -1,10 +1,11 @@ package handles import ( + "encoding/json" "fmt" + "github.com/alist-org/alist/v3/internal/task" "net/url" stdpath "path" - "strings" "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/conf" @@ -208,14 +209,30 @@ func FsArchiveList(c *gin.Context) { }) } +type StringOrArray []string + +func (s *StringOrArray) UnmarshalJSON(data []byte) error { + var value string + if err := json.Unmarshal(data, &value); err == nil { + *s = []string{value} + return nil + } + var sliceValue []string + if err := json.Unmarshal(data, &sliceValue); err != nil { + return err + } + *s = sliceValue + return nil +} + type ArchiveDecompressReq struct { - SrcDir string `json:"src_dir" form:"src_dir"` - DstDir string `json:"dst_dir" form:"dst_dir"` - Name string `json:"name" form:"name"` - ArchivePass string `json:"archive_pass" form:"archive_pass"` - InnerPath string `json:"inner_path" form:"inner_path"` - CacheFull bool `json:"cache_full" form:"cache_full"` - PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` + SrcDir string `json:"src_dir" form:"src_dir"` + DstDir string `json:"dst_dir" form:"dst_dir"` + Name StringOrArray `json:"name" form:"name"` + ArchivePass string `json:"archive_pass" form:"archive_pass"` + InnerPath string `json:"inner_path" form:"inner_path"` + CacheFull bool `json:"cache_full" form:"cache_full"` + PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` } func FsArchiveDecompress(c *gin.Context) { @@ -229,41 +246,51 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, req.Name)) - if err != nil { - common.ErrorResp(c, err, 403) - return + srcPaths := make([]string, 0, len(req.Name)) + for _, name := range req.Name { + srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + srcPaths = append(srcPaths, srcPath) } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - t, err := fs.ArchiveDecompress(c, srcPath, dstDir, model.ArchiveDecompressArgs{ - ArchiveInnerArgs: model.ArchiveInnerArgs{ - ArchiveArgs: model.ArchiveArgs{ - LinkArgs: model.LinkArgs{ - Header: c.Request.Header, - Type: c.Query("type"), - HttpReq: c.Request, + tasks := make([]task.TaskExtensionInfo, 0, len(srcPaths)) + for _, srcPath := range srcPaths { + t, e := fs.ArchiveDecompress(c, srcPath, dstDir, model.ArchiveDecompressArgs{ + ArchiveInnerArgs: model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: req.ArchivePass, }, - Password: req.ArchivePass, + InnerPath: utils.FixAndCleanPath(req.InnerPath), }, - InnerPath: utils.FixAndCleanPath(req.InnerPath), - }, - CacheFull: req.CacheFull, - PutIntoNewDir: req.PutIntoNewDir, - }) - if err != nil { - if errors.Is(err, errs.WrongArchivePassword) { - common.ErrorResp(c, err, 202) - } else { - common.ErrorResp(c, err, 500) + CacheFull: req.CacheFull, + PutIntoNewDir: req.PutIntoNewDir, + }) + if e != nil { + if errors.Is(e, errs.WrongArchivePassword) { + common.ErrorResp(c, e, 202) + } else { + common.ErrorResp(c, e, 500) + } + return + } + if t != nil { + tasks = append(tasks, t) } - return } common.SuccessResp(c, gin.H{ - "task": getTaskInfo(t), + "task": getTaskInfos(tasks), }) } @@ -376,7 +403,7 @@ func ArchiveInternalExtract(c *gin.Context) { func ArchiveExtensions(c *gin.Context) { var ext []string for key := range tool.Tools { - ext = append(ext, strings.TrimPrefix(key, ".")) + ext = append(ext, key) } common.SuccessResp(c, ext) } From 5668e4a4ea005690105bf174b6c26b4dec7bb5c4 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 27 Mar 2025 23:21:42 +0800 Subject: [PATCH 480/659] feat(doubao): add Doubao driver (#8232 closes #8020 #8206) * feat(doubao): implement List() * feat(doubao): implement Link() * feat(doubao): implement MakeDir() * refactor(doubao): add type Object to store key * feat(doubao): implement Move() * feat(doubao): implement Rename() * feat(doubao): implement Remove() --- drivers/all.go | 1 + drivers/doubao/driver.go | 174 +++++++++++++++++++++++++++++++++++++++ drivers/doubao/meta.go | 34 ++++++++ drivers/doubao/types.go | 64 ++++++++++++++ drivers/doubao/util.go | 38 +++++++++ 5 files changed, 311 insertions(+) create mode 100644 drivers/doubao/driver.go create mode 100644 drivers/doubao/meta.go create mode 100644 drivers/doubao/types.go create mode 100644 drivers/doubao/util.go diff --git a/drivers/all.go b/drivers/all.go index 963f0c4453c..a14e80fbc59 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -22,6 +22,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/crypt" + _ "github.com/alist-org/alist/v3/drivers/doubao" _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" diff --git a/drivers/doubao/driver.go b/drivers/doubao/driver.go new file mode 100644 index 00000000000..b847ffa90b4 --- /dev/null +++ b/drivers/doubao/driver.go @@ -0,0 +1,174 @@ +package doubao + +import ( + "context" + "errors" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" +) + +type Doubao struct { + model.Storage + Addition +} + +func (d *Doubao) Config() driver.Config { + return config +} + +func (d *Doubao) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Doubao) Init(ctx context.Context) error { + // TODO login / refresh token + //op.MustSaveDriverStorage(d) + return nil +} + +func (d *Doubao) Drop(ctx context.Context) error { + return nil +} + +func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var files []model.Obj + var r NodeInfoResp + _, err := d.request("/samantha/aispace/node_info", "POST", func(req *resty.Request) { + req.SetBody(base.Json{ + "node_id": dir.GetID(), + "need_full_path": false, + }) + }, &r) + if err != nil { + return nil, err + } + + for _, child := range r.Data.Children { + files = append(files, &Object{ + Object: model.Object{ + ID: child.ID, + Path: child.ParentID, + Name: child.Name, + Size: int64(child.Size), + Modified: time.Unix(int64(child.UpdateTime), 0), + Ctime: time.Unix(int64(child.CreateTime), 0), + IsFolder: child.NodeType == 1, + }, + Key: child.Key, + }) + } + return files, nil +} + +func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if u, ok := file.(*Object); ok { + var r GetFileUrlResp + _, err := d.request("/alice/message/get_file_url", "POST", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{u.Key}, + "type": "file", + }) + }, &r) + if err != nil { + return nil, err + } + return &model.Link{ + URL: r.Data.FileUrls[0].MainURL, + }, nil + } + return nil, errors.New("can't convert obj to URL") +} + +func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + var r UploadNodeResp + _, err := d.request("/samantha/aispace/upload_node", "POST", func(req *resty.Request) { + req.SetBody(base.Json{ + "node_list": []base.Json{ + { + "local_id": uuid.New().String(), + "name": dirName, + "parent_id": parentDir.GetID(), + "node_type": 1, + }, + }, + }) + }, &r) + return err +} + +func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + var r UploadNodeResp + _, err := d.request("/samantha/aispace/move_node", "POST", func(req *resty.Request) { + req.SetBody(base.Json{ + "node_list": []base.Json{ + {"id": srcObj.GetID()}, + }, + "current_parent_id": srcObj.GetPath(), + "target_parent_id": dstDir.GetID(), + }) + }, &r) + return err +} + +func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + var r BaseResp + _, err := d.request("/samantha/aispace/rename_node", "POST", func(req *resty.Request) { + req.SetBody(base.Json{ + "node_id": srcObj.GetID(), + "node_name": newName, + }) + }, &r) + return err +} + +func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error { + var r BaseResp + _, err := d.request("/samantha/aispace/delete_node", "POST", func(req *resty.Request) { + req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}}) + }, &r) + return err +} + +func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // TODO upload file, optional + return nil, errs.NotImplement +} + +func (d *Doubao) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Doubao) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Doubao) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Doubao) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *Doubao) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Doubao)(nil) diff --git a/drivers/doubao/meta.go b/drivers/doubao/meta.go new file mode 100644 index 00000000000..bb9e3f258d0 --- /dev/null +++ b/drivers/doubao/meta.go @@ -0,0 +1,34 @@ +package doubao + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + // driver.RootPath + driver.RootID + // define other + Cookie string `json:"cookie" type:"text"` +} + +var config = driver.Config{ + Name: "Doubao", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: true, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Doubao{} + }) +} diff --git a/drivers/doubao/types.go b/drivers/doubao/types.go new file mode 100644 index 00000000000..f9611d86ced --- /dev/null +++ b/drivers/doubao/types.go @@ -0,0 +1,64 @@ +package doubao + +import "github.com/alist-org/alist/v3/internal/model" + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +type NodeInfoResp struct { + BaseResp + Data struct { + NodeInfo NodeInfo `json:"node_info"` + Children []NodeInfo `json:"children"` + NextCursor string `json:"next_cursor"` + HasMore bool `json:"has_more"` + } `json:"data"` +} + +type NodeInfo struct { + ID string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` // 0: 文件, 1: 文件夹 + Size int `json:"size"` + Source int `json:"source"` + NameReviewStatus int `json:"name_review_status"` + ContentReviewStatus int `json:"content_review_status"` + RiskReviewStatus int `json:"risk_review_status"` + ConversationID string `json:"conversation_id"` + ParentID string `json:"parent_id"` + CreateTime int `json:"create_time"` + UpdateTime int `json:"update_time"` +} + +type GetFileUrlResp struct { + BaseResp + Data struct { + FileUrls []struct { + URI string `json:"uri"` + MainURL string `json:"main_url"` + BackURL string `json:"back_url"` + } `json:"file_urls"` + } `json:"data"` +} + +type UploadNodeResp struct { + BaseResp + Data struct { + NodeList []struct { + LocalID string `json:"local_id"` + ID string `json:"id"` + ParentID string `json:"parent_id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` // 0: 文件, 1: 文件夹 + } `json:"node_list"` + } `json:"data"` +} + +type Object struct { + model.Object + Key string +} diff --git a/drivers/doubao/util.go b/drivers/doubao/util.go new file mode 100644 index 00000000000..977691c03a6 --- /dev/null +++ b/drivers/doubao/util.go @@ -0,0 +1,38 @@ +package doubao + +import ( + "errors" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + log "github.com/sirupsen/logrus" +) + +// do others that not defined in Driver interface +func (d *Doubao) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + url := "https://www.doubao.com" + path + req := base.RestyClient.R() + req.SetHeader("Cookie", d.Cookie) + if callback != nil { + callback(req) + } + var r BaseResp + req.SetResult(&r) + res, err := req.Execute(method, url) + log.Debugln(res.String()) + if err != nil { + return nil, err + } + + // 业务状态码检查（优先于HTTP状态码） + if r.Code != 0 { + return res.Body(), errors.New(r.Msg) + } + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + return nil, err + } + } + return res.Body(), nil +} From c38dc6df7c9defa23b2aa6826c24ccc43cdc94f6 Mon Sep 17 00:00:00 2001 From: never lee Date: Thu, 27 Mar 2025 23:22:08 +0800 Subject: [PATCH 481/659] fix(115_open): support multipart upload (#8229) Co-authored-by: neverlee --- drivers/115_open/driver.go | 15 +--- drivers/115_open/upload.go | 140 +++++++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 14 deletions(-) create mode 100644 drivers/115_open/upload.go diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 00337c0b920..0eb943ac067 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -2,7 +2,6 @@ package _115_open import ( "context" - "encoding/base64" "fmt" "io" "net/http" @@ -16,7 +15,6 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" sdk "github.com/xhofe/115-sdk-go" ) @@ -265,18 +263,7 @@ func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStre return err } // 4. upload - ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) - if err != nil { - return err - } - bucket, err := ossClient.Bucket(resp.Bucket) - if err != nil { - return err - } - err = bucket.PutObject(resp.Object, tempF, - oss.Callback(base64.StdEncoding.EncodeToString([]byte(resp.Callback.Value.Callback))), - oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(resp.Callback.Value.CallbackVar))), - ) + err = d.multpartUpload(ctx, tempF, file, up, tokenResp, resp) if err != nil { return err } diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go new file mode 100644 index 00000000000..282582eff12 --- /dev/null +++ b/drivers/115_open/upload.go @@ -0,0 +1,140 @@ +package _115_open + +import ( + "context" + "encoding/base64" + "io" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/avast/retry-go" + sdk "github.com/xhofe/115-sdk-go" +) + +func calPartSize(fileSize int64) int64 { + var partSize int64 = 20 * utils.MB + if fileSize > partSize { + if fileSize > 1*utils.TB { // file Size over 1TB + partSize = 5 * utils.GB // file part size 5GB + } else if fileSize > 768*utils.GB { // over 768GB + partSize = 109951163 // ≈ 104.8576MB, split 1TB into 10,000 part + } else if fileSize > 512*utils.GB { // over 512GB + partSize = 82463373 // ≈ 78.6432MB + } else if fileSize > 384*utils.GB { // over 384GB + partSize = 54975582 // ≈ 52.4288MB + } else if fileSize > 256*utils.GB { // over 256GB + partSize = 41231687 // ≈ 39.3216MB + } else if fileSize > 128*utils.GB { // over 128GB + partSize = 27487791 // ≈ 26.2144MB + } + } + return partSize +} + +func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { + ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) + if err != nil { + return err + } + bucket, err := ossClient.Bucket(initResp.Bucket) + if err != nil { + return err + } + + err = bucket.PutObject(initResp.Object, tempF, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), + ) + + return err +} + +// type CallbackResult struct { +// State bool `json:"state"` +// Code int `json:"code"` +// Message string `json:"message"` +// Data struct { +// PickCode string `json:"pick_code"` +// FileName string `json:"file_name"` +// FileSize int64 `json:"file_size"` +// FileID string `json:"file_id"` +// ThumbURL string `json:"thumb_url"` +// Sha1 string `json:"sha1"` +// Aid int `json:"aid"` +// Cid string `json:"cid"` +// } `json:"data"` +// } + +func (d *Open115) multpartUpload(ctx context.Context, tempF model.File, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { + fileSize := stream.GetSize() + chunkSize := calPartSize(fileSize) + + ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) + if err != nil { + return err + } + bucket, err := ossClient.Bucket(initResp.Bucket) + if err != nil { + return err + } + + imur, err := bucket.InitiateMultipartUpload(initResp.Object, oss.Sequential()) + if err != nil { + return err + } + + partNum := (stream.GetSize() + chunkSize - 1) / chunkSize + parts := make([]oss.UploadPart, partNum) + offset := int64(0) + for i := int64(1); i <= partNum; i++ { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + + partSize := chunkSize + if i == partNum { + partSize = fileSize - (i-1)*chunkSize + } + rd := utils.NewMultiReadable(io.LimitReader(stream, partSize)) + err = retry.Do(func() error { + _ = rd.Reset() + rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd) + part, err := bucket.UploadPart(imur, rateLimitedRd, partSize, int(i)) + if err != nil { + return err + } + parts[i-1] = part + return nil + }, + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second)) + if err != nil { + return err + } + + if i == partNum { + offset = fileSize + } else { + offset += partSize + } + up(float64(offset) / float64(fileSize)) + } + + // callbackRespBytes := make([]byte, 1024) + _, err = bucket.CompleteMultipartUpload( + imur, + parts, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), + // oss.CallbackResult(&callbackRespBytes), + ) + if err != nil { + return err + } + + return nil +} From 7b62dcb88c5af31cc3bf5f2cd97032d64d14b43f Mon Sep 17 00:00:00 2001 From: Ljcbaby <46277145+ljcbaby@users.noreply.github.com> Date: Thu, 27 Mar 2025 23:22:55 +0800 Subject: [PATCH 482/659] fix(baidu_netdisk): deplicate retry (#8210 redo #7972, link #8180) --- drivers/baidu_netdisk/driver.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 6ea62197b70..4397d413632 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -20,6 +20,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/errgroup" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/avast/retry-go" log "github.com/sirupsen/logrus" ) @@ -260,7 +261,10 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } } // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread) + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) sem := semaphore.NewWeighted(3) for i, partseq := range precreateResp.BlockList { if utils.IsCanceled(upCtx) { From 0cde4e73d614a4bf0b28872391a13d0f8a6d166a Mon Sep 17 00:00:00 2001 From: jerry <109275116+jerry-harm@users.noreply.github.com> Date: Thu, 27 Mar 2025 23:25:23 +0800 Subject: [PATCH 483/659] feat(ipfs): better ipfs support (#8225) * feat: :sparkles: better ipfs support fixed mfs crud, added ipns support * Update driver.go clean up --- drivers/ipfs_api/driver.go | 75 +++++++++++++++++++++++--------------- drivers/ipfs_api/meta.go | 4 +- 2 files changed, 48 insertions(+), 31 deletions(-) diff --git a/drivers/ipfs_api/driver.go b/drivers/ipfs_api/driver.go index 777606564a2..e59da7ca334 100644 --- a/drivers/ipfs_api/driver.go +++ b/drivers/ipfs_api/driver.go @@ -4,13 +4,13 @@ import ( "context" "fmt" "net/url" - stdpath "path" "path/filepath" "strings" + shell "github.com/ipfs/go-ipfs-api" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" - shell "github.com/ipfs/go-ipfs-api" ) type IPFS struct { @@ -44,27 +44,32 @@ func (d *IPFS) Drop(ctx context.Context) error { func (d *IPFS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { path := dir.GetPath() - if path[len(path):] != "/" { - path += "/" + switch d.Mode { + case "ipfs": + path, _ = url.JoinPath("/ipfs", path) + case "ipns": + path, _ = url.JoinPath("/ipns", path) + case "mfs": + fileStat, err := d.sh.FilesStat(ctx, path) + if err != nil { + return nil, err + } + path, _ = url.JoinPath("/ipfs", fileStat.Hash) + default: + return nil, fmt.Errorf("mode error") } - path_cid, err := d.sh.FilesStat(ctx, path) - if err != nil { - return nil, err - } - - dirs, err := d.sh.List(path_cid.Hash) + dirs, err := d.sh.List(path) if err != nil { return nil, err } objlist := []model.Obj{} for _, file := range dirs { - gateurl := *d.gateURL - gateurl.Path = "ipfs/" + file.Hash + gateurl := *d.gateURL.JoinPath("/ipfs/" + file.Hash) gateurl.RawQuery = "filename=" + url.PathEscape(file.Name) objlist = append(objlist, &model.ObjectURL{ - Object: model.Object{ID: file.Hash, Name: file.Name, Size: int64(file.Size), IsFolder: file.Type == 1}, + Object: model.Object{ID: "/ipfs/" + file.Hash, Name: file.Name, Size: int64(file.Size), IsFolder: file.Type == 1}, Url: model.Url{Url: gateurl.String()}, }) } @@ -73,11 +78,15 @@ func (d *IPFS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] } func (d *IPFS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - link := d.Gateway + "/ipfs/" + file.GetID() + "/?filename=" + url.PathEscape(file.GetName()) - return &model.Link{URL: link}, nil + gateurl := d.gateURL.JoinPath(file.GetID()) + gateurl.RawQuery = "filename=" + url.PathEscape(file.GetName()) + return &model.Link{URL: gateurl.String()}, nil } func (d *IPFS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if d.Mode != "mfs" { + return fmt.Errorf("only write in mfs mode") + } path := parentDir.GetPath() if path[len(path):] != "/" { path += "/" @@ -86,42 +95,48 @@ func (d *IPFS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) } func (d *IPFS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if d.Mode != "mfs" { + return fmt.Errorf("only write in mfs mode") + } return d.sh.FilesMv(ctx, srcObj.GetPath(), dstDir.GetPath()) } func (d *IPFS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if d.Mode != "mfs" { + return fmt.Errorf("only write in mfs mode") + } newFileName := filepath.Dir(srcObj.GetPath()) + "/" + newName return d.sh.FilesMv(ctx, srcObj.GetPath(), strings.ReplaceAll(newFileName, "\\", "/")) } func (d *IPFS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - // TODO copy obj, optional - fmt.Println(srcObj.GetPath()) - fmt.Println(dstDir.GetPath()) + if d.Mode != "mfs" { + return fmt.Errorf("only write in mfs mode") + } newFileName := dstDir.GetPath() + "/" + filepath.Base(srcObj.GetPath()) - fmt.Println(newFileName) return d.sh.FilesCp(ctx, srcObj.GetPath(), strings.ReplaceAll(newFileName, "\\", "/")) } func (d *IPFS) Remove(ctx context.Context, obj model.Obj) error { - // TODO remove obj, optional + if d.Mode != "mfs" { + return fmt.Errorf("only write in mfs mode") + } return d.sh.FilesRm(ctx, obj.GetPath(), true) } func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { - // TODO upload file, optional - _, err := d.sh.Add(driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + if d.Mode != "mfs" { + return fmt.Errorf("only write in mfs mode") + } + outHash, err := d.sh.Add(driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, - }), ToFiles(stdpath.Join(dstDir.GetPath(), s.GetName()))) - return err -} - -func ToFiles(dstDir string) shell.AddOpts { - return func(rb *shell.RequestBuilder) error { - rb.Option("to-files", dstDir) - return nil + })) + if err != nil { + return err } + err = d.sh.FilesCp(ctx, "/ipfs/"+outHash, dstDir.GetPath()+"/"+strings.ReplaceAll(s.GetName(), "\\", "/")) + return err } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/ipfs_api/meta.go b/drivers/ipfs_api/meta.go index cdc3042434b..c145644c580 100644 --- a/drivers/ipfs_api/meta.go +++ b/drivers/ipfs_api/meta.go @@ -8,14 +8,16 @@ import ( type Addition struct { // Usually one of two driver.RootPath + Mode string `json:"mode" options:"ipfs,ipns,mfs" type:"select" required:"true"` Endpoint string `json:"endpoint" default:"http://127.0.0.1:5001"` - Gateway string `json:"gateway" default:"https://ipfs.io"` + Gateway string `json:"gateway" default:"http://127.0.0.1:8080"` } var config = driver.Config{ Name: "IPFS API", DefaultRoot: "/", LocalSort: true, + OnlyProxy: false, } func init() { From e4bd223d1c0eb376fc812bbe3427614e48271a40 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Thu, 3 Apr 2025 20:29:53 +0800 Subject: [PATCH 484/659] fix(deps): update 115-sdk-go to v0.1.5 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5ed8a27baa5..f8a238f143e 100644 --- a/go.mod +++ b/go.mod @@ -111,7 +111,7 @@ require ( github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect github.com/therootcompany/xz v1.0.1 // indirect github.com/ulikunitz/xz v0.5.12 // indirect - github.com/xhofe/115-sdk-go v0.1.4 + github.com/xhofe/115-sdk-go v0.1.5 github.com/yuin/goldmark v1.7.8 go4.org v0.0.0-20230225012048-214862532bf5 resty.dev/v3 v3.0.0-beta.2 // indirect diff --git a/go.sum b/go.sum index bf98a8cd784..1681a3a0471 100644 --- a/go.sum +++ b/go.sum @@ -606,8 +606,8 @@ github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXo github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xhofe/115-sdk-go v0.1.4 h1:erIWuWH+kZQOEHM+YZK8Y6sWQ2s/SFJIFh/WeCtjiiY= -github.com/xhofe/115-sdk-go v0.1.4/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= +github.com/xhofe/115-sdk-go v0.1.5 h1:2+E92l6AX0+ABAkrdmDa9PE5ONN7wVLCaKkK80zETOg= +github.com/xhofe/115-sdk-go v0.1.5/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= From 37640221c05adb5f09bb06a53840253b7c0abbb2 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 3 Apr 2025 20:34:27 +0800 Subject: [PATCH 485/659] fix(doubao): update file size type to int64 (#8289) --- drivers/doubao/driver.go | 6 +++--- drivers/doubao/types.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/doubao/driver.go b/drivers/doubao/driver.go index b847ffa90b4..04f74325df0 100644 --- a/drivers/doubao/driver.go +++ b/drivers/doubao/driver.go @@ -55,9 +55,9 @@ func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( ID: child.ID, Path: child.ParentID, Name: child.Name, - Size: int64(child.Size), - Modified: time.Unix(int64(child.UpdateTime), 0), - Ctime: time.Unix(int64(child.CreateTime), 0), + Size: child.Size, + Modified: time.Unix(child.UpdateTime, 0), + Ctime: time.Unix(child.CreateTime, 0), IsFolder: child.NodeType == 1, }, Key: child.Key, diff --git a/drivers/doubao/types.go b/drivers/doubao/types.go index f9611d86ced..2dc5a61dac0 100644 --- a/drivers/doubao/types.go +++ b/drivers/doubao/types.go @@ -22,15 +22,15 @@ type NodeInfo struct { Name string `json:"name"` Key string `json:"key"` NodeType int `json:"node_type"` // 0: 文件, 1: 文件夹 - Size int `json:"size"` + Size int64 `json:"size"` Source int `json:"source"` NameReviewStatus int `json:"name_review_status"` ContentReviewStatus int `json:"content_review_status"` RiskReviewStatus int `json:"risk_review_status"` ConversationID string `json:"conversation_id"` ParentID string `json:"parent_id"` - CreateTime int `json:"create_time"` - UpdateTime int `json:"update_time"` + CreateTime int64 `json:"create_time"` + UpdateTime int64 `json:"update_time"` } type GetFileUrlResp struct { From affd0cecd1a131d78a5d3e695ecaeb8d98397cb5 Mon Sep 17 00:00:00 2001 From: YangXu <47767754+Three-taile-dragon@users.noreply.github.com> Date: Thu, 3 Apr 2025 20:35:14 +0800 Subject: [PATCH 486/659] fix(pikpak&pikpak_share): update algorithms (#8278) --- drivers/pikpak/driver.go | 2 +- drivers/pikpak/util.go | 56 ++++++++++++++++------------------ drivers/pikpak_share/driver.go | 2 +- drivers/pikpak_share/util.go | 56 ++++++++++++++++------------------ 4 files changed, 56 insertions(+), 60 deletions(-) diff --git a/drivers/pikpak/driver.go b/drivers/pikpak/driver.go index 504b1d0e9f9..6c64e6fb066 100644 --- a/drivers/pikpak/driver.go +++ b/drivers/pikpak/driver.go @@ -69,7 +69,7 @@ func (d *PikPak) Init(ctx context.Context) (err error) { d.ClientVersion = PCClientVersion d.PackageName = PCPackageName d.Algorithms = PCAlgorithms - d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.5.6.4831 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" + d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.6.11.4955 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" } if d.Addition.CaptchaToken != "" && d.Addition.RefreshToken == "" { diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index f2594e78f5e..61396aa4ece 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -28,34 +28,32 @@ import ( ) var AndroidAlgorithms = []string{ - "7xOq4Z8s", - "QE9/9+IQco", - "WdX5J9CPLZp", - "NmQ5qFAXqH3w984cYhMeC5TJR8j", - "cc44M+l7GDhav", - "KxGjo/wHB+Yx8Lf7kMP+/m9I+", - "wla81BUVSmDkctHDpUT", - "c6wMr1sm1WxiR3i8LDAm3W", - "hRLrEQCFNYi0PFPV", - "o1J41zIraDtJPNuhBu7Ifb/q3", - "U", - "RrbZvV0CTu3gaZJ56PVKki4IeP", - "NNuRbLckJqUp1Do0YlrKCUP", - "UUwnBbipMTvInA0U0E9", - "VzGc", + "SOP04dGzk0TNO7t7t9ekDbAmx+eq0OI1ovEx", + "nVBjhYiND4hZ2NCGyV5beamIr7k6ifAsAbl", + "Ddjpt5B/Cit6EDq2a6cXgxY9lkEIOw4yC1GDF28KrA", + "VVCogcmSNIVvgV6U+AochorydiSymi68YVNGiz", + "u5ujk5sM62gpJOsB/1Gu/zsfgfZO", + "dXYIiBOAHZgzSruaQ2Nhrqc2im", + "z5jUTBSIpBN9g4qSJGlidNAutX6", + "KJE2oveZ34du/g1tiimm", } var WebAlgorithms = []string{ - "fyZ4+p77W1U4zcWBUwefAIFhFxvADWtT1wzolCxhg9q7etmGUjXr", - "uSUX02HYJ1IkyLdhINEFcCf7l2", - "iWt97bqD/qvjIaPXB2Ja5rsBWtQtBZZmaHH2rMR41", - "3binT1s/5a1pu3fGsN", - "8YCCU+AIr7pg+yd7CkQEY16lDMwi8Rh4WNp5", - "DYS3StqnAEKdGddRP8CJrxUSFh", - "crquW+4", - "ryKqvW9B9hly+JAymXCIfag5Z", - "Hr08T/NDTX1oSJfHk90c", - "i", + "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", + "+r6CQVxjzJV6LCV", + "F", + "pFJRC", + "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", + "/750aCr4lm/Sly/c", + "RB+DT/gZCrbV", + "", + "CyLsf7hdkIRxRm215hl", + "7xHvLi2tOYP0Y92b", + "ZGTXXxu8E/MIWaEDB+Sm/", + "1UI3", + "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", + "ihtqpG6FMt65+Xk+tWUH2", + "NhXXU9rg4XXdzo7u5o", } var PCAlgorithms = []string{ @@ -80,17 +78,17 @@ const ( const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - AndroidClientVersion = "1.49.3" + AndroidClientVersion = "1.53.2" AndroidPackageName = "com.pikcloud.pikpak" - AndroidSdkVersion = "2.0.4.204101" + AndroidSdkVersion = "2.0.6.206003" WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - WebClientVersion = "undefined" + WebClientVersion = "2.0.0" WebPackageName = "drive.mypikpak.com" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" - PCClientVersion = "undefined" // 2.5.6.4831 + PCClientVersion = "undefined" // 2.6.11.4955 PCPackageName = "mypikpak.com" PCSdkVersion = "8.0.3" ) diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index d527a1ab1a4..d6341bd990a 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -66,7 +66,7 @@ func (d *PikPakShare) Init(ctx context.Context) error { d.ClientVersion = PCClientVersion d.PackageName = PCPackageName d.Algorithms = PCAlgorithms - d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.5.6.4831 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" + d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.6.11.4955 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" } // 获取CaptchaToken diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index 172a61487d8..4111779f4b2 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -17,34 +17,32 @@ import ( ) var AndroidAlgorithms = []string{ - "7xOq4Z8s", - "QE9/9+IQco", - "WdX5J9CPLZp", - "NmQ5qFAXqH3w984cYhMeC5TJR8j", - "cc44M+l7GDhav", - "KxGjo/wHB+Yx8Lf7kMP+/m9I+", - "wla81BUVSmDkctHDpUT", - "c6wMr1sm1WxiR3i8LDAm3W", - "hRLrEQCFNYi0PFPV", - "o1J41zIraDtJPNuhBu7Ifb/q3", - "U", - "RrbZvV0CTu3gaZJ56PVKki4IeP", - "NNuRbLckJqUp1Do0YlrKCUP", - "UUwnBbipMTvInA0U0E9", - "VzGc", + "SOP04dGzk0TNO7t7t9ekDbAmx+eq0OI1ovEx", + "nVBjhYiND4hZ2NCGyV5beamIr7k6ifAsAbl", + "Ddjpt5B/Cit6EDq2a6cXgxY9lkEIOw4yC1GDF28KrA", + "VVCogcmSNIVvgV6U+AochorydiSymi68YVNGiz", + "u5ujk5sM62gpJOsB/1Gu/zsfgfZO", + "dXYIiBOAHZgzSruaQ2Nhrqc2im", + "z5jUTBSIpBN9g4qSJGlidNAutX6", + "KJE2oveZ34du/g1tiimm", } var WebAlgorithms = []string{ - "fyZ4+p77W1U4zcWBUwefAIFhFxvADWtT1wzolCxhg9q7etmGUjXr", - "uSUX02HYJ1IkyLdhINEFcCf7l2", - "iWt97bqD/qvjIaPXB2Ja5rsBWtQtBZZmaHH2rMR41", - "3binT1s/5a1pu3fGsN", - "8YCCU+AIr7pg+yd7CkQEY16lDMwi8Rh4WNp5", - "DYS3StqnAEKdGddRP8CJrxUSFh", - "crquW+4", - "ryKqvW9B9hly+JAymXCIfag5Z", - "Hr08T/NDTX1oSJfHk90c", - "i", + "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", + "+r6CQVxjzJV6LCV", + "F", + "pFJRC", + "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", + "/750aCr4lm/Sly/c", + "RB+DT/gZCrbV", + "", + "CyLsf7hdkIRxRm215hl", + "7xHvLi2tOYP0Y92b", + "ZGTXXxu8E/MIWaEDB+Sm/", + "1UI3", + "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", + "ihtqpG6FMt65+Xk+tWUH2", + "NhXXU9rg4XXdzo7u5o", } var PCAlgorithms = []string{ @@ -63,17 +61,17 @@ var PCAlgorithms = []string{ const ( AndroidClientID = "YNxT9w7GMdWvEOKa" AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - AndroidClientVersion = "1.49.3" + AndroidClientVersion = "1.53.2" AndroidPackageName = "com.pikcloud.pikpak" - AndroidSdkVersion = "2.0.4.204101" + AndroidSdkVersion = "2.0.6.206003" WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" - WebClientVersion = "undefined" + WebClientVersion = "2.0.0" WebPackageName = "drive.mypikpak.com" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" - PCClientVersion = "undefined" // 2.5.6.4831 + PCClientVersion = "undefined" // 2.6.11.4955 PCPackageName = "mypikpak.com" PCSdkVersion = "8.0.3" ) From a6304285b6271633d5c881ebff4472f93fc19e33 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Thu, 3 Apr 2025 20:35:52 +0800 Subject: [PATCH 487/659] fix: revert "refactor(net): pass request header" (#8269) https://github.com/AlistGo/alist/pull/8031/commits/5be50e77d9ad8d67e343aa7e9380bcdd2506ae8f --- internal/net/serve.go | 2 +- internal/stream/util.go | 12 ++---------- server/common/proxy.go | 4 ++-- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/internal/net/serve.go b/internal/net/serve.go index 8b6b3d1d234..63e1cb45b87 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -114,7 +114,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time // 使用请求的Context // 不然从sendContent读不到数据，即使请求断开CopyBuffer也会一直堵塞 - ctx := context.WithValue(r.Context(), "request_header", &r.Header) + ctx := context.WithValue(r.Context(), "request_header", r.Header) switch { case len(ranges) == 0: reader, err := RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) diff --git a/internal/stream/util.go b/internal/stream/util.go index b2c76754040..01019482e15 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -19,11 +19,7 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl } rangeReaderFunc := func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { if link.Concurrency != 0 || link.PartSize != 0 { - requestHeader := ctx.Value("request_header") - if requestHeader == nil { - requestHeader = &http.Header{} - } - header := net.ProcessHeader(*(requestHeader.(*http.Header)), link.Header) + header := net.ProcessHeader(nil, link.Header) down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize @@ -64,11 +60,7 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl } func RequestRangedHttp(ctx context.Context, link *model.Link, offset, length int64) (*http.Response, error) { - requestHeader := ctx.Value("request_header") - if requestHeader == nil { - requestHeader = &http.Header{} - } - header := net.ProcessHeader(*(requestHeader.(*http.Header)), link.Header) + header := net.ProcessHeader(nil, link.Header) header = http_range.ApplyRangeToHttpHeader(http_range.Range{Start: offset, Length: length}, header) return net.RequestHttp(ctx, "GET", header, link.URL) diff --git a/server/common/proxy.go b/server/common/proxy.go index c14af6fad5e..f9e1e4bb0e8 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -50,9 +50,9 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { requestHeader := ctx.Value("request_header") if requestHeader == nil { - requestHeader = &http.Header{} + requestHeader = http.Header{} } - header := net.ProcessHeader(*(requestHeader.(*http.Header)), link.Header) + header := net.ProcessHeader(requestHeader.(http.Header), link.Header) down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize From 465dd1703deda982cfff7c1fce4047932f78108e Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 3 Apr 2025 20:40:19 +0800 Subject: [PATCH 488/659] feat(cloudreve): s3 policy support (#8245) * feat(cloudreve): s3 policy support * fix(cloudreve): correct potential off-by-one error in `etags` initialization --- drivers/cloudreve/driver.go | 2 + drivers/cloudreve/types.go | 11 +++--- drivers/cloudreve/util.go | 79 +++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 5 deletions(-) diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index d0ab30b6c11..8c2321b8f40 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -162,6 +162,8 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File switch r.Policy.Type { case "onedrive": err = d.upOneDrive(ctx, stream, u, up) + case "s3": + err = d.upS3(ctx, stream, u, up) case "remote": // 从机存储 err = d.upRemote(ctx, stream, u, up) case "local": // 本机存储 diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index a7c3919e8a9..8a465f01a3b 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -21,11 +21,12 @@ type Policy struct { } type UploadInfo struct { - SessionID string `json:"sessionID"` - ChunkSize int `json:"chunkSize"` - Expires int `json:"expires"` - UploadURLs []string `json:"uploadURLs"` - Credential string `json:"credential,omitempty"` + SessionID string `json:"sessionID"` + ChunkSize int `json:"chunkSize"` + Expires int `json:"expires"` + UploadURLs []string `json:"uploadURLs"` + Credential string `json:"credential,omitempty"` // local + CompleteURL string `json:"completeURL,omitempty"` // s3 } type DirectoryResp struct { diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index cffa7988bb3..1fd5ed8abae 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -312,3 +312,82 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u } return nil } + +func (d *Cloudreve) upS3(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { + var finish int64 = 0 + var chunk int = 0 + var etags []string + DEFAULT := int64(u.ChunkSize) + for finish < stream.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + utils.Log.Debugf("[Cloudreve-S3] upload: %d", finish) + var byteSize = DEFAULT + left := stream.GetSize() - finish + if left < DEFAULT { + byteSize = left + } + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequest("PUT", u.UploadURLs[chunk], + driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.ContentLength = byteSize + finish += byteSize + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + _ = res.Body.Close() + etags = append(etags, res.Header.Get("ETag")) + up(float64(finish) * 100 / float64(stream.GetSize())) + chunk++ + } + + // s3LikeFinishUpload + // https://github.com/cloudreve/frontend/blob/b485bf297974cbe4834d2e8e744ae7b7e5b2ad39/src/component/Uploader/core/api/index.ts#L204-L252 + bodyBuilder := &strings.Builder{} + bodyBuilder.WriteString("") + for i, etag := range etags { + bodyBuilder.WriteString(fmt.Sprintf( + `%d%s`, + i+1, // PartNumber 从 1 开始 + etag, + )) + } + bodyBuilder.WriteString("") + req, err := http.NewRequest( + "POST", + u.CompleteURL, + strings.NewReader(bodyBuilder.String()), + ) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/xml") + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body)) + } + + // 上传成功发送回调请求 + err = d.request(http.MethodGet, "/callback/s3/"+u.SessionID, nil, nil) + if err != nil { + return err + } + return nil +} From 31c55a2adf6f4dc9f972cf779019be8f2ea95ddb Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Thu, 3 Apr 2025 20:41:05 +0800 Subject: [PATCH 489/659] fix(archive): unable to preview (#8248) * fix(archive): unable to preview * fix bug --- internal/archive/tool/helper.go | 47 ++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/internal/archive/tool/helper.go b/internal/archive/tool/helper.go index 8f71900ac56..20da34467b0 100644 --- a/internal/archive/tool/helper.go +++ b/internal/archive/tool/helper.go @@ -29,7 +29,6 @@ type ArchiveReader interface { func GenerateMetaTreeFromFolderTraversal(r ArchiveReader) (bool, []model.ObjTree) { encrypted := false dirMap := make(map[string]*model.ObjectTree) - dirMap["."] = &model.ObjectTree{} for _, file := range r.Files() { if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { encrypted = true @@ -44,7 +43,7 @@ func GenerateMetaTreeFromFolderTraversal(r ArchiveReader) (bool, []model.ObjTree dir = stdpath.Dir(name) dirObj = dirMap[dir] if dirObj == nil { - isNewFolder = true + isNewFolder = dir != "." dirObj = &model.ObjectTree{} dirObj.IsFolder = true dirObj.Name = stdpath.Base(dir) @@ -60,41 +59,45 @@ func GenerateMetaTreeFromFolderTraversal(r ArchiveReader) (bool, []model.ObjTree dir = strings.TrimSuffix(name, "/") dirObj = dirMap[dir] if dirObj == nil { - isNewFolder = true + isNewFolder = dir != "." dirObj = &model.ObjectTree{} dirMap[dir] = dirObj } dirObj.IsFolder = true dirObj.Name = stdpath.Base(dir) dirObj.Modified = file.FileInfo().ModTime() - dirObj.Children = make([]model.ObjTree, 0) } if isNewFolder { // 将 文件夹 添加到 父文件夹 - dir = stdpath.Dir(dir) - pDirObj := dirMap[dir] - if pDirObj != nil { - pDirObj.Children = append(pDirObj.Children, dirObj) - continue - } - + // 考虑压缩包仅记录文件的路径，不记录文件夹 + // 循环创建所有父文件夹 + parentDir := stdpath.Dir(dir) for { - // 考虑压缩包仅记录文件的路径，不记录文件夹 - pDirObj = &model.ObjectTree{} - pDirObj.IsFolder = true - pDirObj.Name = stdpath.Base(dir) - pDirObj.Modified = file.FileInfo().ModTime() - dirMap[dir] = pDirObj - pDirObj.Children = append(pDirObj.Children, dirObj) - dir = stdpath.Dir(dir) - if dirMap[dir] != nil { + parentDirObj := dirMap[parentDir] + if parentDirObj == nil { + parentDirObj = &model.ObjectTree{} + if parentDir != "." { + parentDirObj.IsFolder = true + parentDirObj.Name = stdpath.Base(parentDir) + parentDirObj.Modified = file.FileInfo().ModTime() + } + dirMap[parentDir] = parentDirObj + } + parentDirObj.Children = append(parentDirObj.Children, dirObj) + + parentDir = stdpath.Dir(parentDir) + if dirMap[parentDir] != nil { break } - dirObj = pDirObj + dirObj = parentDirObj } } } - return encrypted, dirMap["."].GetChildren() + if len(dirMap) > 0 { + return encrypted, dirMap["."].GetChildren() + } else { + return encrypted, nil + } } func MakeModelObj(file os.FileInfo) *model.Object { From af18cb138bce30fb5d927b30fa80fb13f182fea1 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Thu, 3 Apr 2025 20:41:59 +0800 Subject: [PATCH 490/659] feat(139): add option ReportRealSize (#8244 close #8141) * feat(139): handle family upload errors * feat(139): add option `ReportRealSize` * Update drivers/139/driver.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- drivers/139/driver.go | 34 +++++++++++++++++++++++++++------- drivers/139/meta.go | 1 + drivers/139/types.go | 7 +++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index c6b30335770..f367c431c1b 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -3,6 +3,7 @@ package _139 import ( "context" "encoding/base64" + "encoding/xml" "fmt" "io" "net/http" @@ -740,14 +741,20 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr break } } + var reportSize int64 + if d.ReportRealSize { + reportSize = stream.GetSize() + } else { + reportSize = 0 + } data := base.Json{ "manualRename": 2, "operation": 0, "fileCount": 1, - "totalSize": 0, // 去除上传大小限制 + "totalSize": reportSize, "uploadContentList": []base.Json{{ "contentName": stream.GetName(), - "contentSize": 0, // 去除上传大小限制 + "contentSize": reportSize, // "digest": "5a3231986ce7a6b46e408612d385bafa" }}, "parentCatalogID": dstDir.GetID(), @@ -765,10 +772,10 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "operation": 0, "path": path.Join(dstDir.GetPath(), dstDir.GetID()), "seqNo": random.String(32), //序列号不能为空 - "totalSize": 0, + "totalSize": reportSize, "uploadContentList": []base.Json{{ "contentName": stream.GetName(), - "contentSize": 0, + "contentSize": reportSize, // "digest": "5a3231986ce7a6b46e408612d385bafa" }}, }) @@ -779,6 +786,9 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr if err != nil { return err } + if resp.Data.Result.ResultCode != "0" { + return fmt.Errorf("get file upload url failed with result code: %s, message: %s", resp.Data.Result.ResultCode, resp.Data.Result.ResultDesc) + } // Progress p := driver.NewProgress(stream.GetSize(), up) @@ -820,13 +830,23 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr if err != nil { return err } - _ = res.Body.Close() - log.Debugf("%+v", res) if res.StatusCode != http.StatusOK { + res.Body.Close() return fmt.Errorf("unexpected status code: %d", res.StatusCode) } + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("error reading response body: %v", err) + } + var result InterLayerUploadResult + err = xml.Unmarshal(bodyBytes, &result) + if err != nil { + return fmt.Errorf("error parsing XML: %v", err) + } + if result.ResultCode != 0 { + return fmt.Errorf("upload failed with result code: %d, message: %s", result.ResultCode, result.Msg) + } } - return nil default: return errs.NotImplement diff --git a/drivers/139/meta.go b/drivers/139/meta.go index d80b8566131..866aadb4192 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -12,6 +12,7 @@ type Addition struct { Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` CloudID string `json:"cloud_id"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` + ReportRealSize bool `json:"report_real_size" type:"bool" default:"true" help:"Enable to report the real file size during upload"` } var config = driver.Config{ diff --git a/drivers/139/types.go b/drivers/139/types.go index ac7079d8d18..50ae1f81242 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -143,6 +143,13 @@ type UploadResp struct { } `json:"data"` } +type InterLayerUploadResult struct { + XMLName xml.Name `xml:"result"` + Text string `xml:",chardata"` + ResultCode int `xml:"resultCode"` + Msg string `xml:"msg"` +} + type CloudContent struct { ContentID string `json:"contentID"` //Modifier string `json:"modifier"` From 2e21df066105c078f3a7d435ab8232eae644172a Mon Sep 17 00:00:00 2001 From: New Future Date: Thu, 3 Apr 2025 20:43:21 +0800 Subject: [PATCH 491/659] feat(driver): add Azure Blob Storage driver (#8261) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add azure-blob driver * fix nested folders copy * feat(driver): add Azure Blob Storage driver 实现 Azure Blob Storage 驱动，支持以下功能： - 使用共享密钥身份验证初始化连接 - 列出目录和文件 - 生成临时 SAS URL 进行文件访问 - 创建目录 - 移动和重命名文件/文件夹 - 复制文件/文件夹 - 删除文件/文件夹 - 上传文件并支持进度跟踪 此驱动允许用户通过 AList 平台无缝访问和管理 Azure Blob Storage 中的数据。 * feat(driver): update help doc for Azure Blob * doc(readme): add new driver * Update drivers/azure_blob/driver.go fix(azure): fix name check Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update README.md doc(readme): fix the link Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix(azure): fix log and link --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 1 + drivers/all.go | 1 + drivers/azure_blob/driver.go | 313 +++++++++++++++++++++++++++ drivers/azure_blob/meta.go | 27 +++ drivers/azure_blob/types.go | 20 ++ drivers/azure_blob/util.go | 401 +++++++++++++++++++++++++++++++++++ go.mod | 6 + go.sum | 6 + 8 files changed, 775 insertions(+) create mode 100644 drivers/azure_blob/driver.go create mode 100644 drivers/azure_blob/meta.go create mode 100644 drivers/azure_blob/types.go create mode 100644 drivers/azure_blob/util.go diff --git a/README.md b/README.md index d1189188c41..1261839e429 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] [Dropbox](https://www.dropbox.com/) - [x] [FeijiPan](https://www.feijipan.com/) - [x] [dogecloud](https://www.dogecloud.com/product/oss) + - [x] [Azure Blob Storage](https://azure.microsoft.com/products/storage/blobs) - [x] Easy to deploy and out-of-the-box - [x] File preview (PDF, markdown, code, plain text, ...) - [x] Image preview in gallery mode diff --git a/drivers/all.go b/drivers/all.go index a14e80fbc59..083d01dcd8c 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -16,6 +16,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/aliyundrive" _ "github.com/alist-org/alist/v3/drivers/aliyundrive_open" _ "github.com/alist-org/alist/v3/drivers/aliyundrive_share" + _ "github.com/alist-org/alist/v3/drivers/azure_blob" _ "github.com/alist-org/alist/v3/drivers/baidu_netdisk" _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" diff --git a/drivers/azure_blob/driver.go b/drivers/azure_blob/driver.go new file mode 100644 index 00000000000..6836533a070 --- /dev/null +++ b/drivers/azure_blob/driver.go @@ -0,0 +1,313 @@ +package azure_blob + +import ( + "context" + "fmt" + "io" + "path" + "regexp" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) +// Azure Blob Storage based on the blob APIs +// Link: https://learn.microsoft.com/rest/api/storageservices/blob-service-rest-api +type AzureBlob struct { + model.Storage + Addition + client *azblob.Client + containerClient *container.Client + config driver.Config +} + +// Config returns the driver configuration. +func (d *AzureBlob) Config() driver.Config { + return d.config +} + +// GetAddition returns additional settings specific to Azure Blob Storage. +func (d *AzureBlob) GetAddition() driver.Additional { + return &d.Addition +} + +// Init initializes the Azure Blob Storage client using shared key authentication. +func (d *AzureBlob) Init(ctx context.Context) error { + // Validate the endpoint URL + accountName := extractAccountName(d.Addition.Endpoint) + if !regexp.MustCompile(`^[a-z0-9]+$`).MatchString(accountName) { + return fmt.Errorf("invalid storage account name: must be chars of lowercase letters or numbers only") + } + + credential, err := azblob.NewSharedKeyCredential(accountName, d.Addition.AccessKey) + if err != nil { + return fmt.Errorf("failed to create credential: %w", err) + } + + // Check if Endpoint is just account name + endpoint := d.Addition.Endpoint + if accountName == endpoint { + endpoint = fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + } + // Initialize Azure Blob client with retry policy + client, err := azblob.NewClientWithSharedKeyCredential(endpoint, credential, + &azblob.ClientOptions{ClientOptions: azcore.ClientOptions{ + Retry: policy.RetryOptions{ + MaxRetries: MaxRetries, + RetryDelay: RetryDelay, + }, + }}) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + d.client = client + + // Ensure container exists or create it + containerName := strings.Trim(d.Addition.ContainerName, "/ \\") + if containerName == "" { + return fmt.Errorf("container name cannot be empty") + } + return d.createContainerIfNotExists(ctx, containerName) +} + +// Drop releases resources associated with the Azure Blob client. +func (d *AzureBlob) Drop(ctx context.Context) error { + d.client = nil + return nil +} + +// List retrieves blobs and directories under the specified path. +func (d *AzureBlob) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + prefix := ensureTrailingSlash(dir.GetPath()) + + pager := d.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ + Prefix: &prefix, + }) + + var objs []model.Obj + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list blobs: %w", err) + } + + // Process directories + for _, blobPrefix := range page.Segment.BlobPrefixes { + objs = append(objs, &model.Object{ + Name: path.Base(strings.TrimSuffix(*blobPrefix.Name, "/")), + Path: *blobPrefix.Name, + Modified: *blobPrefix.Properties.LastModified, + Ctime: *blobPrefix.Properties.CreationTime, + IsFolder: true, + }) + } + + // Process files + for _, blob := range page.Segment.BlobItems { + if strings.HasSuffix(*blob.Name, "/") { + continue + } + objs = append(objs, &model.Object{ + Name: path.Base(*blob.Name), + Path: *blob.Name, + Size: *blob.Properties.ContentLength, + Modified: *blob.Properties.LastModified, + Ctime: *blob.Properties.CreationTime, + IsFolder: false, + }) + } + } + return objs, nil +} + +// Link generates a temporary SAS URL for accessing a blob. +func (d *AzureBlob) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + blobClient := d.containerClient.NewBlobClient(file.GetPath()) + expireDuration := time.Hour * time.Duration(d.SignURLExpire) + + sasURL, err := blobClient.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expireDuration), nil) + if err != nil { + return nil, fmt.Errorf("failed to generate SAS URL: %w", err) + } + return &model.Link{URL: sasURL}, nil +} + +// MakeDir creates a virtual directory by uploading an empty blob as a marker. +func (d *AzureBlob) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + dirPath := path.Join(parentDir.GetPath(), dirName) + if err := d.mkDir(ctx, dirPath); err != nil { + return nil, fmt.Errorf("failed to create directory marker: %w", err) + } + + return &model.Object{ + Path: dirPath, + Name: dirName, + IsFolder: true, + }, nil +} + +// Move relocates an object (file or directory) to a new directory. +func (d *AzureBlob) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcPath := srcObj.GetPath() + dstPath := path.Join(dstDir.GetPath(), srcObj.GetName()) + + if err := d.moveOrRename(ctx, srcPath, dstPath, srcObj.IsDir(), srcObj.GetSize()); err != nil { + return nil, fmt.Errorf("move operation failed: %w", err) + } + + return &model.Object{ + Path: dstPath, + Name: srcObj.GetName(), + Modified: time.Now(), + IsFolder: srcObj.IsDir(), + Size: srcObj.GetSize(), + }, nil +} + +// Rename changes the name of an existing object. +func (d *AzureBlob) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + srcPath := srcObj.GetPath() + dstPath := path.Join(path.Dir(srcPath), newName) + + if err := d.moveOrRename(ctx, srcPath, dstPath, srcObj.IsDir(), srcObj.GetSize()); err != nil { + return nil, fmt.Errorf("rename operation failed: %w", err) + } + + return &model.Object{ + Path: dstPath, + Name: newName, + Modified: time.Now(), + IsFolder: srcObj.IsDir(), + Size: srcObj.GetSize(), + }, nil +} + +// Copy duplicates an object (file or directory) to a specified destination directory. +func (d *AzureBlob) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + dstPath := path.Join(dstDir.GetPath(), srcObj.GetName()) + + // Handle directory copying using flat listing + if srcObj.IsDir() { + srcPrefix := srcObj.GetPath() + srcPrefix = ensureTrailingSlash(srcPrefix) + + // Get all blobs under the source directory + blobs, err := d.flattenListBlobs(ctx, srcPrefix) + if err != nil { + return nil, fmt.Errorf("failed to list source directory contents: %w", err) + } + + // Process each blob - copy to destination + for _, blob := range blobs { + // Skip the directory marker itself + if *blob.Name == srcPrefix { + continue + } + + // Calculate relative path from source + relPath := strings.TrimPrefix(*blob.Name, srcPrefix) + itemDstPath := path.Join(dstPath, relPath) + + if strings.HasSuffix(itemDstPath, "/") || (blob.Metadata["hdi_isfolder"] != nil && *blob.Metadata["hdi_isfolder"] == "true") { + // Create directory marker at destination + err := d.mkDir(ctx, itemDstPath) + if err != nil { + return nil, fmt.Errorf("failed to create directory marker [%s]: %w", itemDstPath, err) + } + } else { + // Copy the blob + if err := d.copyFile(ctx, *blob.Name, itemDstPath); err != nil { + return nil, fmt.Errorf("failed to copy %s: %w", *blob.Name, err) + } + } + + } + + // Create directory marker at destination if needed + if len(blobs) == 0 { + err := d.mkDir(ctx, dstPath) + if err != nil { + return nil, fmt.Errorf("failed to create directory [%s]: %w", dstPath, err) + } + } + + return &model.Object{ + Path: dstPath, + Name: srcObj.GetName(), + Modified: time.Now(), + IsFolder: true, + }, nil + } + + // Copy a single file + if err := d.copyFile(ctx, srcObj.GetPath(), dstPath); err != nil { + return nil, fmt.Errorf("failed to copy blob: %w", err) + } + return &model.Object{ + Path: dstPath, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: time.Now(), + IsFolder: false, + }, nil +} + +// Remove deletes a specified blob or recursively deletes a directory and its contents. +func (d *AzureBlob) Remove(ctx context.Context, obj model.Obj) error { + path := obj.GetPath() + + // Handle recursive directory deletion + if obj.IsDir() { + return d.deleteFolder(ctx, path) + } + + // Delete single file + return d.deleteFile(ctx, path, false) +} + +// Put uploads a file stream to Azure Blob Storage with progress tracking. +func (d *AzureBlob) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + blobPath := path.Join(dstDir.GetPath(), stream.GetName()) + blobClient := d.containerClient.NewBlockBlobClient(blobPath) + + // Determine optimal upload options based on file size + options := optimizedUploadOptions(stream.GetSize()) + + // Track upload progress + progressTracker := &progressTracker{ + total: stream.GetSize(), + updateProgress: up, + } + + // Wrap stream to handle context cancellation and progress tracking + limitedStream := driver.NewLimitedUploadStream(ctx, io.TeeReader(stream, progressTracker)) + + // Upload the stream to Azure Blob Storage + _, err := blobClient.UploadStream(ctx, limitedStream, options) + if err != nil { + return nil, fmt.Errorf("failed to upload file: %w", err) + } + + return &model.Object{ + Path: blobPath, + Name: stream.GetName(), + Size: stream.GetSize(), + Modified: time.Now(), + IsFolder: false, + }, nil +} + +// The following methods related to archive handling are not implemented yet. +// func (d *AzureBlob) GetArchiveMeta(...) {...} +// func (d *AzureBlob) ListArchive(...) {...} +// func (d *AzureBlob) Extract(...) {...} +// func (d *AzureBlob) ArchiveDecompress(...) {...} + +// Ensure AzureBlob implements the driver.Driver interface. +var _ driver.Driver = (*AzureBlob)(nil) diff --git a/drivers/azure_blob/meta.go b/drivers/azure_blob/meta.go new file mode 100644 index 00000000000..8e42bdd6b6d --- /dev/null +++ b/drivers/azure_blob/meta.go @@ -0,0 +1,27 @@ +package azure_blob + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + Endpoint string `json:"endpoint" required:"true" default:"https://.blob.core.windows.net/" help:"e.g. https://accountname.blob.core.windows.net/. The full endpoint URL for Azure Storage, including the unique storage account name (3 ~ 24 numbers and lowercase letters only)."` + AccessKey string `json:"access_key" required:"true" help:"The access key for Azure Storage, used for authentication. https://learn.microsoft.com/azure/storage/common/storage-account-keys-manage"` + ContainerName string `json:"container_name" required:"true" help:"The name of the container in Azure Storage (created in the Azure portal). https://learn.microsoft.com/azure/storage/blobs/blob-containers-portal"` + SignURLExpire int `json:"sign_url_expire" type:"number" default:"4" help:"The expiration time for SAS URLs, in hours."` +} + +var config = driver.Config{ + Name: "Azure Blob Storage", + LocalSort: true, + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &AzureBlob{ + config: config, + } + }) +} diff --git a/drivers/azure_blob/types.go b/drivers/azure_blob/types.go new file mode 100644 index 00000000000..01323e51273 --- /dev/null +++ b/drivers/azure_blob/types.go @@ -0,0 +1,20 @@ +package azure_blob + +import "github.com/alist-org/alist/v3/internal/driver" + +// progressTracker is used to track upload progress +type progressTracker struct { + total int64 + current int64 + updateProgress driver.UpdateProgress +} + +// Write implements io.Writer to track progress +func (pt *progressTracker) Write(p []byte) (n int, err error) { + n = len(p) + pt.current += int64(n) + if pt.updateProgress != nil && pt.total > 0 { + pt.updateProgress(float64(pt.current) * 100 / float64(pt.total)) + } + return n, nil +} diff --git a/drivers/azure_blob/util.go b/drivers/azure_blob/util.go new file mode 100644 index 00000000000..2adf3a0f343 --- /dev/null +++ b/drivers/azure_blob/util.go @@ -0,0 +1,401 @@ +package azure_blob + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "path" + "sort" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + log "github.com/sirupsen/logrus" +) + +const ( + // MaxRetries defines the maximum number of retry attempts for Azure operations + MaxRetries = 3 + // RetryDelay defines the base delay between retries + RetryDelay = 3 * time.Second + // MaxBatchSize defines the maximum number of operations in a single batch request + MaxBatchSize = 128 +) + +// extractAccountName 从 Azure 存储 Endpoint 中提取账户名 +func extractAccountName(endpoint string) string { + // 移除协议前缀 + endpoint = strings.TrimPrefix(endpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + + // 获取第一个点之前的部分（即账户名） + parts := strings.Split(endpoint, ".") + if len(parts) > 0 { + // to lower case + return strings.ToLower(parts[0]) + } + return "" +} + +// isNotFoundError checks if the error is a "not found" type error +func isNotFoundError(err error) bool { + var storageErr *azcore.ResponseError + if errors.As(err, &storageErr) { + return storageErr.StatusCode == 404 + } + // Fallback to string matching for backwards compatibility + return err != nil && strings.Contains(err.Error(), "BlobNotFound") +} + +// flattenListBlobs - Optimize blob listing to handle pagination better +func (d *AzureBlob) flattenListBlobs(ctx context.Context, prefix string) ([]container.BlobItem, error) { + // Standardize prefix format + prefix = ensureTrailingSlash(prefix) + + var blobItems []container.BlobItem + pager := d.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Prefix: &prefix, + Include: container.ListBlobsInclude{ + Metadata: true, + }, + }) + + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list blobs: %w", err) + } + + for _, blob := range page.Segment.BlobItems { + blobItems = append(blobItems, *blob) + } + } + + return blobItems, nil +} + +// batchDeleteBlobs - Simplify batch deletion logic +func (d *AzureBlob) batchDeleteBlobs(ctx context.Context, blobPaths []string) error { + if len(blobPaths) == 0 { + return nil + } + + // Process in batches of MaxBatchSize + for i := 0; i < len(blobPaths); i += MaxBatchSize { + end := min(i+MaxBatchSize, len(blobPaths)) + currentBatch := blobPaths[i:end] + + // Create batch builder + batchBuilder, err := d.containerClient.NewBatchBuilder() + if err != nil { + return fmt.Errorf("failed to create batch builder: %w", err) + } + + // Add delete operations + for _, blobPath := range currentBatch { + if err := batchBuilder.Delete(blobPath, nil); err != nil { + return fmt.Errorf("failed to add delete operation for %s: %w", blobPath, err) + } + } + + // Submit batch + responses, err := d.containerClient.SubmitBatch(ctx, batchBuilder, nil) + if err != nil { + return fmt.Errorf("batch delete request failed: %w", err) + } + + // Check responses + for _, resp := range responses.Responses { + if resp.Error != nil && !isNotFoundError(resp.Error) { + // 获取 blob 名称以提供更好的错误信息 + blobName := "unknown" + if resp.BlobName != nil { + blobName = *resp.BlobName + } + return fmt.Errorf("failed to delete blob %s: %v", blobName, resp.Error) + } + } + } + + return nil +} + +// deleteFolder recursively deletes a directory and all its contents +func (d *AzureBlob) deleteFolder(ctx context.Context, prefix string) error { + // Ensure directory path ends with slash + prefix = ensureTrailingSlash(prefix) + + // Get all blobs under the directory using flattenListBlobs + globs, err := d.flattenListBlobs(ctx, prefix) + if err != nil { + return fmt.Errorf("failed to list blobs for deletion: %w", err) + } + + // If there are blobs in the directory, delete them + if len(globs) > 0 { + // 分离文件和目录标记 + var filePaths []string + var dirPaths []string + + for _, blob := range globs { + blobName := *blob.Name + if isDirectory(blob) { + // remove trailing slash for directory names + dirPaths = append(dirPaths, strings.TrimSuffix(blobName, "/")) + } else { + filePaths = append(filePaths, blobName) + } + } + + // 先删除文件，再删除目录 + if len(filePaths) > 0 { + if err := d.batchDeleteBlobs(ctx, filePaths); err != nil { + return err + } + } + if len(dirPaths) > 0 { + // 按路径深度分组 + depthMap := make(map[int][]string) + for _, dir := range dirPaths { + depth := strings.Count(dir, "/") // 计算目录深度 + depthMap[depth] = append(depthMap[depth], dir) + } + + // 按深度从大到小排序 + var depths []int + for depth := range depthMap { + depths = append(depths, depth) + } + sort.Sort(sort.Reverse(sort.IntSlice(depths))) + + // 按深度逐层批量删除 + for _, depth := range depths { + batch := depthMap[depth] + if err := d.batchDeleteBlobs(ctx, batch); err != nil { + return err + } + } + } + } + + // 最后删除目录标记本身 + return d.deleteEmptyDirectory(ctx, prefix) +} + +// deleteFile deletes a single file or blob with better error handling +func (d *AzureBlob) deleteFile(ctx context.Context, path string, isDir bool) error { + blobClient := d.containerClient.NewBlobClient(path) + _, err := blobClient.Delete(ctx, nil) + if err != nil && !(isDir && isNotFoundError(err)) { + return err + } + return nil +} + +// copyFile copies a single blob from source path to destination path +func (d *AzureBlob) copyFile(ctx context.Context, srcPath, dstPath string) error { + srcBlob := d.containerClient.NewBlobClient(srcPath) + dstBlob := d.containerClient.NewBlobClient(dstPath) + + // Use configured expiration time for SAS URL + expireDuration := time.Hour * time.Duration(d.SignURLExpire) + srcURL, err := srcBlob.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expireDuration), nil) + if err != nil { + return fmt.Errorf("failed to generate source SAS URL: %w", err) + } + + _, err = dstBlob.StartCopyFromURL(ctx, srcURL, nil) + return err + +} + +// createContainerIfNotExists - Create container if not exists +// Clean up commented code +func (d *AzureBlob) createContainerIfNotExists(ctx context.Context, containerName string) error { + serviceClient := d.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(containerName) + + var options = service.CreateContainerOptions{} + _, err := containerClient.Create(ctx, &options) + if err != nil { + var responseErr *azcore.ResponseError + if errors.As(err, &responseErr) && responseErr.ErrorCode != "ContainerAlreadyExists" { + return fmt.Errorf("failed to create or access container [%s]: %w", containerName, err) + } + } + + d.containerClient = containerClient + return nil +} + +// mkDir creates a virtual directory marker by uploading an empty blob with metadata. +func (d *AzureBlob) mkDir(ctx context.Context, fullDirName string) error { + dirPath := ensureTrailingSlash(fullDirName) + blobClient := d.containerClient.NewBlockBlobClient(dirPath) + + // Upload an empty blob with metadata indicating it's a directory + _, err := blobClient.Upload(ctx, struct { + *bytes.Reader + io.Closer + }{ + Reader: bytes.NewReader([]byte{}), + Closer: io.NopCloser(nil), + }, &blockblob.UploadOptions{ + Metadata: map[string]*string{ + "hdi_isfolder": to.Ptr("true"), + }, + }) + return err +} + +// ensureTrailingSlash ensures the provided path ends with a trailing slash. +func ensureTrailingSlash(path string) string { + if !strings.HasSuffix(path, "/") { + return path + "/" + } + return path +} + +// moveOrRename moves or renames blobs or directories from source to destination. +func (d *AzureBlob) moveOrRename(ctx context.Context, srcPath, dstPath string, isDir bool, srcSize int64) error { + if isDir { + // Normalize paths for directory operations + srcPath = ensureTrailingSlash(srcPath) + dstPath = ensureTrailingSlash(dstPath) + + // List all blobs under the source directory + blobs, err := d.flattenListBlobs(ctx, srcPath) + if err != nil { + return fmt.Errorf("failed to list blobs: %w", err) + } + + // Iterate and copy each blob to the destination + for _, item := range blobs { + srcBlobName := *item.Name + relPath := strings.TrimPrefix(srcBlobName, srcPath) + itemDstPath := path.Join(dstPath, relPath) + + if isDirectory(item) { + // Create directory marker at destination + if err := d.mkDir(ctx, itemDstPath); err != nil { + return fmt.Errorf("failed to create directory marker [%s]: %w", itemDstPath, err) + } + } else { + // Copy file blob to destination + if err := d.copyFile(ctx, srcBlobName, itemDstPath); err != nil { + return fmt.Errorf("failed to copy blob [%s]: %w", srcBlobName, err) + } + } + } + + // Handle empty directories by creating a marker at destination + if len(blobs) == 0 { + if err := d.mkDir(ctx, dstPath); err != nil { + return fmt.Errorf("failed to create directory [%s]: %w", dstPath, err) + } + } + + // Delete source directory and its contents + if err := d.deleteFolder(ctx, srcPath); err != nil { + log.Warnf("failed to delete source directory [%s]: %v\n, and try again", srcPath, err) + // Retry deletion once more and ignore the result + if err := d.deleteFolder(ctx, srcPath); err != nil { + log.Errorf("Retry deletion of source directory [%s] failed: %v", srcPath, err) + } + } + + return nil + } + + // Single file move or rename operation + if err := d.copyFile(ctx, srcPath, dstPath); err != nil { + return fmt.Errorf("failed to copy file: %w", err) + } + + // Delete source file after successful copy + if err := d.deleteFile(ctx, srcPath, false); err != nil { + log.Errorf("Error deleting source file [%s]: %v", srcPath, err) + } + return nil +} + +// optimizedUploadOptions returns the optimal upload options based on file size +func optimizedUploadOptions(fileSize int64) *azblob.UploadStreamOptions { + options := &azblob.UploadStreamOptions{ + BlockSize: 4 * 1024 * 1024, // 4MB block size + Concurrency: 4, // Default concurrency + } + + // For large files, increase block size and concurrency + if fileSize > 256*1024*1024 { // For files larger than 256MB + options.BlockSize = 8 * 1024 * 1024 // 8MB blocks + options.Concurrency = 8 // More concurrent uploads + } + + // For very large files (>1GB) + if fileSize > 1024*1024*1024 { + options.BlockSize = 16 * 1024 * 1024 // 16MB blocks + options.Concurrency = 16 // Higher concurrency + } + + return options +} + +// isDirectory determines if a blob represents a directory +// Checks multiple indicators: path suffix, metadata, and content type +func isDirectory(blob container.BlobItem) bool { + // Check path suffix + if strings.HasSuffix(*blob.Name, "/") { + return true + } + + // Check metadata for directory marker + if blob.Metadata != nil { + if val, ok := blob.Metadata["hdi_isfolder"]; ok && val != nil && *val == "true" { + return true + } + // Azure Storage Explorer and other tools may use different metadata keys + if val, ok := blob.Metadata["is_directory"]; ok && val != nil && strings.ToLower(*val) == "true" { + return true + } + } + + // Check content type (some tools mark directories with specific content types) + if blob.Properties != nil && blob.Properties.ContentType != nil { + contentType := strings.ToLower(*blob.Properties.ContentType) + if blob.Properties.ContentLength != nil && *blob.Properties.ContentLength == 0 && (contentType == "application/directory" || contentType == "directory") { + return true + } + } + + return false +} + +// deleteEmptyDirectory deletes a directory only if it's empty +func (d *AzureBlob) deleteEmptyDirectory(ctx context.Context, dirPath string) error { + // Directory is empty, delete the directory marker + blobClient := d.containerClient.NewBlobClient(strings.TrimSuffix(dirPath, "/")) + _, err := blobClient.Delete(ctx, nil) + + // Also try deleting with trailing slash (for different directory marker formats) + if err != nil && isNotFoundError(err) { + blobClient = d.containerClient.NewBlobClient(dirPath) + _, err = blobClient.Delete(ctx, nil) + } + + // Ignore not found errors + if err != nil && isNotFoundError(err) { + log.Infof("Directory [%s] not found during deletion: %v", dirPath, err) + return nil + } + + return err +} diff --git a/go.mod b/go.mod index f8a238f143e..97a477d33b1 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,12 @@ require ( gorm.io/gorm v1.25.11 ) +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect +) + require ( github.com/STARRY-S/zip v0.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect diff --git a/go.sum b/go.sum index 1681a3a0471..86fb779e5d8 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,12 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= From ab68faef444a04f60247ee93be4442ec84104769 Mon Sep 17 00:00:00 2001 From: asdfghjkl <61342682+anobodys@users.noreply.github.com> Date: Thu, 3 Apr 2025 20:44:49 +0800 Subject: [PATCH 492/659] fix(baidu_netdisk): add another video crack api (#8275) Co-authored-by: anobodys --- drivers/baidu_netdisk/driver.go | 2 ++ drivers/baidu_netdisk/meta.go | 3 ++- drivers/baidu_netdisk/types.go | 2 +- drivers/baidu_netdisk/util.go | 45 ++++++++++++++++++++++++++++++++- 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 4397d413632..3cc1ae9ed97 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -78,6 +78,8 @@ func (d *BaiduNetdisk) List(ctx context.Context, dir model.Obj, args model.ListA func (d *BaiduNetdisk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if d.DownloadAPI == "crack" { return d.linkCrack(file, args) + } else if d.DownloadAPI == "crack_video" { + return d.linkCrackVideo(file, args) } return d.linkOfficial(file, args) } diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index e9226a0d37a..27571056e11 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -10,7 +10,7 @@ type Addition struct { driver.RootPath OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` + DownloadAPI string `json:"download_api" type:"select" options:"official,crack,crack_video" default:"official"` ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` @@ -19,6 +19,7 @@ type Addition struct { UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` + OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` } var config = driver.Config{ diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index 728273b8dab..ed9b09df8ee 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -17,7 +17,7 @@ type TokenErrResp struct { type File struct { //TkbindId int `json:"tkbind_id"` //OwnerType int `json:"owner_type"` - //Category int `json:"category"` + Category int `json:"category"` //RealCategory string `json:"real_category"` FsId int64 `json:"fs_id"` //OperId int `json:"oper_id"` diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index a4fc13f8514..1249b3f470f 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -79,6 +79,12 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall return retry.Unrecoverable(err2) } } + + if 31023 == errno && d.DownloadAPI == "crack_video" { + result = res.Body() + return nil + } + return fmt.Errorf("req: [%s] ,errno: %d, refer to https://pan.baidu.com/union/doc/", furl, errno) } result = res.Body() @@ -131,7 +137,16 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { if len(resp.List) == 0 { break } - res = append(res, resp.List...) + + if d.OnlyListVideoFile { + for _, file := range resp.List { + if file.Isdir == 1 || file.Category == 1 { + res = append(res, file) + } + } + } else { + res = append(res, resp.List...) + } } return res, nil } @@ -187,6 +202,34 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, _ model.LinkArgs) (*model.Link, }, nil } +func (d *BaiduNetdisk) linkCrackVideo(file model.Obj, _ model.LinkArgs) (*model.Link, error) { + param := map[string]string{ + "type": "VideoURL", + "path": fmt.Sprintf("%s", file.GetPath()), + "fs_id": file.GetID(), + "devuid": "0%1", + "clienttype": "1", + "channel": "android_15_25010PN30C_bd-netdisk_1523a", + "nom3u8": "1", + "dlink": "1", + "media": "1", + "origin": "dlna", + } + resp, err := d.request("https://pan.baidu.com/api/mediainfo", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(param) + }, nil) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: utils.Json.Get(resp, "info", "dlink").ToString(), + Header: http.Header{ + "User-Agent": []string{d.CustomCrackUA}, + }, + }, nil +} + func (d *BaiduNetdisk) manage(opera string, filelist any) ([]byte, error) { params := map[string]string{ "method": "filemanager", From 3375c26c413ff31190a59fab5e40696348c099e9 Mon Sep 17 00:00:00 2001 From: xiaoQQya <46475319+xiaoQQya@users.noreply.github.com> Date: Thu, 3 Apr 2025 20:50:29 +0800 Subject: [PATCH 493/659] perf(quark_uc&quark_uc_tv): native proxy multithreading (#8287) * perf(quark_uc): native proxy multithreading * perf(quark_uc_tv): native proxy multithreading * chore(fs): file query result add id --- drivers/quark_uc/driver.go | 2 +- drivers/quark_uc_tv/driver.go | 9 ++++++--- server/handles/fsread.go | 6 ++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 04757b1b1cf..0f8884fac53 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -74,7 +74,7 @@ func (d *QuarkOrUC) Link(ctx context.Context, file model.Obj, args model.LinkArg "Referer": []string{d.conf.referer}, "User-Agent": []string{ua}, }, - Concurrency: 2, + Concurrency: 3, PartSize: 10 * utils.MB, }, nil } diff --git a/drivers/quark_uc_tv/driver.go b/drivers/quark_uc_tv/driver.go index ff7ccf20f7a..a857e2dd841 100644 --- a/drivers/quark_uc_tv/driver.go +++ b/drivers/quark_uc_tv/driver.go @@ -125,7 +125,6 @@ func (d *QuarkUCTV) List(ctx context.Context, dir model.Obj, args model.ListArgs } func (d *QuarkUCTV) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - files := &model.Link{} var fileLink FileLink _, err := d.request(ctx, "/file", "GET", func(req *resty.Request) { req.SetQueryParams(map[string]string{ @@ -139,8 +138,12 @@ func (d *QuarkUCTV) Link(ctx context.Context, file model.Obj, args model.LinkArg if err != nil { return nil, err } - files.URL = fileLink.Data.DownloadURL - return files, nil + + return &model.Link{ + URL: fileLink.Data.DownloadURL, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil } func (d *QuarkUCTV) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 0a62f1ffba2..73bde23b6de 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -33,6 +33,8 @@ type DirReq struct { } type ObjResp struct { + Id string `json:"id"` + Path string `json:"path"` Name string `json:"name"` Size int64 `json:"size"` IsDir bool `json:"is_dir"` @@ -210,6 +212,8 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { for _, obj := range objs { thumb, _ := model.GetThumb(obj) resp = append(resp, ObjResp{ + Id: obj.GetID(), + Path: obj.GetPath(), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), @@ -326,6 +330,8 @@ func FsGet(c *gin.Context) { thumb, _ := model.GetThumb(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ + Id: obj.GetID(), + Path: obj.GetPath(), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), From ddffacf07b8a63a3e065bdac7c6ea1b2ec63bdc4 Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sat, 12 Apr 2025 16:55:31 +0800 Subject: [PATCH 494/659] perf: optimize IO read/write usage (#8243) * perf: optimize IO read/write usage * . * Update drivers/139/driver.go Co-authored-by: MadDogOwner --------- Co-authored-by: MadDogOwner --- drivers/115/util.go | 2 +- drivers/123/driver.go | 21 +----- drivers/123/upload.go | 34 +++++---- drivers/139/driver.go | 106 +++++++++++++--------------- drivers/139/util.go | 1 + drivers/189pc/utils.go | 108 ++++++++++++++++++----------- drivers/aliyundrive_open/upload.go | 24 +++---- drivers/baidu_netdisk/driver.go | 46 +++++++++--- drivers/baidu_photo/driver.go | 50 ++++++++++--- drivers/cloudreve/util.go | 6 +- drivers/github/util.go | 3 +- drivers/ilanzou/driver.go | 27 +++----- drivers/mopan/driver.go | 3 - drivers/netease_music/util.go | 1 - drivers/onedrive/util.go | 2 +- drivers/onedrive_app/util.go | 2 +- drivers/pikpak/util.go | 16 ++--- drivers/quark_uc/driver.go | 77 ++++++++++---------- drivers/thunder/driver.go | 14 ++-- drivers/thunder_browser/driver.go | 21 +++--- drivers/thunderx/driver.go | 19 +++-- internal/archive/archives/utils.go | 3 +- internal/archive/iso9660/utils.go | 11 +-- internal/fs/archive.go | 8 ++- internal/model/obj.go | 4 +- internal/net/request.go | 3 +- internal/stream/stream.go | 87 ++++++++++------------- internal/stream/util.go | 43 ++++++++++++ server/handles/fsup.go | 30 ++++---- 29 files changed, 429 insertions(+), 343 deletions(-) diff --git a/drivers/115/util.go b/drivers/115/util.go index 7298f565d0c..fc17fe3cebf 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -405,7 +405,7 @@ func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.Upload if _, err = tmpF.ReadAt(buf, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { continue } - if part, err = bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(buf)), + if part, err = bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, bytes.NewReader(buf)), chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { break } diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 7d457138fde..32c053e22ab 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -2,11 +2,8 @@ package _123 import ( "context" - "crypto/md5" "encoding/base64" - "encoding/hex" "fmt" - "io" "net/http" "net/url" "sync" @@ -18,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" @@ -187,25 +185,12 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error { func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { etag := file.GetHash().GetHash(utils.MD5) + var err error if len(etag) < utils.MD5.Width { - // const DEFAULT int64 = 10485760 - h := md5.New() - // need to calculate md5 of the full content - tempFile, err := file.CacheFullInTempFile() + _, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5) if err != nil { return err } - defer func() { - _ = tempFile.Close() - }() - if _, err = utils.CopyWithBuffer(h, tempFile); err != nil { - return err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err - } - etag = hex.EncodeToString(h.Sum(nil)) } data := base.Json{ "driveId": 0, diff --git a/drivers/123/upload.go b/drivers/123/upload.go index dc148c4c93f..b0482a9f4c9 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "math" "net/http" "strconv" @@ -70,27 +69,33 @@ func (d *Pan123) completeS3(ctx context.Context, upReq *UploadResp, file model.F } func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, up driver.UpdateProgress) error { - chunkSize := int64(1024 * 1024 * 16) + tmpF, err := file.CacheFullInTempFile() + if err != nil { + return err + } // fetch s3 pre signed urls - chunkCount := int(math.Ceil(float64(file.GetSize()) / float64(chunkSize))) + size := file.GetSize() + chunkSize := min(size, 16*utils.MB) + chunkCount := int(size / chunkSize) + lastChunkSize := size % chunkSize + if lastChunkSize > 0 { + chunkCount++ + } else { + lastChunkSize = chunkSize + } // only 1 batch is allowed - isMultipart := chunkCount > 1 batchSize := 1 getS3UploadUrl := d.getS3Auth - if isMultipart { + if chunkCount > 1 { batchSize = 10 getS3UploadUrl = d.getS3PreSignedUrls } - limited := driver.NewLimitedUploadStream(ctx, file) for i := 1; i <= chunkCount; i += batchSize { if utils.IsCanceled(ctx) { return ctx.Err() } start := i - end := i + batchSize - if end > chunkCount+1 { - end = chunkCount + 1 - } + end := min(i+batchSize, chunkCount+1) s3PreSignedUrls, err := getS3UploadUrl(ctx, upReq, start, end) if err != nil { return err @@ -102,9 +107,9 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi } curSize := chunkSize if j == chunkCount { - curSize = file.GetSize() - (int64(chunkCount)-1)*chunkSize + curSize = lastChunkSize } - err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.LimitReader(limited, chunkSize), curSize, false, getS3UploadUrl) + err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.NewSectionReader(tmpF, chunkSize*int64(j-1), curSize), curSize, false, getS3UploadUrl) if err != nil { return err } @@ -115,12 +120,12 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi return d.completeS3(ctx, upReq, file, chunkCount > 1) } -func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSignedUrls *S3PreSignedURLs, cur, end int, reader io.Reader, curSize int64, retry bool, getS3UploadUrl func(ctx context.Context, upReq *UploadResp, start int, end int) (*S3PreSignedURLs, error)) error { +func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSignedUrls *S3PreSignedURLs, cur, end int, reader *io.SectionReader, curSize int64, retry bool, getS3UploadUrl func(ctx context.Context, upReq *UploadResp, start int, end int) (*S3PreSignedURLs, error)) error { uploadUrl := s3PreSignedUrls.Data.PreSignedUrls[strconv.Itoa(cur)] if uploadUrl == "" { return fmt.Errorf("upload url is empty, s3PreSignedUrls: %+v", s3PreSignedUrls) } - req, err := http.NewRequest("PUT", uploadUrl, reader) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } @@ -143,6 +148,7 @@ func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSign } s3PreSignedUrls.Data.PreSignedUrls = newS3PreSignedUrls.Data.PreSignedUrls // retry + reader.Seek(0, io.SeekStart) return d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, cur, end, reader, curSize, true, getS3UploadUrl) } if res.StatusCode != http.StatusOK { diff --git a/drivers/139/driver.go b/drivers/139/driver.go index f367c431c1b..0af5a4f781a 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -2,20 +2,19 @@ package _139 import ( "context" - "encoding/base64" "encoding/xml" "fmt" "io" "net/http" "path" "strconv" - "strings" "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + streamPkg "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" @@ -72,28 +71,29 @@ func (d *Yun139) Init(ctx context.Context) error { default: return errs.NotImplement } - if d.ref != nil { - return nil - } - decode, err := base64.StdEncoding.DecodeString(d.Authorization) - if err != nil { - return err - } - decodeStr := string(decode) - splits := strings.Split(decodeStr, ":") - if len(splits) < 2 { - return fmt.Errorf("authorization is invalid, splits < 2") - } - d.Account = splits[1] - _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ - "qryUserExternInfoReq": base.Json{ - "commonAccountInfo": base.Json{ - "account": d.getAccount(), - "accountType": 1, - }, - }, - }, nil) - return err + // if d.ref != nil { + // return nil + // } + // decode, err := base64.StdEncoding.DecodeString(d.Authorization) + // if err != nil { + // return err + // } + // decodeStr := string(decode) + // splits := strings.Split(decodeStr, ":") + // if len(splits) < 2 { + // return fmt.Errorf("authorization is invalid, splits < 2") + // } + // d.Account = splits[1] + // _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ + // "qryUserExternInfoReq": base.Json{ + // "commonAccountInfo": base.Json{ + // "account": d.getAccount(), + // "accountType": 1, + // }, + // }, + // }, nil) + // return err + return nil } func (d *Yun139) InitReference(storage driver.Driver) error { @@ -503,23 +503,15 @@ func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { } } -const ( - _ = iota //ignore first value by assigning to blank identifier - KB = 1 << (10 * iota) - MB - GB - TB -) - func (d *Yun139) getPartSize(size int64) int64 { if d.CustomUploadPartSize != 0 { return d.CustomUploadPartSize } // 网盘对于分片数量存在上限 - if size/GB > 30 { - return 512 * MB + if size/utils.GB > 30 { + return 512 * utils.MB } - return 100 * MB + return 100 * utils.MB } func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { @@ -527,29 +519,28 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr case MetaPersonalNew: var err error fullHash := stream.GetHash().GetHash(utils.SHA256) - if len(fullHash) <= 0 { - tmpF, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - fullHash, err = utils.HashFile(utils.SHA256, tmpF) + if len(fullHash) != utils.SHA256.Width { + _, fullHash, err = streamPkg.CacheFullInTempFileAndHash(stream, utils.SHA256) if err != nil { return err } } - partInfos := []PartInfo{} - var partSize = d.getPartSize(stream.GetSize()) - part := (stream.GetSize() + partSize - 1) / partSize - if part == 0 { + size := stream.GetSize() + var partSize = d.getPartSize(size) + part := size / partSize + if size%partSize > 0 { + part++ + } else if part == 0 { part = 1 } + partInfos := make([]PartInfo, 0, part) for i := int64(0); i < part; i++ { if utils.IsCanceled(ctx) { return ctx.Err() } start := i * partSize - byteSize := stream.GetSize() - start + byteSize := size - start if byteSize > partSize { byteSize = partSize } @@ -577,7 +568,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "contentType": "application/octet-stream", "parallelUpload": false, "partInfos": firstPartInfos, - "size": stream.GetSize(), + "size": size, "parentFileId": dstDir.GetID(), "name": stream.GetName(), "type": "file", @@ -630,7 +621,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } // Progress - p := driver.NewProgress(stream.GetSize(), up) + p := driver.NewProgress(size, up) rateLimited := driver.NewLimitedUploadStream(ctx, stream) // 上传所有分片 @@ -790,12 +781,14 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return fmt.Errorf("get file upload url failed with result code: %s, message: %s", resp.Data.Result.ResultCode, resp.Data.Result.ResultDesc) } + size := stream.GetSize() // Progress - p := driver.NewProgress(stream.GetSize(), up) - - var partSize = d.getPartSize(stream.GetSize()) - part := (stream.GetSize() + partSize - 1) / partSize - if part == 0 { + p := driver.NewProgress(size, up) + var partSize = d.getPartSize(size) + part := size / partSize + if size%partSize > 0 { + part++ + } else if part == 0 { part = 1 } rateLimited := driver.NewLimitedUploadStream(ctx, stream) @@ -805,10 +798,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } start := i * partSize - byteSize := stream.GetSize() - start - if byteSize > partSize { - byteSize = partSize - } + byteSize := min(size-start, partSize) limitReader := io.LimitReader(rateLimited, byteSize) // Update Progress @@ -820,7 +810,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr req = req.WithContext(ctx) req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) - req.Header.Set("contentSize", strconv.FormatInt(stream.GetSize(), 10)) + req.Header.Set("contentSize", strconv.FormatInt(size, 10)) req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) req.Header.Set("rangeType", "0") diff --git a/drivers/139/util.go b/drivers/139/util.go index 3e1a61edc81..53defef528e 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -67,6 +67,7 @@ func (d *Yun139) refreshToken() error { if len(splits) < 3 { return fmt.Errorf("authorization is invalid, splits < 3") } + d.Account = splits[1] strs := strings.Split(splits[2], "|") if len(strs) < 4 { return fmt.Errorf("authorization is invalid, strs < 4") diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index fb1a183ab38..c391f7e676f 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -3,16 +3,15 @@ package _189pc import ( "bytes" "context" - "crypto/md5" "encoding/base64" "encoding/hex" "encoding/xml" "fmt" "io" - "math" "net/http" "net/http/cookiejar" "net/url" + "os" "regexp" "sort" "strconv" @@ -28,6 +27,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/errgroup" "github.com/alist-org/alist/v3/pkg/utils" @@ -473,12 +473,8 @@ func (y *Cloud189PC) refreshSession() (err error) { // 普通上传 // 无法上传大小为0的文件 func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { - var sliceSize = partSize(file.GetSize()) - count := int(math.Ceil(float64(file.GetSize()) / float64(sliceSize))) - lastPartSize := file.GetSize() % sliceSize - if file.GetSize() > 0 && lastPartSize == 0 { - lastPartSize = sliceSize - } + size := file.GetSize() + sliceSize := partSize(size) params := Params{ "parentFolderId": dstDir.GetID(), @@ -512,22 +508,29 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo retry.DelayType(retry.BackOffDelay)) sem := semaphore.NewWeighted(3) - fileMd5 := md5.New() - silceMd5 := md5.New() + count := int(size / sliceSize) + lastPartSize := size % sliceSize + if lastPartSize > 0 { + count++ + } else { + lastPartSize = sliceSize + } + fileMd5 := utils.MD5.NewFunc() + silceMd5 := utils.MD5.NewFunc() silceMd5Hexs := make([]string, 0, count) - + teeReader := io.TeeReader(file, io.MultiWriter(fileMd5, silceMd5)) + byteSize := sliceSize for i := 1; i <= count; i++ { if utils.IsCanceled(upCtx) { break } - byteData := make([]byte, sliceSize) if i == count { - byteData = byteData[:lastPartSize] + byteSize = lastPartSize } - + byteData := make([]byte, byteSize) // 读取块 silceMd5.Reset() - if _, err := io.ReadFull(io.TeeReader(file, io.MultiWriter(fileMd5, silceMd5)), byteData); err != io.EOF && err != nil { + if _, err := io.ReadFull(teeReader, byteData); err != io.EOF && err != nil { sem.Release(1) return nil, err } @@ -607,24 +610,43 @@ func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream m // 快传 func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { - tempFile, err := file.CacheFullInTempFile() - if err != nil { - return nil, err - } - - var sliceSize = partSize(file.GetSize()) - count := int(math.Ceil(float64(file.GetSize()) / float64(sliceSize))) - lastSliceSize := file.GetSize() % sliceSize - if file.GetSize() > 0 && lastSliceSize == 0 { + var ( + cache = file.GetFile() + tmpF *os.File + err error + ) + size := file.GetSize() + if _, ok := cache.(io.ReaderAt); !ok && size > 0 { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF + } + sliceSize := partSize(size) + count := int(size / sliceSize) + lastSliceSize := size % sliceSize + if lastSliceSize > 0 { + count++ + } else { lastSliceSize = sliceSize } //step.1 优先计算所需信息 byteSize := sliceSize - fileMd5 := md5.New() - silceMd5 := md5.New() - silceMd5Hexs := make([]string, 0, count) + fileMd5 := utils.MD5.NewFunc() + sliceMd5 := utils.MD5.NewFunc() + sliceMd5Hexs := make([]string, 0, count) partInfos := make([]string, 0, count) + writers := []io.Writer{fileMd5, sliceMd5} + if tmpF != nil { + writers = append(writers, tmpF) + } + written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() @@ -634,19 +656,31 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode byteSize = lastSliceSize } - silceMd5.Reset() - if _, err := utils.CopyWithBufferN(io.MultiWriter(fileMd5, silceMd5), tempFile, byteSize); err != nil && err != io.EOF { + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), file, byteSize) + written += n + if err != nil && err != io.EOF { return nil, err } - md5Byte := silceMd5.Sum(nil) - silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte))) + md5Byte := sliceMd5.Sum(nil) + sliceMd5Hexs = append(sliceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte))) partInfos = append(partInfos, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte))) + sliceMd5.Reset() + } + + if tmpF != nil { + if size > 0 && written != size { + return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, size) + } + _, err = tmpF.Seek(0, io.SeekStart) + if err != nil { + return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") + } } fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) sliceMd5Hex := fileMd5Hex - if file.GetSize() > sliceSize { - sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(silceMd5Hexs, "\n"))) + if size > sliceSize { + sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(sliceMd5Hexs, "\n"))) } fullUrl := UPLOAD_URL @@ -712,7 +746,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } // step.4 上传切片 - _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, io.NewSectionReader(tempFile, offset, byteSize), isFamily) + _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, io.NewSectionReader(cache, offset, byteSize), isFamily) if err != nil { return err } @@ -794,11 +828,7 @@ func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, isFamily bool, uplo // 旧版本上传，家庭云不支持覆盖 func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { - tempFile, err := file.CacheFullInTempFile() - if err != nil { - return nil, err - } - fileMd5, err := utils.HashFile(utils.MD5, tempFile) + tempFile, fileMd5, err := stream.CacheFullInTempFileAndHash(file, utils.MD5) if err != nil { return nil, err } diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index fb730de6966..4114c195182 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -1,7 +1,6 @@ package aliyundrive_open import ( - "bytes" "context" "encoding/base64" "fmt" @@ -15,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + streamPkg "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" @@ -131,16 +131,19 @@ func (d *AliyundriveOpen) calProofCode(stream model.FileStreamer) (string, error return "", err } length := proofRange.End - proofRange.Start - buf := bytes.NewBuffer(make([]byte, 0, length)) reader, err := stream.RangeRead(http_range.Range{Start: proofRange.Start, Length: length}) if err != nil { return "", err } - _, err = utils.CopyWithBufferN(buf, reader, length) + buf := make([]byte, length) + n, err := io.ReadFull(reader, buf) + if err == io.ErrUnexpectedEOF { + return "", fmt.Errorf("can't read data, expected=%d, got=%d", len(buf), n) + } if err != nil { return "", err } - return base64.StdEncoding.EncodeToString(buf.Bytes()), nil + return base64.StdEncoding.EncodeToString(buf), nil } func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { @@ -183,25 +186,18 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m _, err, e := d.requestReturnErrResp("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) - var tmpF model.File if err != nil { if e.Code != "PreHashMatched" || !rapidUpload { return nil, err } log.Debugf("[aliyundrive_open] pre_hash matched, start rapid upload") - hi := stream.GetHash() - hash := hi.GetHash(utils.SHA1) - if len(hash) <= 0 { - tmpF, err = stream.CacheFullInTempFile() + hash := stream.GetHash().GetHash(utils.SHA1) + if len(hash) != utils.SHA1.Width { + _, hash, err = streamPkg.CacheFullInTempFileAndHash(stream, utils.SHA1) if err != nil { return nil, err } - hash, err = utils.HashFile(utils.SHA1, tmpF) - if err != nil { - return nil, err - } - } delete(createData, "pre_hash") diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 3cc1ae9ed97..c33e0b32b05 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -6,8 +6,8 @@ import ( "encoding/hex" "errors" "io" - "math" "net/url" + "os" stdpath "path" "strconv" "time" @@ -15,6 +15,7 @@ import ( "golang.org/x/sync/semaphore" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -185,16 +186,30 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F return newObj, nil } - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return nil, err + var ( + cache = stream.GetFile() + tmpF *os.File + err error + ) + if _, ok := cache.(io.ReaderAt); !ok { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF } streamSize := stream.GetSize() sliceSize := d.getSliceSize(streamSize) - count := int(math.Max(math.Ceil(float64(streamSize)/float64(sliceSize)), 1)) + count := int(streamSize / sliceSize) lastBlockSize := streamSize % sliceSize - if streamSize > 0 && lastBlockSize == 0 { + if lastBlockSize > 0 { + count++ + } else { lastBlockSize = sliceSize } @@ -207,6 +222,11 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F sliceMd5H := md5.New() sliceMd5H2 := md5.New() slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) + writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} + if tmpF != nil { + writers = append(writers, tmpF) + } + written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { @@ -215,13 +235,23 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if i == count { byteSize = lastBlockSize } - _, err := utils.CopyWithBufferN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize) + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) + written += n if err != nil && err != io.EOF { return nil, err } blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) sliceMd5H.Reset() } + if tmpF != nil { + if written != streamSize { + return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + } + _, err = tmpF.Seek(0, io.SeekStart) + if err != nil { + return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") + } + } contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(blockList) @@ -291,7 +321,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F "partseq": strconv.Itoa(partseq), } err := d.uploadSlice(ctx, params, stream.GetName(), - driver.NewLimitedUploadStream(ctx, io.NewSectionReader(tempFile, offset, byteSize))) + driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) if err != nil { return err } diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index eeee746f71d..5a34fcb4639 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -7,7 +7,7 @@ import ( "errors" "fmt" "io" - "math" + "os" "regexp" "strconv" "strings" @@ -16,6 +16,7 @@ import ( "golang.org/x/sync/semaphore" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -241,11 +242,21 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil // TODO: // 暂时没有找到妙传方式 - - // 需要获取完整文件md5,必须支持 io.Seek - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return nil, err + var ( + cache = stream.GetFile() + tmpF *os.File + err error + ) + if _, ok := cache.(io.ReaderAt); !ok { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF } const DEFAULT int64 = 1 << 22 @@ -253,9 +264,11 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil // 计算需要的数据 streamSize := stream.GetSize() - count := int(math.Ceil(float64(streamSize) / float64(DEFAULT))) + count := int(streamSize / DEFAULT) lastBlockSize := streamSize % DEFAULT - if lastBlockSize == 0 { + if lastBlockSize > 0 { + count++ + } else { lastBlockSize = DEFAULT } @@ -266,6 +279,11 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil sliceMd5H := md5.New() sliceMd5H2 := md5.New() slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) + writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} + if tmpF != nil { + writers = append(writers, tmpF) + } + written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() @@ -273,13 +291,23 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if i == count { byteSize = lastBlockSize } - _, err := utils.CopyWithBufferN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize) + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) + written += n if err != nil && err != io.EOF { return nil, err } sliceMD5List = append(sliceMD5List, hex.EncodeToString(sliceMd5H.Sum(nil))) sliceMd5H.Reset() } + if tmpF != nil { + if written != streamSize { + return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + } + _, err = tmpF.Seek(0, io.SeekStart) + if err != nil { + return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") + } + } contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(sliceMD5List) @@ -291,7 +319,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil "rtype": "1", "ctype": "11", "path": fmt.Sprintf("/%s", stream.GetName()), - "size": fmt.Sprint(stream.GetSize()), + "size": fmt.Sprint(streamSize), "slice-md5": sliceMd5, "content-md5": contentMd5, "block_list": blockListStr, @@ -343,7 +371,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil r.SetContext(ctx) r.SetQueryParams(uploadParams) r.SetFileReader("file", stream.GetName(), - driver.NewLimitedUploadStream(ctx, io.NewSectionReader(tempFile, offset, byteSize))) + driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) }, nil) if err != nil { return err diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 1fd5ed8abae..196d7303337 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -204,7 +204,7 @@ func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u Up req.SetContentLength(true) req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) req.SetHeader("User-Agent", d.getUA()) - req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) + req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) }, nil) if err != nil { break @@ -239,7 +239,7 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U return err } req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), - driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { return err } @@ -280,7 +280,7 @@ func (d *Cloudreve) upOneDrive(ctx context.Context, stream model.FileStreamer, u if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { return err } diff --git a/drivers/github/util.go b/drivers/github/util.go index 03318784f72..7ddf8746c8f 100644 --- a/drivers/github/util.go +++ b/drivers/github/util.go @@ -5,7 +5,6 @@ import ( "context" "errors" "fmt" - "io" "strings" "text/template" "time" @@ -159,7 +158,7 @@ func signCommit(m *map[string]interface{}, entity *openpgp.Entity) (string, erro if err != nil { return "", err } - if _, err = io.Copy(armorWriter, &sigBuffer); err != nil { + if _, err = utils.CopyWithBuffer(armorWriter, &sigBuffer); err != nil { return "", err } _ = armorWriter.Close() diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go index 39a311ddbc0..044193d3584 100644 --- a/drivers/ilanzou/driver.go +++ b/drivers/ilanzou/driver.go @@ -2,7 +2,6 @@ package template import ( "context" - "crypto/md5" "encoding/base64" "encoding/hex" "fmt" @@ -17,6 +16,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/foxxorcat/mopan-sdk-go" "github.com/go-resty/resty/v2" @@ -273,23 +273,14 @@ func (d *ILanZou) Remove(ctx context.Context, obj model.Obj) error { const DefaultPartSize = 1024 * 1024 * 8 func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - h := md5.New() - // need to calculate md5 of the full content - tempFile, err := s.CacheFullInTempFile() - if err != nil { - return nil, err - } - defer func() { - _ = tempFile.Close() - }() - if _, err = utils.CopyWithBuffer(h, tempFile); err != nil { - return nil, err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return nil, err + etag := s.GetHash().GetHash(utils.MD5) + var err error + if len(etag) != utils.MD5.Width { + _, etag, err = stream.CacheFullInTempFileAndHash(s, utils.MD5) + if err != nil { + return nil, err + } } - etag := hex.EncodeToString(h.Sum(nil)) // get upToken res, err := d.proved("/7n/getUpToken", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ @@ -309,7 +300,7 @@ func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreame key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: &driver.SimpleReaderWithSize{ - Reader: tempFile, + Reader: s, Size: s.GetSize(), }, UpdateProgress: up, diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index 736d612a96b..f8f14300571 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -269,9 +269,6 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if err != nil { return nil, err } - defer func() { - _ = file.Close() - }() // step.1 uploadPartData, err := mopan.InitUploadPartData(ctx, mopan.UpdloadFileParam{ diff --git a/drivers/netease_music/util.go b/drivers/netease_music/util.go index 2e78be14b97..217181062ca 100644 --- a/drivers/netease_music/util.go +++ b/drivers/netease_music/util.go @@ -227,7 +227,6 @@ func (d *NeteaseMusic) putSongStream(ctx context.Context, stream model.FileStrea if err != nil { return err } - defer tmp.Close() u := uploader{driver: d, file: tmp} diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 554349679d0..e256b7ae262 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -220,7 +220,7 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { return err } diff --git a/drivers/onedrive_app/util.go b/drivers/onedrive_app/util.go index 1b01324e09a..5c3b6c922d8 100644 --- a/drivers/onedrive_app/util.go +++ b/drivers/onedrive_app/util.go @@ -170,7 +170,7 @@ func (d *OnedriveAPP) upBig(ctx context.Context, dstDir model.Obj, stream model. if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(byteData))) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { return err } diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 61396aa4ece..f88f085cd25 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -7,13 +7,6 @@ import ( "crypto/sha1" "encoding/hex" "fmt" - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - jsoniter "github.com/json-iterator/go" - "github.com/pkg/errors" "io" "net/http" "path/filepath" @@ -24,7 +17,14 @@ import ( "time" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/go-resty/resty/v2" + jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" ) var AndroidAlgorithms = []string{ @@ -516,7 +516,7 @@ func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSi continue } - b := driver.NewLimitedUploadStream(ctx, bytes.NewBuffer(buf)) + b := driver.NewLimitedUploadStream(ctx, bytes.NewReader(buf)) if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, OssOption(params)...); err == nil { break } diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 0f8884fac53..7f497494502 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -3,9 +3,8 @@ package quark import ( "bytes" "context" - "crypto/md5" - "crypto/sha1" "encoding/hex" + "hash" "io" "net/http" "time" @@ -14,6 +13,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + streamPkg "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" @@ -136,33 +136,33 @@ func (d *QuarkOrUC) Remove(ctx context.Context, obj model.Obj) error { } func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - defer func() { - _ = tempFile.Close() - }() - m := md5.New() - _, err = utils.CopyWithBuffer(m, tempFile) - if err != nil { - return err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err - } - md5Str := hex.EncodeToString(m.Sum(nil)) - s := sha1.New() - _, err = utils.CopyWithBuffer(s, tempFile) - if err != nil { - return err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err + md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) + var ( + md5 hash.Hash + sha1 hash.Hash + ) + writers := []io.Writer{} + if len(md5Str) != utils.MD5.Width { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if len(sha1Str) != utils.SHA1.Width { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + if len(writers) > 0 { + _, err := streamPkg.CacheFullInTempFileAndWriter(stream, io.MultiWriter(writers...)) + if err != nil { + return err + } + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } } - sha1Str := hex.EncodeToString(s.Sum(nil)) // pre pre, err := d.upPre(stream, dstDir.GetID()) if err != nil { @@ -178,27 +178,28 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File return nil } // part up - partSize := pre.Metadata.PartSize - var part []byte - md5s := make([]string, 0) - defaultBytes := make([]byte, partSize) total := stream.GetSize() left := total + partSize := int64(pre.Metadata.PartSize) + part := make([]byte, partSize) + count := int(total / partSize) + if total%partSize > 0 { + count++ + } + md5s := make([]string, 0, count) partNumber := 1 for left > 0 { if utils.IsCanceled(ctx) { return ctx.Err() } - if left > int64(partSize) { - part = defaultBytes - } else { - part = make([]byte, left) + if left < partSize { + part = part[:left] } - _, err := io.ReadFull(tempFile, part) + n, err := io.ReadFull(stream, part) if err != nil { return err } - left -= int64(len(part)) + left -= int64(n) log.Debugf("left: %d", left) reader := driver.NewLimitedUploadStream(ctx, bytes.NewReader(part)) m, err := d.upPart(ctx, pre, stream.GetMimetype(), partNumber, reader) diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 7f41d003838..51396ee8038 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -12,6 +12,7 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" @@ -333,22 +334,17 @@ func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error { } func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { - hi := file.GetHash() - gcid := hi.GetHash(hash_extend.GCID) + gcid := file.GetHash().GetHash(hash_extend.GCID) + var err error if len(gcid) < hash_extend.GCID.Width { - tFile, err := file.CacheFullInTempFile() - if err != nil { - return err - } - - gcid, err = utils.HashFile(hash_extend.GCID, tFile, file.GetSize()) + _, gcid, err = stream.CacheFullInTempFileAndHash(file, hash_extend.GCID, file.GetSize()) if err != nil { return err } } var resp UploadTaskResponse - _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "kind": FILE, diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go index 7ce71f7d265..0b38d07714f 100644 --- a/drivers/thunder_browser/driver.go +++ b/drivers/thunder_browser/driver.go @@ -4,10 +4,15 @@ import ( "context" "errors" "fmt" + "io" + "net/http" + "strings" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + streamPkg "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" @@ -15,9 +20,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" - "io" - "net/http" - "strings" ) type ThunderBrowser struct { @@ -456,15 +458,10 @@ func (xc *XunLeiBrowserCommon) Remove(ctx context.Context, obj model.Obj) error } func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - hi := stream.GetHash() - gcid := hi.GetHash(hash_extend.GCID) + gcid := stream.GetHash().GetHash(hash_extend.GCID) + var err error if len(gcid) < hash_extend.GCID.Width { - tFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - - gcid, err = utils.HashFile(hash_extend.GCID, tFile, stream.GetSize()) + _, gcid, err = streamPkg.CacheFullInTempFileAndHash(stream, hash_extend.GCID, stream.GetSize()) if err != nil { return err } @@ -481,7 +478,7 @@ func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream } var resp UploadTaskResponse - _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&js) }, &resp) diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go index 2194bdc6e9c..6ee8901a4fc 100644 --- a/drivers/thunderx/driver.go +++ b/drivers/thunderx/driver.go @@ -3,11 +3,15 @@ package thunderx import ( "context" "fmt" + "net/http" + "strings" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" @@ -15,8 +19,6 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" - "net/http" - "strings" ) type ThunderX struct { @@ -364,22 +366,17 @@ func (xc *XunLeiXCommon) Remove(ctx context.Context, obj model.Obj) error { } func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { - hi := file.GetHash() - gcid := hi.GetHash(hash_extend.GCID) + gcid := file.GetHash().GetHash(hash_extend.GCID) + var err error if len(gcid) < hash_extend.GCID.Width { - tFile, err := file.CacheFullInTempFile() - if err != nil { - return err - } - - gcid, err = utils.HashFile(hash_extend.GCID, tFile, file.GetSize()) + _, gcid, err = stream.CacheFullInTempFileAndHash(file, hash_extend.GCID, file.GetSize()) if err != nil { return err } } var resp UploadTaskResponse - _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "kind": FILE, diff --git a/internal/archive/archives/utils.go b/internal/archive/archives/utils.go index fdae10091f6..2f499a10feb 100644 --- a/internal/archive/archives/utils.go +++ b/internal/archive/archives/utils.go @@ -10,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/mholt/archives" ) @@ -73,7 +74,7 @@ func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgres return err } defer f.Close() - _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ + _, err = utils.CopyWithBuffer(f, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ Reader: rc, Size: stat.Size(), diff --git a/internal/archive/iso9660/utils.go b/internal/archive/iso9660/utils.go index 12de8e6ea28..0e4cfb1caf3 100644 --- a/internal/archive/iso9660/utils.go +++ b/internal/archive/iso9660/utils.go @@ -1,14 +1,15 @@ package iso9660 import ( + "os" + stdpath "path" + "strings" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/kdomanski/iso9660" - "io" - "os" - stdpath "path" - "strings" ) func getImage(ss *stream.SeekableStream) (*iso9660.Image, error) { @@ -66,7 +67,7 @@ func decompress(f *iso9660.File, path string, up model.UpdateProgress) error { return err } defer file.Close() - _, err = io.Copy(file, &stream.ReaderUpdatingProgress{ + _, err = utils.CopyWithBuffer(file, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ Reader: f.Reader(), Size: f.Size(), diff --git a/internal/fs/archive.go b/internal/fs/archive.go index b056decf9a2..dbae9b338de 100644 --- a/internal/fs/archive.go +++ b/internal/fs/archive.go @@ -90,9 +90,11 @@ func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadT t.SetTotalBytes(total) t.status = "getting src object" for _, s := range ss { - _, err = s.CacheFullInTempFileAndUpdateProgress(func(p float64) { - t.SetProgress((float64(cur) + float64(s.GetSize())*p/100.0) / float64(total)) - }) + if s.GetFile() == nil { + _, err = stream.CacheFullInTempFileAndUpdateProgress(s, func(p float64) { + t.SetProgress((float64(cur) + float64(s.GetSize())*p/100.0) / float64(total)) + }) + } cur += s.GetSize() if err != nil { return nil, err diff --git a/internal/model/obj.go b/internal/model/obj.go index 552b1241e6e..f0fce7a133a 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -2,6 +2,7 @@ package model import ( "io" + "os" "sort" "strings" "time" @@ -48,7 +49,8 @@ type FileStreamer interface { RangeRead(http_range.Range) (io.Reader, error) //for a non-seekable Stream, if Read is called, this function won't work CacheFullInTempFile() (File, error) - CacheFullInTempFileAndUpdateProgress(up UpdateProgress) (File, error) + SetTmpFile(r *os.File) + GetFile() File } type UpdateProgress func(percentage float64) diff --git a/internal/net/request.go b/internal/net/request.go index d4f9321c585..a1ff6d20cf9 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -248,8 +248,9 @@ func (d *downloader) sendChunkTask(newConcurrency bool) error { size: finalSize, id: d.nextChunk, buf: buf, + + newConcurrency: newConcurrency, } - ch.newConcurrency = newConcurrency d.pos += finalSize d.nextChunk++ d.chunkChannel <- ch diff --git a/internal/stream/stream.go b/internal/stream/stream.go index f6b045a0238..64160915792 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -94,27 +94,17 @@ func (f *FileStream) CacheFullInTempFile() (model.File, error) { f.Add(tmpF) f.tmpFile = tmpF f.Reader = tmpF - return f.tmpFile, nil + return tmpF, nil } -func (f *FileStream) CacheFullInTempFileAndUpdateProgress(up model.UpdateProgress) (model.File, error) { +func (f *FileStream) GetFile() model.File { if f.tmpFile != nil { - return f.tmpFile, nil + return f.tmpFile } if file, ok := f.Reader.(model.File); ok { - return file, nil - } - tmpF, err := utils.CreateTempFile(&ReaderUpdatingProgress{ - Reader: f, - UpdateProgress: up, - }, f.GetSize()) - if err != nil { - return nil, err + return file } - f.Add(tmpF) - f.tmpFile = tmpF - f.Reader = tmpF - return f.tmpFile, nil + return nil } const InMemoryBufMaxSize = 10 // Megabytes @@ -127,31 +117,36 @@ func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { // 参考 internal/net/request.go httpRange.Length = f.GetSize() - httpRange.Start } - if f.peekBuff != nil && httpRange.Start < int64(f.peekBuff.Len()) && httpRange.Start+httpRange.Length-1 < int64(f.peekBuff.Len()) { + size := httpRange.Start + httpRange.Length + if f.peekBuff != nil && size <= int64(f.peekBuff.Len()) { return io.NewSectionReader(f.peekBuff, httpRange.Start, httpRange.Length), nil } - if f.tmpFile == nil { - if httpRange.Start == 0 && httpRange.Length <= InMemoryBufMaxSizeBytes && f.peekBuff == nil { - bufSize := utils.Min(httpRange.Length, f.GetSize()) - newBuf := bytes.NewBuffer(make([]byte, 0, bufSize)) - n, err := utils.CopyWithBufferN(newBuf, f.Reader, bufSize) + var cache io.ReaderAt = f.GetFile() + if cache == nil { + if size <= InMemoryBufMaxSizeBytes { + bufSize := min(size, f.GetSize()) + // 使用bytes.Buffer作为io.CopyBuffer的写入对象，CopyBuffer会调用Buffer.ReadFrom + // 即使被写入的数据量与Buffer.Cap一致，Buffer也会扩大 + buf := make([]byte, bufSize) + n, err := io.ReadFull(f.Reader, buf) if err != nil { return nil, err } - if n != bufSize { + if n != int(bufSize) { return nil, fmt.Errorf("stream RangeRead did not get all data in peek, expect =%d ,actual =%d", bufSize, n) } - f.peekBuff = bytes.NewReader(newBuf.Bytes()) + f.peekBuff = bytes.NewReader(buf) f.Reader = io.MultiReader(f.peekBuff, f.Reader) - return io.NewSectionReader(f.peekBuff, httpRange.Start, httpRange.Length), nil + cache = f.peekBuff } else { - _, err := f.CacheFullInTempFile() + var err error + cache, err = f.CacheFullInTempFile() if err != nil { return nil, err } } } - return io.NewSectionReader(f.tmpFile, httpRange.Start, httpRange.Length), nil + return io.NewSectionReader(cache, httpRange.Start, httpRange.Length), nil } var _ model.FileStreamer = (*SeekableStream)(nil) @@ -176,13 +171,13 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) if len(fs.Mimetype) == 0 { fs.Mimetype = utils.GetMimeType(fs.Obj.GetName()) } - ss := SeekableStream{FileStream: fs, Link: link} + ss := &SeekableStream{FileStream: fs, Link: link} if ss.Reader != nil { result, ok := ss.Reader.(model.File) if ok { ss.mFile = result ss.Closers.Add(result) - return &ss, nil + return ss, nil } } if ss.Link != nil { @@ -198,7 +193,7 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) ss.mFile = mFile ss.Reader = mFile ss.Closers.Add(mFile) - return &ss, nil + return ss, nil } if ss.Link.RangeReadCloser != nil { ss.rangeReadCloser = &RateLimitRangeReadCloser{ @@ -206,7 +201,7 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) Limiter: ServerDownloadLimit, } ss.Add(ss.rangeReadCloser) - return &ss, nil + return ss, nil } if len(ss.Link.URL) > 0 { rrc, err := GetRangeReadCloserFromLink(ss.GetSize(), link) @@ -219,10 +214,12 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) } ss.rangeReadCloser = rrc ss.Add(rrc) - return &ss, nil + return ss, nil } } - + if fs.Reader != nil { + return ss, nil + } return nil, fmt.Errorf("illegal seekableStream") } @@ -248,7 +245,7 @@ func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, erro } return rc, nil } - return nil, fmt.Errorf("can't find mFile or rangeReadCloser") + return ss.FileStream.RangeRead(httpRange) } //func (f *FileStream) GetReader() io.Reader { @@ -278,7 +275,7 @@ func (ss *SeekableStream) CacheFullInTempFile() (model.File, error) { if ss.tmpFile != nil { return ss.tmpFile, nil } - if _, ok := ss.mFile.(*os.File); ok { + if ss.mFile != nil { return ss.mFile, nil } tmpF, err := utils.CreateTempFile(ss, ss.GetSize()) @@ -288,27 +285,17 @@ func (ss *SeekableStream) CacheFullInTempFile() (model.File, error) { ss.Add(tmpF) ss.tmpFile = tmpF ss.Reader = tmpF - return ss.tmpFile, nil + return tmpF, nil } -func (ss *SeekableStream) CacheFullInTempFileAndUpdateProgress(up model.UpdateProgress) (model.File, error) { +func (ss *SeekableStream) GetFile() model.File { if ss.tmpFile != nil { - return ss.tmpFile, nil - } - if _, ok := ss.mFile.(*os.File); ok { - return ss.mFile, nil + return ss.tmpFile } - tmpF, err := utils.CreateTempFile(&ReaderUpdatingProgress{ - Reader: ss, - UpdateProgress: up, - }, ss.GetSize()) - if err != nil { - return nil, err + if ss.mFile != nil { + return ss.mFile } - ss.Add(tmpF) - ss.tmpFile = tmpF - ss.Reader = tmpF - return ss.tmpFile, nil + return nil } func (f *FileStream) SetTmpFile(r *os.File) { diff --git a/internal/stream/util.go b/internal/stream/util.go index 01019482e15..5b935a9043e 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -2,6 +2,7 @@ package stream import ( "context" + "encoding/hex" "fmt" "io" "net/http" @@ -96,3 +97,45 @@ func (r *ReaderWithCtx) Close() error { } return nil } + +func CacheFullInTempFileAndUpdateProgress(stream model.FileStreamer, up model.UpdateProgress) (model.File, error) { + if cache := stream.GetFile(); cache != nil { + up(100) + return cache, nil + } + tmpF, err := utils.CreateTempFile(&ReaderUpdatingProgress{ + Reader: stream, + UpdateProgress: up, + }, stream.GetSize()) + if err == nil { + stream.SetTmpFile(tmpF) + } + return tmpF, err +} + +func CacheFullInTempFileAndWriter(stream model.FileStreamer, w io.Writer) (model.File, error) { + if cache := stream.GetFile(); cache != nil { + _, err := cache.Seek(0, io.SeekStart) + if err == nil { + _, err = utils.CopyWithBuffer(w, cache) + if err == nil { + _, err = cache.Seek(0, io.SeekStart) + } + } + return cache, err + } + tmpF, err := utils.CreateTempFile(io.TeeReader(stream, w), stream.GetSize()) + if err == nil { + stream.SetTmpFile(tmpF) + } + return tmpF, err +} + +func CacheFullInTempFileAndHash(stream model.FileStreamer, hashType *utils.HashType, params ...any) (model.File, string, error) { + h := hashType.NewFunc(params...) + tmpF, err := CacheFullInTempFileAndWriter(stream, h) + if err != nil { + return nil, "", err + } + return tmpF, hex.EncodeToString(h.Sum(nil)), err +} diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 15a6328b60b..41344fb8d56 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -1,8 +1,6 @@ package handles import ( - "github.com/alist-org/alist/v3/internal/task" - "github.com/alist-org/alist/v3/pkg/utils" "io" "net/url" stdpath "path" @@ -12,6 +10,8 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" ) @@ -44,7 +44,7 @@ func FsStream(c *gin.Context) { } if !overwrite { if res, _ := fs.Get(c, path, &fs.GetArgs{NoLog: true}); res != nil { - _, _ = io.Copy(io.Discard, c.Request.Body) + _, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body) common.ErrorStrResp(c, "file exists", 403) return } @@ -66,6 +66,10 @@ func FsStream(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + mimetype := c.GetHeader("Content-Type") + if len(mimetype) == 0 { + mimetype = utils.GetMimeType(name) + } s := &stream.FileStream{ Obj: &model.Object{ Name: name, @@ -74,7 +78,7 @@ func FsStream(c *gin.Context) { HashInfo: utils.NewHashInfoByMap(h), }, Reader: c.Request.Body, - Mimetype: c.GetHeader("Content-Type"), + Mimetype: mimetype, WebPutAsTask: asTask, } var t task.TaskExtensionInfo @@ -89,6 +93,9 @@ func FsStream(c *gin.Context) { return } if t == nil { + if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 { + _, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body) + } common.SuccessResp(c) return } @@ -114,7 +121,7 @@ func FsForm(c *gin.Context) { } if !overwrite { if res, _ := fs.Get(c, path, &fs.GetArgs{NoLog: true}); res != nil { - _, _ = io.Copy(io.Discard, c.Request.Body) + _, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body) common.ErrorStrResp(c, "file exists", 403) return } @@ -150,6 +157,10 @@ func FsForm(c *gin.Context) { if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { h[utils.SHA256] = sha256 } + mimetype := file.Header.Get("Content-Type") + if len(mimetype) == 0 { + mimetype = utils.GetMimeType(name) + } s := stream.FileStream{ Obj: &model.Object{ Name: name, @@ -158,7 +169,7 @@ func FsForm(c *gin.Context) { HashInfo: utils.NewHashInfoByMap(h), }, Reader: f, - Mimetype: file.Header.Get("Content-Type"), + Mimetype: mimetype, WebPutAsTask: asTask, } var t task.TaskExtensionInfo @@ -168,12 +179,7 @@ func FsForm(c *gin.Context) { }{f} t, err = fs.PutAsTask(c, dir, &s) } else { - ss, err := stream.NewSeekableStream(s, nil) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - err = fs.PutDirectly(c, dir, ss, true) + err = fs.PutDirectly(c, dir, &s, true) } if err != nil { common.ErrorResp(c, err, 500) From a4bfbf8a83f39708b1890d700608dc3b3737501a Mon Sep 17 00:00:00 2001 From: jerry <109275116+jerry-harm@users.noreply.github.com> Date: Sat, 12 Apr 2025 17:01:30 +0800 Subject: [PATCH 495/659] fix(ipfs): fix problems (#8252) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: :bug: (ipfs): fix the list error caused by not proper join path function 使用更加规范的路径拼接，修复了有中文或符号的路径无法正常访问的问题 * refactor: 命名规范 * 删除多余的条件判断 * fix: 使用withresult方法重构代码，添加get方法，提高性能 * fix: 允许get方法获取目录 去除多余的判断 * fix: 允许copy,rename,move进行覆写 * fix: 修复move方法导致的目录被删除 * refactor: 整理关于返回Path的代码 * fix: 修复由于get方法导致的ipfs路径无法访问 * fix: 修复path处理错误的get方法 修复get方法，删除意外加入的目录 * fix: fix path join use path join instead of filepath join to avoid os problem * fix: rm filepath ref --------- Co-authored-by: Andy Hsu --- drivers/ipfs_api/driver.go | 136 ++++++++++++++++++++++++------------- drivers/ipfs_api/meta.go | 4 +- 2 files changed, 91 insertions(+), 49 deletions(-) diff --git a/drivers/ipfs_api/driver.go b/drivers/ipfs_api/driver.go index e59da7ca334..264cef28c05 100644 --- a/drivers/ipfs_api/driver.go +++ b/drivers/ipfs_api/driver.go @@ -4,8 +4,7 @@ import ( "context" "fmt" "net/url" - "path/filepath" - "strings" + "path" shell "github.com/ipfs/go-ipfs-api" @@ -43,78 +42,115 @@ func (d *IPFS) Drop(ctx context.Context) error { } func (d *IPFS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - switch d.Mode { - case "ipfs": - path, _ = url.JoinPath("/ipfs", path) - case "ipns": - path, _ = url.JoinPath("/ipns", path) - case "mfs": - fileStat, err := d.sh.FilesStat(ctx, path) - if err != nil { - return nil, err + var ipfsPath string + cid := dir.GetID() + if cid != "" { + ipfsPath = path.Join("/ipfs", cid) + } else { + // 可能出现ipns dns解析失败的情况，需要重复获取cid，其他情况应该不会出错 + ipfsPath = dir.GetPath() + switch d.Mode { + case "ipfs": + ipfsPath = path.Join("/ipfs", ipfsPath) + case "ipns": + ipfsPath = path.Join("/ipns", ipfsPath) + case "mfs": + fileStat, err := d.sh.FilesStat(ctx, ipfsPath) + if err != nil { + return nil, err + } + ipfsPath = path.Join("/ipfs", fileStat.Hash) + default: + return nil, fmt.Errorf("mode error") } - path, _ = url.JoinPath("/ipfs", fileStat.Hash) - default: - return nil, fmt.Errorf("mode error") } - - dirs, err := d.sh.List(path) + dirs, err := d.sh.List(ipfsPath) if err != nil { return nil, err } objlist := []model.Obj{} for _, file := range dirs { - gateurl := *d.gateURL.JoinPath("/ipfs/" + file.Hash) - gateurl.RawQuery = "filename=" + url.PathEscape(file.Name) - objlist = append(objlist, &model.ObjectURL{ - Object: model.Object{ID: "/ipfs/" + file.Hash, Name: file.Name, Size: int64(file.Size), IsFolder: file.Type == 1}, - Url: model.Url{Url: gateurl.String()}, - }) + objlist = append(objlist, &model.Object{ID: file.Hash, Name: file.Name, Size: int64(file.Size), IsFolder: file.Type == 1}) } return objlist, nil } func (d *IPFS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - gateurl := d.gateURL.JoinPath(file.GetID()) - gateurl.RawQuery = "filename=" + url.PathEscape(file.GetName()) + gateurl := d.gateURL.JoinPath("/ipfs/", file.GetID()) + gateurl.RawQuery = "filename=" + url.QueryEscape(file.GetName()) return &model.Link{URL: gateurl.String()}, nil } -func (d *IPFS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { +func (d *IPFS) Get(ctx context.Context, rawPath string) (model.Obj, error) { + rawPath = path.Join(d.GetRootPath(), rawPath) + var ipfsPath string + switch d.Mode { + case "ipfs": + ipfsPath = path.Join("/ipfs", rawPath) + case "ipns": + ipfsPath = path.Join("/ipns", rawPath) + case "mfs": + fileStat, err := d.sh.FilesStat(ctx, rawPath) + if err != nil { + return nil, err + } + ipfsPath = path.Join("/ipfs", fileStat.Hash) + default: + return nil, fmt.Errorf("mode error") + } + file, err := d.sh.FilesStat(ctx, ipfsPath) + if err != nil { + return nil, err + } + return &model.Object{ID: file.Hash, Name: path.Base(rawPath), Path: rawPath, Size: int64(file.Size), IsFolder: file.Type == "directory"}, nil +} + +func (d *IPFS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if d.Mode != "mfs" { - return fmt.Errorf("only write in mfs mode") + return nil, fmt.Errorf("only write in mfs mode") + } + dirPath := parentDir.GetPath() + err := d.sh.FilesMkdir(ctx, path.Join(dirPath, dirName), shell.FilesMkdir.Parents(true)) + if err != nil { + return nil, err } - path := parentDir.GetPath() - if path[len(path):] != "/" { - path += "/" + file, err := d.sh.FilesStat(ctx, path.Join(dirPath, dirName)) + if err != nil { + return nil, err } - return d.sh.FilesMkdir(ctx, path+dirName) + return &model.Object{ID: file.Hash, Name: dirName, Path: path.Join(dirPath, dirName), Size: int64(file.Size), IsFolder: true}, nil } -func (d *IPFS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *IPFS) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if d.Mode != "mfs" { - return fmt.Errorf("only write in mfs mode") + return nil, fmt.Errorf("only write in mfs mode") } - return d.sh.FilesMv(ctx, srcObj.GetPath(), dstDir.GetPath()) + dstPath := path.Join(dstDir.GetPath(), path.Base(srcObj.GetPath())) + d.sh.FilesRm(ctx, dstPath, true) + return &model.Object{ID: srcObj.GetID(), Name: srcObj.GetName(), Path: dstPath, Size: int64(srcObj.GetSize()), IsFolder: srcObj.IsDir()}, + d.sh.FilesMv(ctx, srcObj.GetPath(), dstDir.GetPath()) } -func (d *IPFS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { +func (d *IPFS) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if d.Mode != "mfs" { - return fmt.Errorf("only write in mfs mode") + return nil, fmt.Errorf("only write in mfs mode") } - newFileName := filepath.Dir(srcObj.GetPath()) + "/" + newName - return d.sh.FilesMv(ctx, srcObj.GetPath(), strings.ReplaceAll(newFileName, "\\", "/")) + dstPath := path.Join(path.Dir(srcObj.GetPath()), newName) + d.sh.FilesRm(ctx, dstPath, true) + return &model.Object{ID: srcObj.GetID(), Name: newName, Path: dstPath, Size: int64(srcObj.GetSize()), + IsFolder: srcObj.IsDir()}, d.sh.FilesMv(ctx, srcObj.GetPath(), dstPath) } -func (d *IPFS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *IPFS) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if d.Mode != "mfs" { - return fmt.Errorf("only write in mfs mode") + return nil, fmt.Errorf("only write in mfs mode") } - newFileName := dstDir.GetPath() + "/" + filepath.Base(srcObj.GetPath()) - return d.sh.FilesCp(ctx, srcObj.GetPath(), strings.ReplaceAll(newFileName, "\\", "/")) + dstPath := path.Join(dstDir.GetPath(), path.Base(srcObj.GetPath())) + d.sh.FilesRm(ctx, dstPath, true) + return &model.Object{ID: srcObj.GetID(), Name: srcObj.GetName(), Path: dstPath, Size: int64(srcObj.GetSize()), IsFolder: srcObj.IsDir()}, + d.sh.FilesCp(ctx, path.Join("/ipfs/", srcObj.GetID()), dstPath, shell.FilesCp.Parents(true)) } func (d *IPFS) Remove(ctx context.Context, obj model.Obj) error { @@ -124,19 +160,25 @@ func (d *IPFS) Remove(ctx context.Context, obj model.Obj) error { return d.sh.FilesRm(ctx, obj.GetPath(), true) } -func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { +func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if d.Mode != "mfs" { - return fmt.Errorf("only write in mfs mode") + return nil, fmt.Errorf("only write in mfs mode") } outHash, err := d.sh.Add(driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ Reader: s, UpdateProgress: up, })) if err != nil { - return err + return nil, err + } + dstPath := path.Join(dstDir.GetPath(), s.GetName()) + if s.GetExist() != nil { + d.sh.FilesRm(ctx, dstPath, true) } - err = d.sh.FilesCp(ctx, "/ipfs/"+outHash, dstDir.GetPath()+"/"+strings.ReplaceAll(s.GetName(), "\\", "/")) - return err + err = d.sh.FilesCp(ctx, path.Join("/ipfs/", outHash), dstPath, shell.FilesCp.Parents(true)) + gateurl := d.gateURL.JoinPath("/ipfs/", outHash) + gateurl.RawQuery = "filename=" + url.QueryEscape(s.GetName()) + return &model.Object{ID: outHash, Name: s.GetName(), Path: dstPath, Size: int64(s.GetSize()), IsFolder: s.IsDir()}, err } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/ipfs_api/meta.go b/drivers/ipfs_api/meta.go index c145644c580..3837bec2cbf 100644 --- a/drivers/ipfs_api/meta.go +++ b/drivers/ipfs_api/meta.go @@ -9,8 +9,8 @@ type Addition struct { // Usually one of two driver.RootPath Mode string `json:"mode" options:"ipfs,ipns,mfs" type:"select" required:"true"` - Endpoint string `json:"endpoint" default:"http://127.0.0.1:5001"` - Gateway string `json:"gateway" default:"http://127.0.0.1:8080"` + Endpoint string `json:"endpoint" default:"http://127.0.0.1:5001" required:"true"` + Gateway string `json:"gateway" default:"http://127.0.0.1:8080" required:"true"` } var config = driver.Config{ From a2f266277c44326a073999699b9b3e6015a9659d Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sat, 12 Apr 2025 17:01:52 +0800 Subject: [PATCH 496/659] fix(net): unexpected write (#8291 close #8281) --- internal/net/serve.go | 16 +++++++++------- server/common/proxy.go | 37 +++++++++++++++++++++++++++++-------- server/handles/down.go | 35 +++++++++++++++-------------------- server/webdav/webdav.go | 10 ++++++---- 4 files changed, 59 insertions(+), 39 deletions(-) diff --git a/internal/net/serve.go b/internal/net/serve.go index 63e1cb45b87..bdeac0ac5f8 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -52,19 +52,19 @@ import ( // // If the caller has set w's ETag header formatted per RFC 7232, section 2.3, // ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range. -func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReadCloser model.RangeReadCloserIF) { +func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReadCloser model.RangeReadCloserIF) error { defer RangeReadCloser.Close() setLastModified(w, modTime) done, rangeReq := checkPreconditions(w, r, modTime) if done { - return + return nil } if size < 0 { // since too many functions need file size to work, // will not implement the support of unknown file size here http.Error(w, "negative content size not supported", http.StatusInternalServerError) - return + return nil } code := http.StatusOK @@ -103,7 +103,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time fallthrough default: http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) - return + return nil } if sumRangesSize(ranges) > size { @@ -124,7 +124,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time code = http.StatusTooManyRequests } http.Error(w, err.Error(), code) - return + return nil } sendContent = reader case len(ranges) == 1: @@ -147,7 +147,7 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time code = http.StatusTooManyRequests } http.Error(w, err.Error(), code) - return + return nil } sendSize = ra.Length code = http.StatusPartialContent @@ -205,9 +205,11 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time if err == ErrExceedMaxConcurrency { code = http.StatusTooManyRequests } - http.Error(w, err.Error(), code) + w.WriteHeader(code) + return err } } + return nil } func ProcessHeader(origin, override http.Header) http.Header { result := http.Header{} diff --git a/server/common/proxy.go b/server/common/proxy.go index f9e1e4bb0e8..ca7f6325d7d 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -39,11 +39,10 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. return nil } else if link.RangeReadCloser != nil { attachHeader(w, file) - net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ + return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ RangeReadCloserIF: link.RangeReadCloser, Limiter: stream.ServerDownloadLimit, }) - return nil } else if link.Concurrency != 0 || link.PartSize != 0 { attachHeader(w, file) size := file.GetSize() @@ -66,11 +65,10 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. rc, err := down.Download(ctx, req) return rc, err } - net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ + return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ RangeReadCloserIF: &model.RangeReadCloser{RangeReader: rangeReader}, Limiter: stream.ServerDownloadLimit, }) - return nil } else { //transparent proxy header := net.ProcessHeader(r.Header, link.Header) @@ -90,10 +88,7 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. Limiter: stream.ServerDownloadLimit, Ctx: r.Context(), }) - if err != nil { - return err - } - return nil + return err } } func attachHeader(w http.ResponseWriter, file model.Obj) { @@ -133,3 +128,29 @@ func ProxyRange(link *model.Link, size int64) { link.RangeReadCloser = nil } } + +type InterceptResponseWriter struct { + http.ResponseWriter + io.Writer +} + +func (iw *InterceptResponseWriter) Write(p []byte) (int, error) { + return iw.Writer.Write(p) +} + +type WrittenResponseWriter struct { + http.ResponseWriter + written bool +} + +func (ww *WrittenResponseWriter) Write(p []byte) (int, error) { + n, err := ww.ResponseWriter.Write(p) + if !ww.written && n > 0 { + ww.written = true + } + return n, err +} + +func (ww *WrittenResponseWriter) IsWritten() bool { + return ww.written +} diff --git a/server/handles/down.go b/server/handles/down.go index 1153881fdda..2c5c2fafc51 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - "net/http" stdpath "path" "strconv" "strings" @@ -129,15 +128,16 @@ func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange boo if proxyRange { common.ProxyRange(link, file.GetSize()) } + Writer := &common.WrittenResponseWriter{ResponseWriter: c.Writer} //优先处理md文件 if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) { - w := c.Writer buf := bytes.NewBuffer(make([]byte, 0, file.GetSize())) - err = common.Proxy(&proxyResponseWriter{ResponseWriter: w, Writer: buf}, c.Request, link, file) + w := &common.InterceptResponseWriter{ResponseWriter: Writer, Writer: buf} + err = common.Proxy(w, c.Request, link, file) if err == nil && buf.Len() > 0 { - if w.Status() < 200 || w.Status() > 300 { - w.Write(buf.Bytes()) + if c.Writer.Status() < 200 || c.Writer.Status() > 300 { + c.Writer.Write(buf.Bytes()) return } @@ -148,19 +148,23 @@ func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange boo buf.Reset() err = bluemonday.UGCPolicy().SanitizeReaderToWriter(&html, buf) if err == nil { - w.Header().Set("Content-Length", strconv.FormatInt(int64(buf.Len()), 10)) - w.Header().Set("Content-Type", "text/html; charset=utf-8") - _, err = utils.CopyWithBuffer(c.Writer, buf) + Writer.Header().Set("Content-Length", strconv.FormatInt(int64(buf.Len()), 10)) + Writer.Header().Set("Content-Type", "text/html; charset=utf-8") + _, err = utils.CopyWithBuffer(Writer, buf) } } } } else { - err = common.Proxy(c.Writer, c.Request, link, file) + err = common.Proxy(Writer, c.Request, link, file) } - if err != nil { - common.ErrorResp(c, err, 500, true) + if err == nil { return } + if Writer.IsWritten() { + log.Errorf("%s %s local proxy error: %+v", c.Request.Method, c.Request.URL.Path, err) + } else { + common.ErrorResp(c, err, 500, true) + } } // TODO need optimize @@ -182,12 +186,3 @@ func canProxy(storage driver.Driver, filename string) bool { } return false } - -type proxyResponseWriter struct { - http.ResponseWriter - io.Writer -} - -func (pw *proxyResponseWriter) Write(p []byte) (int, error) { - return pw.Writer.Write(p) -} diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 1b7ec6ff72b..f22e15aadb9 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -24,7 +24,6 @@ import ( "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" - log "github.com/sirupsen/logrus" ) type Handler struct { @@ -59,7 +58,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { status, err = h.handleOptions(brw, r) case "GET", "HEAD", "POST": useBufferedWriter = false - status, err = h.handleGetHeadPost(w, r) + Writer := &common.WrittenResponseWriter{ResponseWriter: w} + status, err = h.handleGetHeadPost(Writer, r) + if status != 0 && Writer.IsWritten() { + status = 0 + } case "DELETE": status, err = h.handleDelete(brw, r) case "PUT": @@ -247,8 +250,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta } err = common.Proxy(w, r, link, fi) if err != nil { - log.Errorf("webdav proxy error: %+v", err) - return http.StatusInternalServerError, err + return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) } } else if storage.GetStorage().WebdavProxy() && downProxyUrl != "" { u := fmt.Sprintf("%s%s?sign=%s", From 4f5cabc725e3db6e143aae0008753325c74ee44d Mon Sep 17 00:00:00 2001 From: j2rong4cn <36783515+j2rong4cn@users.noreply.github.com> Date: Sat, 12 Apr 2025 17:02:51 +0800 Subject: [PATCH 497/659] feat: add h2c for http server (#8294) * feat: add h2c for http server * chore(config): add EnableH2c option --- cmd/server.go | 16 +++++++++++----- go.mod | 2 +- go.sum | 2 ++ internal/conf/config.go | 1 + 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index d9206cfeb18..4263f02021d 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -4,9 +4,6 @@ import ( "context" "errors" "fmt" - ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" - "github.com/KirCute/sftpd-alist" - "github.com/alist-org/alist/v3/internal/fs" "net" "net/http" "os" @@ -16,14 +13,19 @@ import ( "syscall" "time" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/KirCute/sftpd-alist" "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" ) // ServerCmd represents the server command @@ -47,11 +49,15 @@ the address is defined in config file`, r := gin.New() r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) server.Init(r) + var httpHandler http.Handler = r + if conf.Conf.Scheme.EnableH2c { + httpHandler = h2c.NewHandler(r, &http2.Server{}) + } var httpSrv, httpsSrv, unixSrv *http.Server if conf.Conf.Scheme.HttpPort != -1 { httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort) utils.Log.Infof("start HTTP server @ %s", httpBase) - httpSrv = &http.Server{Addr: httpBase, Handler: r} + httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler} go func() { err := httpSrv.ListenAndServe() if err != nil && !errors.Is(err, http.ErrServerClosed) { @@ -72,7 +78,7 @@ the address is defined in config file`, } if conf.Conf.Scheme.UnixFile != "" { utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile) - unixSrv = &http.Server{Handler: r} + unixSrv = &http.Server{Handler: httpHandler} go func() { listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile) if err != nil { diff --git a/go.mod b/go.mod index 97a477d33b1..e8afe0e7a62 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 - golang.org/x/net v0.37.0 + golang.org/x/net v0.38.0 golang.org/x/oauth2 v0.22.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 diff --git a/go.sum b/go.sum index 86fb779e5d8..6fbaeb2b3ef 100644 --- a/go.sum +++ b/go.sum @@ -741,6 +741,8 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/internal/conf/config.go b/internal/conf/config.go index 1766ae8406f..cdb86fee3ad 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -35,6 +35,7 @@ type Scheme struct { KeyFile string `json:"key_file" env:"KEY_FILE"` UnixFile string `json:"unix_file" env:"UNIX_FILE"` UnixFilePerm string `json:"unix_file_perm" env:"UNIX_FILE_PERM"` + EnableH2c bool `json:"enable_h2c" env:"ENABLE_H2C"` } type LogConfig struct { From 544a7ea022ad79769d6988141304f47183c69d5e Mon Sep 17 00:00:00 2001 From: Dgs <47767754+dgscyg@users.noreply.github.com> Date: Sat, 12 Apr 2025 17:03:58 +0800 Subject: [PATCH 498/659] fix(pikpak&pikpak_share): fix WebPackageName (#8305) --- drivers/pikpak/util.go | 2 +- drivers/pikpak_share/util.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index f88f085cd25..4cb3fbc3ec8 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -84,7 +84,7 @@ const ( WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" - WebPackageName = "drive.mypikpak.com" + WebPackageName = "mypikpak.com" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index 4111779f4b2..2980069e5f8 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -67,7 +67,7 @@ const ( WebClientID = "YUMx5nI8ZU8Ap8pm" WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" WebClientVersion = "2.0.0" - WebPackageName = "drive.mypikpak.com" + WebPackageName = "mypikpak.com" WebSdkVersion = "8.0.3" PCClientID = "YvtoWO6GNHiuCl7x" PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" From d0ee90cd115503151280c22b9f46e71c35df71dc Mon Sep 17 00:00:00 2001 From: Dgs <47767754+dgscyg@users.noreply.github.com> Date: Sat, 12 Apr 2025 17:05:58 +0800 Subject: [PATCH 499/659] fix(thunder): fix login issue (#8342 close #8288) --- drivers/thunder/driver.go | 132 +++++++++++++++++++++++++++++++------- drivers/thunder/meta.go | 14 ++-- drivers/thunder/types.go | 104 ++++++++++++++++++++++++++++-- drivers/thunder/util.go | 88 +++++++++++++++++++++++-- 4 files changed, 304 insertions(+), 34 deletions(-) diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 51396ee8038..1d2f2a81f65 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -45,26 +45,29 @@ func (x *Thunder) Init(ctx context.Context) (err error) { Common: &Common{ client: base.NewRestyClient(), Algorithms: []string{ - "HPxr4BVygTQVtQkIMwQH33ywbgYG5l4JoR", - "GzhNkZ8pOBsCY+7", - "v+l0ImTpG7c7/", - "e5ztohgVXNP", - "t", - "EbXUWyVVqQbQX39Mbjn2geok3/0WEkAVxeqhtx857++kjJiRheP8l77gO", - "o7dvYgbRMOpHXxCs", - "6MW8TD8DphmakaxCqVrfv7NReRRN7ck3KLnXBculD58MvxjFRqT+", - "kmo0HxCKVfmxoZswLB4bVA/dwqbVAYghSb", - "j", - "4scKJNdd7F27Hv7tbt", + "9uJNVj/wLmdwKrJaVj/omlQ", + "Oz64Lp0GigmChHMf/6TNfxx7O9PyopcczMsnf", + "Eb+L7Ce+Ej48u", + "jKY0", + "ASr0zCl6v8W4aidjPK5KHd1Lq3t+vBFf41dqv5+fnOd", + "wQlozdg6r1qxh0eRmt3QgNXOvSZO6q/GXK", + "gmirk+ciAvIgA/cxUUCema47jr/YToixTT+Q6O", + "5IiCoM9B1/788ntB", + "P07JH0h6qoM6TSUAK2aL9T5s2QBVeY9JWvalf", + "+oK0AN", }, - DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), + DeviceID: func() string { + if len(x.DeviceID) != 32 { + return utils.GetMD5EncodeStr(x.DeviceID) + } + return x.DeviceID + }(), ClientID: "Xp6vsxz_7IYVw2BB", ClientSecret: "Xp6vsy4tN9toTVdMSpomVdXpRmES", - ClientVersion: "7.51.0.8196", + ClientVersion: "8.31.0.9726", PackageName: "com.xunlei.downloadprovider", - UserAgent: "ANDROID-com.xunlei.downloadprovider/7.51.0.8196 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", + UserAgent: "ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", - refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) @@ -80,6 +83,8 @@ func (x *Thunder) Init(ctx context.Context) (err error) { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) op.MustSaveDriverStorage(x) } + // 清空 信任密钥 + x.Addition.CreditKey = "" } x.SetTokenResp(token) return err @@ -93,6 +98,17 @@ func (x *Thunder) Init(ctx context.Context) (err error) { x.SetCaptchaToken(ctoekn) } + if x.Addition.CreditKey != "" { + x.SetCreditKey(x.Addition.CreditKey) + } + + if x.Addition.DeviceID != "" { + x.Common.DeviceID = x.Addition.DeviceID + } else { + x.Addition.DeviceID = x.Common.DeviceID + op.MustSaveDriverStorage(x) + } + // 防止重复登录 identity := x.GetIdentity() if x.identity != identity || !x.IsLogin() { @@ -102,6 +118,8 @@ func (x *Thunder) Init(ctx context.Context) (err error) { if err != nil { return err } + // 清空 信任密钥 + x.Addition.CreditKey = "" x.SetTokenResp(token) } return nil @@ -161,6 +179,17 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) { x.SetCaptchaToken(x.CaptchaToken) } + if x.ExpertAddition.CreditKey != "" { + x.SetCreditKey(x.ExpertAddition.CreditKey) + } + + if x.ExpertAddition.DeviceID != "" { + x.Common.DeviceID = x.ExpertAddition.DeviceID + } else { + x.ExpertAddition.DeviceID = x.Common.DeviceID + op.MustSaveDriverStorage(x) + } + // 签名方法 if x.SignType == "captcha_sign" { x.Common.Timestamp = x.Timestamp @@ -194,6 +223,8 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) { if err != nil { return err } + // 清空 信任密钥 + x.ExpertAddition.CreditKey = "" x.SetTokenResp(token) x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiCommon.RefreshToken(x.TokenResp.RefreshToken) @@ -202,6 +233,8 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) { if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) } + // 清空 信任密钥 + x.ExpertAddition.CreditKey = "" } x.SetTokenResp(token) op.MustSaveDriverStorage(x) @@ -233,7 +266,8 @@ func (x *ThunderExpert) SetTokenResp(token *TokenResp) { type XunLeiCommon struct { *Common - *TokenResp // 登录信息 + *TokenResp // 登录信息 + *CoreLoginResp // core登录信息 refreshTokenFunc func() error } @@ -433,6 +467,10 @@ func (xc *XunLeiCommon) SetTokenResp(tr *TokenResp) { xc.TokenResp = tr } +func (xc *XunLeiCommon) SetCoreTokenResp(tr *CoreLoginResp) { + xc.CoreLoginResp = tr +} + // 携带Authorization和CaptchaToken的请求 func (xc *XunLeiCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { data, err := xc.Common.Request(url, method, func(req *resty.Request) { @@ -461,7 +499,7 @@ func (xc *XunLeiCommon) Request(url string, method string, callback base.ReqCall } return nil, err case 9: // 验证码token过期 - if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.UserID); err != nil { + if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.TokenResp.UserID); err != nil { return nil, err } default: @@ -493,20 +531,25 @@ func (xc *XunLeiCommon) RefreshToken(refreshToken string) (*TokenResp, error) { // 登录 func (xc *XunLeiCommon) Login(username, password string) (*TokenResp, error) { - url := XLUSER_API_URL + "/auth/signin" - err := xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username) + //v3 login拿到 sessionID + sessionID, err := xc.CoreLogin(username, password) if err != nil { return nil, err } + //v1 login拿到令牌 + url := XLUSER_API_URL + "/auth/signin/token" + if err = xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username); err != nil { + return nil, err + } var resp TokenResp _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetPathParam("client_id", xc.ClientID) req.SetBody(&SignInRequest{ - CaptchaToken: xc.GetCaptchaToken(), ClientID: xc.ClientID, ClientSecret: xc.ClientSecret, - Username: username, - Password: password, + Provider: SignProvider, + SigninToken: sessionID, }) }, &resp) if err != nil { @@ -582,3 +625,48 @@ func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string } return nil } + +func (xc *XunLeiCommon) CoreLogin(username string, password string) (sessionID string, err error) { + url := XLUSER_API_BASE_URL + "/xluser.core.login/v3/login" + var resp CoreLoginResp + res, err := xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", "android-ok-http-client/xl-acc-sdk/version-5.0.12.512000") + req.SetBody(&CoreLoginRequest{ + ProtocolVersion: "301", + SequenceNo: "1000012", + PlatformVersion: "10", + IsCompressed: "0", + Appid: APPID, + ClientVersion: "8.31.0.9726", + PeerID: "00000000000000000000000000000000", + AppName: "ANDROID-com.xunlei.downloadprovider", + SdkVersion: "512000", + Devicesign: generateDeviceSign(xc.DeviceID, xc.PackageName), + NetWorkType: "WIFI", + ProviderName: "NONE", + DeviceModel: "M2004J7AC", + DeviceName: "Xiaomi_M2004j7ac", + OSVersion: "12", + Creditkey: xc.GetCreditKey(), + Hl: "zh-CN", + UserName: username, + PassWord: password, + VerifyKey: "", + VerifyCode: "", + IsMd5Pwd: "0", + }) + }, nil) + if err != nil { + return "", err + } + + if err = utils.Json.Unmarshal(res, &resp); err != nil { + return "", err + } + + xc.SetCoreTokenResp(&resp) + + sessionID = resp.SessionID + + return sessionID, nil +} diff --git a/drivers/thunder/meta.go b/drivers/thunder/meta.go index 12b01cbaa16..5e6e251387e 100644 --- a/drivers/thunder/meta.go +++ b/drivers/thunder/meta.go @@ -23,23 +23,25 @@ type ExpertAddition struct { RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` // 签名方法1 - Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"HPxr4BVygTQVtQkIMwQH33ywbgYG5l4JoR,GzhNkZ8pOBsCY+7,v+l0ImTpG7c7/,e5ztohgVXNP,t,EbXUWyVVqQbQX39Mbjn2geok3/0WEkAVxeqhtx857++kjJiRheP8l77gO,o7dvYgbRMOpHXxCs,6MW8TD8DphmakaxCqVrfv7NReRRN7ck3KLnXBculD58MvxjFRqT+,kmo0HxCKVfmxoZswLB4bVA/dwqbVAYghSb,j,4scKJNdd7F27Hv7tbt"` + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"9uJNVj/wLmdwKrJaVj/omlQ,Oz64Lp0GigmChHMf/6TNfxx7O9PyopcczMsnf,Eb+L7Ce+Ej48u,jKY0,ASr0zCl6v8W4aidjPK5KHd1Lq3t+vBFf41dqv5+fnOd,wQlozdg6r1qxh0eRmt3QgNXOvSZO6q/GXK,gmirk+ciAvIgA/cxUUCema47jr/YToixTT+Q6O,5IiCoM9B1/788ntB,P07JH0h6qoM6TSUAK2aL9T5s2QBVeY9JWvalf,+oK0AN"` // 签名方法2 CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` // 验证码 CaptchaToken string `json:"captcha_token"` + // 信任密钥 + CreditKey string `json:"credit_key" help:"credit key,used for login"` // 必要且影响登录,由签名决定 - DeviceID string `json:"device_id" required:"true" default:"9aa5c268e7bcfc197a9ad88e2fb330e5"` + DeviceID string `json:"device_id" default:""` ClientID string `json:"client_id" required:"true" default:"Xp6vsxz_7IYVw2BB"` ClientSecret string `json:"client_secret" required:"true" default:"Xp6vsy4tN9toTVdMSpomVdXpRmES"` - ClientVersion string `json:"client_version" required:"true" default:"7.51.0.8196"` + ClientVersion string `json:"client_version" required:"true" default:"8.31.0.9726"` PackageName string `json:"package_name" required:"true" default:"com.xunlei.downloadprovider"` //不影响登录,影响下载速度 - UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.xunlei.downloadprovider/7.51.0.8196 netWorkType/4G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gdcf98eab238b) (JAVA 0)"` + UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)"` DownloadUserAgent string `json:"download_user_agent" required:"true" default:"Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)"` //优先使用视频链接代替下载链接 @@ -74,6 +76,10 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` CaptchaToken string `json:"captcha_token"` + // 信任密钥 + CreditKey string `json:"credit_key" help:"credit key,used for login"` + // 登录设备ID + DeviceID string `json:"device_id" default:""` } // 登录特征,用于判断是否重新登录 diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index b7355b2a6fa..1fe8432c2bb 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -18,6 +18,10 @@ type ErrResp struct { } func (e *ErrResp) IsError() bool { + if e.ErrorMsg == "success" { + return false + } + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" } @@ -61,13 +65,79 @@ func (t *TokenResp) Token() string { } type SignInRequest struct { - CaptchaToken string `json:"captcha_token"` - ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` - Username string `json:"username"` - Password string `json:"password"` + Provider string `json:"provider"` + SigninToken string `json:"signin_token"` +} + +type CoreLoginRequest struct { + ProtocolVersion string `json:"protocolVersion"` + SequenceNo string `json:"sequenceNo"` + PlatformVersion string `json:"platformVersion"` + IsCompressed string `json:"isCompressed"` + Appid string `json:"appid"` + ClientVersion string `json:"clientVersion"` + PeerID string `json:"peerID"` + AppName string `json:"appName"` + SdkVersion string `json:"sdkVersion"` + Devicesign string `json:"devicesign"` + NetWorkType string `json:"netWorkType"` + ProviderName string `json:"providerName"` + DeviceModel string `json:"deviceModel"` + DeviceName string `json:"deviceName"` + OSVersion string `json:"OSVersion"` + Creditkey string `json:"creditkey"` + Hl string `json:"hl"` + UserName string `json:"userName"` + PassWord string `json:"passWord"` + VerifyKey string `json:"verifyKey"` + VerifyCode string `json:"verifyCode"` + IsMd5Pwd string `json:"isMd5Pwd"` +} + +type CoreLoginResp struct { + Account string `json:"account"` + Creditkey string `json:"creditkey"` + /* Error string `json:"error"` + ErrorCode string `json:"errorCode"` + ErrorDescription string `json:"error_description"`*/ + ExpiresIn int `json:"expires_in"` + IsCompressed string `json:"isCompressed"` + IsSetPassWord string `json:"isSetPassWord"` + KeepAliveMinPeriod string `json:"keepAliveMinPeriod"` + KeepAlivePeriod string `json:"keepAlivePeriod"` + LoginKey string `json:"loginKey"` + NickName string `json:"nickName"` + PlatformVersion string `json:"platformVersion"` + ProtocolVersion string `json:"protocolVersion"` + SecureKey string `json:"secureKey"` + SequenceNo string `json:"sequenceNo"` + SessionID string `json:"sessionID"` + Timestamp string `json:"timestamp"` + UserID string `json:"userID"` + UserName string `json:"userName"` + UserNewNo string `json:"userNewNo"` + Version string `json:"version"` + /* VipList []struct { + ExpireDate string `json:"expireDate"` + IsAutoDeduct string `json:"isAutoDeduct"` + IsVip string `json:"isVip"` + IsYear string `json:"isYear"` + PayID string `json:"payId"` + PayName string `json:"payName"` + Register string `json:"register"` + Vasid string `json:"vasid"` + VasType string `json:"vasType"` + VipDayGrow string `json:"vipDayGrow"` + VipGrow string `json:"vipGrow"` + VipLevel string `json:"vipLevel"` + Icon struct { + General string `json:"general"` + Small string `json:"small"` + } `json:"icon"` + } `json:"vipList"`*/ } /* @@ -251,3 +321,29 @@ type Params struct { PredictSpeed string `json:"predict_speed"` PredictType string `json:"predict_type"` } + +// LoginReviewResp 登录验证响应 +type LoginReviewResp struct { + Creditkey string `json:"creditkey"` + Error string `json:"error"` + ErrorCode string `json:"errorCode"` + ErrorDesc string `json:"errorDesc"` + ErrorDescURL string `json:"errorDescUrl"` + ErrorIsRetry int `json:"errorIsRetry"` + ErrorDescription string `json:"error_description"` + IsCompressed string `json:"isCompressed"` + PlatformVersion string `json:"platformVersion"` + ProtocolVersion string `json:"protocolVersion"` + Reviewurl string `json:"reviewurl"` + SequenceNo string `json:"sequenceNo"` + UserID string `json:"userID"` + VerifyType string `json:"verifyType"` +} + +// ReviewData 验证数据 +type ReviewData struct { + Creditkey string `json:"creditkey"` + Reviewurl string `json:"reviewurl"` + Deviceid string `json:"deviceid"` + Devicesign string `json:"devicesign"` +} diff --git a/drivers/thunder/util.go b/drivers/thunder/util.go index f509e6b2fbc..b7afe56debc 100644 --- a/drivers/thunder/util.go +++ b/drivers/thunder/util.go @@ -1,8 +1,10 @@ package thunder import ( + "crypto/md5" "crypto/sha1" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" @@ -15,10 +17,11 @@ import ( ) const ( - API_URL = "https://api-pan.xunlei.com/drive/v1" - FILE_API_URL = API_URL + "/files" - TASK_API_URL = API_URL + "/tasks" - XLUSER_API_URL = "https://xluser-ssl.xunlei.com/v1" + API_URL = "https://api-pan.xunlei.com/drive/v1" + FILE_API_URL = API_URL + "/files" + TASK_API_URL = API_URL + "/tasks" + XLUSER_API_BASE_URL = "https://xluser-ssl.xunlei.com" + XLUSER_API_URL = XLUSER_API_BASE_URL + "/v1" ) const ( @@ -34,6 +37,12 @@ const ( UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" ) +const ( + SignProvider = "access_end_point_token" + APPID = "40" + APPKey = "34a062aaa22f906fca4fefe9fb3a3021" +) + func GetAction(method string, url string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] return method + ":" + urlpath @@ -44,6 +53,8 @@ type Common struct { captchaToken string + creditKey string + // 签名相关,二选一 Algorithms []string Timestamp, CaptchaSign string @@ -69,6 +80,13 @@ func (c *Common) GetCaptchaToken() string { return c.captchaToken } +func (c *Common) SetCreditKey(creditKey string) { + c.creditKey = creditKey +} +func (c *Common) GetCreditKey() string { + return c.creditKey +} + // 刷新验证码token(登录后) func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { metas := map[string]string{ @@ -170,12 +188,53 @@ func (c *Common) Request(url, method string, callback base.ReqCallback, resp int var erron ErrResp utils.Json.Unmarshal(res.Body(), &erron) if erron.IsError() { + // review_panel 表示需要短信验证码进行验证 + if erron.ErrorMsg == "review_panel" { + return nil, c.getReviewData(res) + } + return nil, &erron } return res.Body(), nil } +// 获取验证所需内容 +func (c *Common) getReviewData(res *resty.Response) error { + var reviewResp LoginReviewResp + var reviewData ReviewData + + if err := utils.Json.Unmarshal(res.Body(), &reviewResp); err != nil { + return err + } + + deviceSign := generateDeviceSign(c.DeviceID, c.PackageName) + + reviewData = ReviewData{ + Creditkey: reviewResp.Creditkey, + Reviewurl: reviewResp.Reviewurl + "&deviceid=" + deviceSign, + Deviceid: deviceSign, + Devicesign: deviceSign, + } + + // 将reviewData转为JSON字符串 + reviewDataJSON, _ := json.MarshalIndent(reviewData, "", " ") + //reviewDataJSON, _ := json.Marshal(reviewData) + + return fmt.Errorf(` +
+ 🔒 本次登录需要验证
+ This login requires verification + +

下面是验证所需要的数据，具体使用方法请参照对应的驱动文档
+ Below are the relevant verification data. For specific usage methods, please refer to the corresponding driver documentation.

+
+ 
%s
+
+
`, string(reviewDataJSON)) +} + // 计算文件Gcid func getGcid(r io.Reader, size int64) (string, error) { calcBlockSize := func(j int64) int64 { @@ -201,3 +260,24 @@ func getGcid(r io.Reader, size int64) (string, error) { } return hex.EncodeToString(hash1.Sum(nil)), nil } + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, APPID, APPKey) + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} From c8470b9a2a08276e7e8517caa70e6d10812a9978 Mon Sep 17 00:00:00 2001 From: Yifan Gao Date: Sat, 12 Apr 2025 17:09:46 +0800 Subject: [PATCH 500/659] fix(fs): remove old target object from cache before updating (#8352) --- internal/op/fs.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/op/fs.go b/internal/op/fs.go index 01727e7598c..99c2fe3411e 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -3,6 +3,7 @@ package op import ( "context" stdpath "path" + "slices" "time" "github.com/Xhofe/go-cache" @@ -25,6 +26,12 @@ func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj key := Key(storage, path) objs, ok := listCache.Get(key) if ok { + for i, obj := range objs { + if obj.GetName() == newObj.GetName() { + objs = slices.Delete(objs, i, i+1) + break + } + } for i, obj := range objs { if obj.GetName() == oldObj.GetName() { objs[i] = newObj From f0b1aeaf8d846b3aee41fed29bf03ad7afa4e72f Mon Sep 17 00:00:00 2001 From: asdfghjkl <61342682+anobodys@users.noreply.github.com> Date: Sat, 12 Apr 2025 17:12:40 +0800 Subject: [PATCH 501/659] feat(doubao): support upload (#8302 close #8335) * feat(doubao): support upload * fix(doubao): fix file list cursor * fix: handle strconv.Atoi err Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: anobodys Co-authored-by: Andy Hsu Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- drivers/doubao/driver.go | 133 ++++-- drivers/doubao/meta.go | 5 +- drivers/doubao/types.go | 353 ++++++++++++++- drivers/doubao/util.go | 950 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 1396 insertions(+), 45 deletions(-) diff --git a/drivers/doubao/driver.go b/drivers/doubao/driver.go index 04f74325df0..a066feee1a7 100644 --- a/drivers/doubao/driver.go +++ b/drivers/doubao/driver.go @@ -3,19 +3,25 @@ package doubao import ( "context" "errors" - "time" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" + "net/http" + "strconv" + "strings" + "time" ) type Doubao struct { model.Storage Addition + *UploadToken + UserId string + uploadThread int } func (d *Doubao) Config() driver.Config { @@ -29,6 +35,31 @@ func (d *Doubao) GetAddition() driver.Additional { func (d *Doubao) Init(ctx context.Context) error { // TODO login / refresh token //op.MustSaveDriverStorage(d) + uploadThread, err := strconv.Atoi(d.UploadThread) + if err != nil || uploadThread < 1 { + d.uploadThread, d.UploadThread = 3, "3" // Set default value + } else { + d.uploadThread = uploadThread + } + + if d.UserId == "" { + userInfo, err := d.getUserInfo() + if err != nil { + return err + } + + d.UserId = strconv.FormatInt(userInfo.UserID, 10) + } + + if d.UploadToken == nil { + uploadToken, err := d.initUploadToken() + if err != nil { + return err + } + + d.UploadToken = uploadToken + } + return nil } @@ -38,18 +69,12 @@ func (d *Doubao) Drop(ctx context.Context) error { func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var files []model.Obj - var r NodeInfoResp - _, err := d.request("/samantha/aispace/node_info", "POST", func(req *resty.Request) { - req.SetBody(base.Json{ - "node_id": dir.GetID(), - "need_full_path": false, - }) - }, &r) + fileList, err := d.getFiles(dir.GetID(), "") if err != nil { return nil, err } - for _, child := range r.Data.Children { + for _, child := range fileList { files = append(files, &Object{ Object: model.Object{ ID: child.ID, @@ -60,34 +85,65 @@ func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( Ctime: time.Unix(child.CreateTime, 0), IsFolder: child.NodeType == 1, }, - Key: child.Key, + Key: child.Key, + NodeType: child.NodeType, }) } + return files, nil } func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var downloadUrl string + if u, ok := file.(*Object); ok { - var r GetFileUrlResp - _, err := d.request("/alice/message/get_file_url", "POST", func(req *resty.Request) { - req.SetBody(base.Json{ - "uris": []string{u.Key}, - "type": "file", - }) - }, &r) - if err != nil { - return nil, err + switch u.NodeType { + case VideoType, AudioType: + var r GetVideoFileUrlResp + _, err := d.request("/samantha/media/get_play_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "key": u.Key, + "node_id": file.GetID(), + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.OriginalMediaInfo.MainURL + default: + var r GetFileUrlResp + _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{u.Key}, + "type": FileNodeType[u.NodeType], + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.FileUrls[0].MainURL } + + // 生成标准的Content-Disposition + contentDisposition := generateContentDisposition(u.Name) + return &model.Link{ - URL: r.Data.FileUrls[0].MainURL, + URL: downloadUrl, + Header: http.Header{ + "User-Agent": []string{UserAgent}, + "Content-Disposition": []string{contentDisposition}, + }, }, nil } + return nil, errors.New("can't convert obj to URL") } func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { var r UploadNodeResp - _, err := d.request("/samantha/aispace/upload_node", "POST", func(req *resty.Request) { + _, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "node_list": []base.Json{ { @@ -104,7 +160,7 @@ func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { var r UploadNodeResp - _, err := d.request("/samantha/aispace/move_node", "POST", func(req *resty.Request) { + _, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "node_list": []base.Json{ {"id": srcObj.GetID()}, @@ -118,7 +174,7 @@ func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error { var r BaseResp - _, err := d.request("/samantha/aispace/rename_node", "POST", func(req *resty.Request) { + _, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "node_id": srcObj.GetID(), "node_name": newName, @@ -134,15 +190,38 @@ func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error { var r BaseResp - _, err := d.request("/samantha/aispace/delete_node", "POST", func(req *resty.Request) { + _, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}}) }, &r) return err } func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - // TODO upload file, optional - return nil, errs.NotImplement + // 根据MIME类型确定数据类型 + mimetype := file.GetMimetype() + dataType := FileDataType + + switch { + case strings.HasPrefix(mimetype, "video/"): + dataType = VideoDataType + case strings.HasPrefix(mimetype, "audio/"): + dataType = VideoDataType // 音频与视频使用相同的处理方式 + case strings.HasPrefix(mimetype, "image/"): + dataType = ImgDataType + } + + // 获取上传配置 + uploadConfig := UploadConfig{} + if err := d.getUploadConfig(&uploadConfig, dataType, file); err != nil { + return nil, err + } + + // 根据文件大小选择上传方式 + if file.GetSize() <= 1*utils.MB { // 小于1MB，使用普通模式上传 + return d.Upload(&uploadConfig, dstDir, file, up, dataType) + } + // 大文件使用分片上传 + return d.UploadByMultipart(ctx, &uploadConfig, file.GetSize(), dstDir, file, up, dataType) } func (d *Doubao) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { diff --git a/drivers/doubao/meta.go b/drivers/doubao/meta.go index bb9e3f258d0..c3d8eb34624 100644 --- a/drivers/doubao/meta.go +++ b/drivers/doubao/meta.go @@ -10,7 +10,8 @@ type Addition struct { // driver.RootPath driver.RootID // define other - Cookie string `json:"cookie" type:"text"` + Cookie string `json:"cookie" type:"text"` + UploadThread string `json:"upload_thread" default:"3"` } var config = driver.Config{ @@ -19,7 +20,7 @@ var config = driver.Config{ OnlyLocal: false, OnlyProxy: false, NoCache: false, - NoUpload: true, + NoUpload: false, NeedMs: false, DefaultRoot: "0", CheckStatus: false, diff --git a/drivers/doubao/types.go b/drivers/doubao/types.go index 2dc5a61dac0..4264eb7d83b 100644 --- a/drivers/doubao/types.go +++ b/drivers/doubao/types.go @@ -1,6 +1,11 @@ package doubao -import "github.com/alist-org/alist/v3/internal/model" +import ( + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "time" +) type BaseResp struct { Code int `json:"code"` @@ -10,14 +15,14 @@ type BaseResp struct { type NodeInfoResp struct { BaseResp Data struct { - NodeInfo NodeInfo `json:"node_info"` - Children []NodeInfo `json:"children"` - NextCursor string `json:"next_cursor"` - HasMore bool `json:"has_more"` + NodeInfo File `json:"node_info"` + Children []File `json:"children"` + NextCursor string `json:"next_cursor"` + HasMore bool `json:"has_more"` } `json:"data"` } -type NodeInfo struct { +type File struct { ID string `json:"id"` Name string `json:"name"` Key string `json:"key"` @@ -44,6 +49,39 @@ type GetFileUrlResp struct { } `json:"data"` } +type GetVideoFileUrlResp struct { + BaseResp + Data struct { + MediaType string `json:"media_type"` + MediaInfo []struct { + Meta struct { + Height string `json:"height"` + Width string `json:"width"` + Format string `json:"format"` + Duration float64 `json:"duration"` + CodecType string `json:"codec_type"` + Definition string `json:"definition"` + } `json:"meta"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"media_info"` + OriginalMediaInfo struct { + Meta struct { + Height string `json:"height"` + Width string `json:"width"` + Format string `json:"format"` + Duration float64 `json:"duration"` + CodecType string `json:"codec_type"` + Definition string `json:"definition"` + } `json:"meta"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"original_media_info"` + PosterURL string `json:"poster_url"` + PlayableStatus int `json:"playable_status"` + } `json:"data"` +} + type UploadNodeResp struct { BaseResp Data struct { @@ -60,5 +98,306 @@ type UploadNodeResp struct { type Object struct { model.Object - Key string + Key string + NodeType int +} + +type UserInfoResp struct { + Data UserInfo `json:"data"` + Message string `json:"message"` +} +type AppUserInfo struct { + BuiAuditInfo string `json:"bui_audit_info"` +} +type AuditInfo struct { +} +type Details struct { +} +type BuiAuditInfo struct { + AuditInfo AuditInfo `json:"audit_info"` + IsAuditing bool `json:"is_auditing"` + AuditStatus int `json:"audit_status"` + LastUpdateTime int `json:"last_update_time"` + UnpassReason string `json:"unpass_reason"` + Details Details `json:"details"` +} +type Connects struct { + Platform string `json:"platform"` + ProfileImageURL string `json:"profile_image_url"` + ExpiredTime int `json:"expired_time"` + ExpiresIn int `json:"expires_in"` + PlatformScreenName string `json:"platform_screen_name"` + UserID int64 `json:"user_id"` + PlatformUID string `json:"platform_uid"` + SecPlatformUID string `json:"sec_platform_uid"` + PlatformAppID int `json:"platform_app_id"` + ModifyTime int `json:"modify_time"` + AccessToken string `json:"access_token"` + OpenID string `json:"open_id"` +} +type OperStaffRelationInfo struct { + HasPassword int `json:"has_password"` + Mobile string `json:"mobile"` + SecOperStaffUserID string `json:"sec_oper_staff_user_id"` + RelationMobileCountryCode int `json:"relation_mobile_country_code"` +} +type UserInfo struct { + AppID int `json:"app_id"` + AppUserInfo AppUserInfo `json:"app_user_info"` + AvatarURL string `json:"avatar_url"` + BgImgURL string `json:"bg_img_url"` + BuiAuditInfo BuiAuditInfo `json:"bui_audit_info"` + CanBeFoundByPhone int `json:"can_be_found_by_phone"` + Connects []Connects `json:"connects"` + CountryCode int `json:"country_code"` + Description string `json:"description"` + DeviceID int `json:"device_id"` + Email string `json:"email"` + EmailCollected bool `json:"email_collected"` + Gender int `json:"gender"` + HasPassword int `json:"has_password"` + HmRegion int `json:"hm_region"` + IsBlocked int `json:"is_blocked"` + IsBlocking int `json:"is_blocking"` + IsRecommendAllowed int `json:"is_recommend_allowed"` + IsVisitorAccount bool `json:"is_visitor_account"` + Mobile string `json:"mobile"` + Name string `json:"name"` + NeedCheckBindStatus bool `json:"need_check_bind_status"` + OdinUserType int `json:"odin_user_type"` + OperStaffRelationInfo OperStaffRelationInfo `json:"oper_staff_relation_info"` + PhoneCollected bool `json:"phone_collected"` + RecommendHintMessage string `json:"recommend_hint_message"` + ScreenName string `json:"screen_name"` + SecUserID string `json:"sec_user_id"` + SessionKey string `json:"session_key"` + UseHmRegion bool `json:"use_hm_region"` + UserCreateTime int `json:"user_create_time"` + UserID int64 `json:"user_id"` + UserIDStr string `json:"user_id_str"` + UserVerified bool `json:"user_verified"` + VerifiedContent string `json:"verified_content"` +} + +// UploadToken 上传令牌配置 +type UploadToken struct { + Alice map[string]UploadAuthToken + Samantha MediaUploadAuthToken +} + +// UploadAuthToken 多种类型的上传配置：图片/文件 +type UploadAuthToken struct { + ServiceID string `json:"service_id"` + UploadPathPrefix string `json:"upload_path_prefix"` + Auth struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` + ExpiredTime time.Time `json:"expired_time"` + CurrentTime time.Time `json:"current_time"` + } `json:"auth"` + UploadHost string `json:"upload_host"` +} + +// MediaUploadAuthToken 媒体上传配置 +type MediaUploadAuthToken struct { + StsToken struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` + ExpiredTime time.Time `json:"expired_time"` + CurrentTime time.Time `json:"current_time"` + } `json:"sts_token"` + UploadInfo struct { + VideoHost string `json:"video_host"` + SpaceName string `json:"space_name"` + } `json:"upload_info"` +} + +type UploadAuthTokenResp struct { + BaseResp + Data UploadAuthToken `json:"data"` +} + +type MediaUploadAuthTokenResp struct { + BaseResp + Data MediaUploadAuthToken `json:"data"` +} + +type ResponseMetadata struct { + RequestID string `json:"RequestId"` + Action string `json:"Action"` + Version string `json:"Version"` + Service string `json:"Service"` + Region string `json:"Region"` + Error struct { + CodeN int `json:"CodeN,omitempty"` + Code string `json:"Code,omitempty"` + Message string `json:"Message,omitempty"` + } `json:"Error,omitempty"` +} + +type UploadConfig struct { + UploadAddress UploadAddress `json:"UploadAddress"` + FallbackUploadAddress FallbackUploadAddress `json:"FallbackUploadAddress"` + InnerUploadAddress InnerUploadAddress `json:"InnerUploadAddress"` + RequestID string `json:"RequestId"` + SDKParam interface{} `json:"SDKParam"` +} + +type UploadConfigResp struct { + ResponseMetadata `json:"ResponseMetadata"` + Result UploadConfig `json:"Result"` +} + +// StoreInfo 存储信息 +type StoreInfo struct { + StoreURI string `json:"StoreUri"` + Auth string `json:"Auth"` + UploadID string `json:"UploadID"` + UploadHeader map[string]interface{} `json:"UploadHeader,omitempty"` + StorageHeader map[string]interface{} `json:"StorageHeader,omitempty"` +} + +// UploadAddress 上传地址信息 +type UploadAddress struct { + StoreInfos []StoreInfo `json:"StoreInfos"` + UploadHosts []string `json:"UploadHosts"` + UploadHeader map[string]interface{} `json:"UploadHeader"` + SessionKey string `json:"SessionKey"` + Cloud string `json:"Cloud"` +} + +// FallbackUploadAddress 备用上传地址 +type FallbackUploadAddress struct { + StoreInfos []StoreInfo `json:"StoreInfos"` + UploadHosts []string `json:"UploadHosts"` + UploadHeader map[string]interface{} `json:"UploadHeader"` + SessionKey string `json:"SessionKey"` + Cloud string `json:"Cloud"` +} + +// UploadNode 上传节点信息 +type UploadNode struct { + Vid string `json:"Vid"` + Vids []string `json:"Vids"` + StoreInfos []StoreInfo `json:"StoreInfos"` + UploadHost string `json:"UploadHost"` + UploadHeader map[string]interface{} `json:"UploadHeader"` + Type string `json:"Type"` + Protocol string `json:"Protocol"` + SessionKey string `json:"SessionKey"` + NodeConfig struct { + UploadMode string `json:"UploadMode"` + } `json:"NodeConfig"` + Cluster string `json:"Cluster"` +} + +// AdvanceOption 高级选项 +type AdvanceOption struct { + Parallel int `json:"Parallel"` + Stream int `json:"Stream"` + SliceSize int `json:"SliceSize"` + EncryptionKey string `json:"EncryptionKey"` +} + +// InnerUploadAddress 内部上传地址 +type InnerUploadAddress struct { + UploadNodes []UploadNode `json:"UploadNodes"` + AdvanceOption AdvanceOption `json:"AdvanceOption"` +} + +// UploadPart 上传分片信息 +type UploadPart struct { + UploadId string `json:"uploadid,omitempty"` + PartNumber string `json:"part_number,omitempty"` + Crc32 string `json:"crc32,omitempty"` + Etag string `json:"etag,omitempty"` + Mode string `json:"mode,omitempty"` +} + +// UploadResp 上传响应体 +type UploadResp struct { + Code int `json:"code"` + ApiVersion string `json:"apiversion"` + Message string `json:"message"` + Data UploadPart `json:"data"` +} + +type VideoCommitUpload struct { + Vid string `json:"Vid"` + VideoMeta struct { + URI string `json:"Uri"` + Height int `json:"Height"` + Width int `json:"Width"` + OriginHeight int `json:"OriginHeight"` + OriginWidth int `json:"OriginWidth"` + Duration float64 `json:"Duration"` + Bitrate int `json:"Bitrate"` + Md5 string `json:"Md5"` + Format string `json:"Format"` + Size int `json:"Size"` + FileType string `json:"FileType"` + Codec string `json:"Codec"` + } `json:"VideoMeta"` + WorkflowInput struct { + TemplateID string `json:"TemplateId"` + } `json:"WorkflowInput"` + GetPosterMode string `json:"GetPosterMode"` +} + +type VideoCommitUploadResp struct { + ResponseMetadata ResponseMetadata `json:"ResponseMetadata"` + Result struct { + RequestID string `json:"RequestId"` + Results []VideoCommitUpload `json:"Results"` + } `json:"Result"` +} + +type CommonResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` // 错误情况下的消息 + Data json.RawMessage `json:"data,omitempty"` // 原始数据,稍后解析 + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + Locale string `json:"locale"` + } `json:"error,omitempty"` +} + +// IsSuccess 判断响应是否成功 +func (r *CommonResp) IsSuccess() bool { + return r.Code == 0 +} + +// GetError 获取错误信息 +func (r *CommonResp) GetError() error { + if r.IsSuccess() { + return nil + } + // 优先使用message字段 + errMsg := r.Message + if errMsg == "" { + errMsg = r.Msg + } + // 如果error对象存在且有详细消息,则使用error中的信息 + if r.Error != nil && r.Error.Message != "" { + errMsg = r.Error.Message + } + + return fmt.Errorf("[doubao] API error (code: %d): %s", r.Code, errMsg) +} + +// UnmarshalData 将data字段解析为指定类型 +func (r *CommonResp) UnmarshalData(v interface{}) error { + if !r.IsSuccess() { + return r.GetError() + } + + if len(r.Data) == 0 { + return nil + } + + return json.Unmarshal(r.Data, v) } diff --git a/drivers/doubao/util.go b/drivers/doubao/util.go index 977691c03a6..348c0aa0c14 100644 --- a/drivers/doubao/util.go +++ b/drivers/doubao/util.go @@ -1,38 +1,970 @@ package doubao import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" - + "fmt" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/errgroup" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" log "github.com/sirupsen/logrus" + "hash/crc32" + "io" + "math" + "math/rand" + "net/http" + "net/url" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +const ( + DirectoryType = 1 + FileType = 2 + LinkType = 3 + ImageType = 4 + PagesType = 5 + VideoType = 6 + AudioType = 7 + MeetingMinutesType = 8 +) + +var FileNodeType = map[int]string{ + 1: "directory", + 2: "file", + 3: "link", + 4: "image", + 5: "pages", + 6: "video", + 7: "audio", + 8: "meeting_minutes", +} + +const ( + BaseURL = "https://www.doubao.com" + FileDataType = "file" + ImgDataType = "image" + VideoDataType = "video" + DefaultChunkSize = int64(5 * 1024 * 1024) // 5MB + MaxRetryAttempts = 3 // 最大重试次数 + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" + Region = "cn-north-1" + UploadTimeout = 3 * time.Minute ) // do others that not defined in Driver interface func (d *Doubao) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - url := "https://www.doubao.com" + path + reqUrl := BaseURL + path req := base.RestyClient.R() req.SetHeader("Cookie", d.Cookie) if callback != nil { callback(req) } - var r BaseResp - req.SetResult(&r) - res, err := req.Execute(method, url) + + var commonResp CommonResp + + res, err := req.Execute(method, reqUrl) log.Debugln(res.String()) if err != nil { return nil, err } - // 业务状态码检查（优先于HTTP状态码） - if r.Code != 0 { - return res.Body(), errors.New(r.Msg) + body := res.Body() + // 先解析为通用响应 + if err = json.Unmarshal(body, &commonResp); err != nil { + return nil, err + } + // 检查响应是否成功 + if !commonResp.IsSuccess() { + return body, commonResp.GetError() } + if resp != nil { - err = utils.Json.Unmarshal(res.Body(), resp) + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func (d *Doubao) getFiles(dirId, cursor string) (resp []File, err error) { + var r NodeInfoResp + + var body = base.Json{ + "node_id": dirId, + } + // 如果有游标，则设置游标和大小 + if cursor != "" { + body["cursor"] = cursor + body["size"] = 50 + } else { + body["need_full_path"] = false + } + + _, err = d.request("/samantha/aispace/node_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &r) + if err != nil { + return nil, err + } + + if r.Data.Children != nil { + resp = r.Data.Children + } + + if r.Data.NextCursor != "-1" { + // 递归获取下一页 + nextFiles, err := d.getFiles(dirId, r.Data.NextCursor) if err != nil { return nil, err } + + resp = append(r.Data.Children, nextFiles...) + } + + return resp, err +} + +func (d *Doubao) getUserInfo() (UserInfo, error) { + var r UserInfoResp + + _, err := d.request("/passport/account/info/v2/", http.MethodGet, nil, &r) + if err != nil { + return UserInfo{}, err + } + + return r.Data, err +} + +// 签名请求 +func (d *Doubao) signRequest(req *resty.Request, method, tokenType, uploadUrl string) error { + parsedUrl, err := url.Parse(uploadUrl) + if err != nil { + return fmt.Errorf("invalid URL format: %w", err) + } + + var accessKeyId, secretAccessKey, sessionToken string + var serviceName string + + if tokenType == VideoDataType { + accessKeyId = d.UploadToken.Samantha.StsToken.AccessKeyID + secretAccessKey = d.UploadToken.Samantha.StsToken.SecretAccessKey + sessionToken = d.UploadToken.Samantha.StsToken.SessionToken + serviceName = "vod" + } else { + accessKeyId = d.UploadToken.Alice[tokenType].Auth.AccessKeyID + secretAccessKey = d.UploadToken.Alice[tokenType].Auth.SecretAccessKey + sessionToken = d.UploadToken.Alice[tokenType].Auth.SessionToken + serviceName = "imagex" + } + + // 当前时间，格式为 ISO8601 + now := time.Now().UTC() + amzDate := now.Format("20060102T150405Z") + dateStamp := now.Format("20060102") + + req.SetHeader("X-Amz-Date", amzDate) + + if sessionToken != "" { + req.SetHeader("X-Amz-Security-Token", sessionToken) + } + + // 计算请求体的SHA256哈希 + var bodyHash string + if req.Body != nil { + bodyBytes, ok := req.Body.([]byte) + if !ok { + return fmt.Errorf("request body must be []byte") + } + + bodyHash = hashSHA256(string(bodyBytes)) + req.SetHeader("X-Amz-Content-Sha256", bodyHash) + } else { + bodyHash = hashSHA256("") + } + + // 创建规范请求 + canonicalURI := parsedUrl.Path + if canonicalURI == "" { + canonicalURI = "/" + } + + // 查询参数按照字母顺序排序 + canonicalQueryString := getCanonicalQueryString(req.QueryParam) + // 规范请求头 + canonicalHeaders, signedHeaders := getCanonicalHeadersFromMap(req.Header) + canonicalRequest := method + "\n" + + canonicalURI + "\n" + + canonicalQueryString + "\n" + + canonicalHeaders + "\n" + + signedHeaders + "\n" + + bodyHash + + algorithm := "AWS4-HMAC-SHA256" + credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", dateStamp, Region, serviceName) + + stringToSign := algorithm + "\n" + + amzDate + "\n" + + credentialScope + "\n" + + hashSHA256(canonicalRequest) + // 计算签名密钥 + signingKey := getSigningKey(secretAccessKey, dateStamp, Region, serviceName) + // 计算签名 + signature := hmacSHA256Hex(signingKey, stringToSign) + // 构建授权头 + authorizationHeader := fmt.Sprintf( + "%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", + algorithm, + accessKeyId, + credentialScope, + signedHeaders, + signature, + ) + + req.SetHeader("Authorization", authorizationHeader) + + return nil +} + +func (d *Doubao) requestApi(url, method, tokenType string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "user-agent": UserAgent, + }) + + if method == http.MethodPost { + req.SetHeader("Content-Type", "text/plain;charset=UTF-8") + } + + if callback != nil { + callback(req) + } + + if resp != nil { + req.SetResult(resp) + } + + // 使用自定义AWS SigV4签名 + err := d.signRequest(req, method, tokenType, url) + if err != nil { + return nil, err + } + + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Doubao) initUploadToken() (*UploadToken, error) { + uploadToken := &UploadToken{ + Alice: make(map[string]UploadAuthToken), + Samantha: MediaUploadAuthToken{}, + } + + fileAuthToken, err := d.getUploadAuthToken(FileDataType) + if err != nil { + return nil, err + } + + imgAuthToken, err := d.getUploadAuthToken(ImgDataType) + if err != nil { + return nil, err + } + + mediaAuthToken, err := d.getSamantaUploadAuthToken() + if err != nil { + return nil, err + } + + uploadToken.Alice[FileDataType] = fileAuthToken + uploadToken.Alice[ImgDataType] = imgAuthToken + uploadToken.Samantha = mediaAuthToken + + return uploadToken, nil +} + +func (d *Doubao) getUploadAuthToken(dataType string) (ut UploadAuthToken, err error) { + var r UploadAuthTokenResp + _, err = d.request("/alice/upload/auth_token", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "scene": "bot_chat", + "data_type": dataType, + }) + }, &r) + + return r.Data, err +} + +func (d *Doubao) getSamantaUploadAuthToken() (mt MediaUploadAuthToken, err error) { + var r MediaUploadAuthTokenResp + _, err = d.request("/samantha/media/get_upload_token", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{}) + }, &r) + + return r.Data, err +} + +// getUploadConfig 获取上传配置信息 +func (d *Doubao) getUploadConfig(upConfig *UploadConfig, dataType string, file model.FileStreamer) error { + tokenType := dataType + // 配置参数函数 + configureParams := func() (string, map[string]string) { + var uploadUrl string + var params map[string]string + // 根据数据类型设置不同的上传参数 + switch dataType { + case VideoDataType: + // 音频/视频类型 - 使用uploadToken.Samantha的配置 + uploadUrl = d.UploadToken.Samantha.UploadInfo.VideoHost + params = map[string]string{ + "Action": "ApplyUploadInner", + "Version": "2020-11-19", + "SpaceName": d.UploadToken.Samantha.UploadInfo.SpaceName, + "FileType": "video", + "IsInner": "1", + "NeedFallback": "true", + "FileSize": strconv.FormatInt(file.GetSize(), 10), + "s": randomString(), + } + case ImgDataType, FileDataType: + // 图片或其他文件类型 - 使用uploadToken.Alice对应配置 + uploadUrl = "https://" + d.UploadToken.Alice[dataType].UploadHost + params = map[string]string{ + "Action": "ApplyImageUpload", + "Version": "2018-08-01", + "ServiceId": d.UploadToken.Alice[dataType].ServiceID, + "NeedFallback": "true", + "FileSize": strconv.FormatInt(file.GetSize(), 10), + "FileExtension": filepath.Ext(file.GetName()), + "s": randomString(), + } + } + return uploadUrl, params + } + + // 获取初始参数 + uploadUrl, params := configureParams() + + tokenRefreshed := false + var configResp UploadConfigResp + + err := d._retryOperation("get upload_config", func() error { + configResp = UploadConfigResp{} + + _, err := d.requestApi(uploadUrl, http.MethodGet, tokenType, func(req *resty.Request) { + req.SetQueryParams(params) + }, &configResp) + if err != nil { + return err + } + + if configResp.ResponseMetadata.Error.Code == "" { + *upConfig = configResp.Result + return nil + } + + // 100028 凭证过期 + if configResp.ResponseMetadata.Error.CodeN == 100028 && !tokenRefreshed { + log.Debugln("[doubao] Upload token expired, re-fetching...") + newToken, err := d.initUploadToken() + if err != nil { + return fmt.Errorf("failed to refresh token: %w", err) + } + + d.UploadToken = newToken + tokenRefreshed = true + uploadUrl, params = configureParams() + + return retry.Error{errors.New("token refreshed, retry needed")} + } + + return fmt.Errorf("get upload_config failed: %s", configResp.ResponseMetadata.Error.Message) + }) + + return err +} + +// uploadNode 上传 文件信息 +func (d *Doubao) uploadNode(uploadConfig *UploadConfig, dir model.Obj, file model.FileStreamer, dataType string) (UploadNodeResp, error) { + reqUuid := uuid.New().String() + var key string + var nodeType int + + mimetype := file.GetMimetype() + switch dataType { + case VideoDataType: + key = uploadConfig.InnerUploadAddress.UploadNodes[0].Vid + if strings.HasPrefix(mimetype, "audio/") { + nodeType = AudioType // 音频类型 + } else { + nodeType = VideoType // 视频类型 + } + case ImgDataType: + key = uploadConfig.InnerUploadAddress.UploadNodes[0].StoreInfos[0].StoreURI + nodeType = ImageType // 图片类型 + default: // FileDataType + key = uploadConfig.InnerUploadAddress.UploadNodes[0].StoreInfos[0].StoreURI + nodeType = FileType // 文件类型 + } + + var r UploadNodeResp + _, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "node_list": []base.Json{ + { + "local_id": reqUuid, + "parent_id": dir.GetID(), + "name": file.GetName(), + "key": key, + "node_content": base.Json{}, + "node_type": nodeType, + "size": file.GetSize(), + }, + }, + "request_id": reqUuid, + }) + }, &r) + + return r, err +} + +// Upload 普通上传实现 +func (d *Doubao) Upload(config *UploadConfig, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, dataType string) (model.Obj, error) { + data, err := io.ReadAll(file) + if err != nil { + return nil, err + } + + // 计算CRC32 + crc32Hash := crc32.NewIEEE() + crc32Hash.Write(data) + crc32Value := hex.EncodeToString(crc32Hash.Sum(nil)) + + // 构建请求路径 + uploadNode := config.InnerUploadAddress.UploadNodes[0] + storeInfo := uploadNode.StoreInfos[0] + uploadUrl := fmt.Sprintf("https://%s/upload/v1/%s", uploadNode.UploadHost, storeInfo.StoreURI) + + uploadResp := UploadResp{} + + if _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Content-Type": "application/octet-stream", + "Content-Crc32": crc32Value, + "Content-Length": fmt.Sprintf("%d", len(data)), + "Content-Disposition": fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI)), + }) + + req.SetBody(data) + }, &uploadResp); err != nil { + return nil, err + } + + if uploadResp.Code != 2000 { + return nil, fmt.Errorf("upload failed: %s", uploadResp.Message) + } + + uploadNodeResp, err := d.uploadNode(config, dstDir, file, dataType) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: uploadNodeResp.Data.NodeList[0].ID, + Name: uploadNodeResp.Data.NodeList[0].Name, + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +// UploadByMultipart 分片上传 +func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fileSize int64, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, dataType string) (model.Obj, error) { + // 构建请求路径 + uploadNode := config.InnerUploadAddress.UploadNodes[0] + storeInfo := uploadNode.StoreInfos[0] + uploadUrl := fmt.Sprintf("https://%s/upload/v1/%s", uploadNode.UploadHost, storeInfo.StoreURI) + // 初始化分片上传 + var uploadID string + err := d._retryOperation("Initialize multipart upload", func() error { + var err error + uploadID, err = d.initMultipartUpload(config, uploadUrl, storeInfo) + return err + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize multipart upload: %w", err) + } + // 准备分片参数 + chunkSize := DefaultChunkSize + if config.InnerUploadAddress.AdvanceOption.SliceSize > 0 { + chunkSize = int64(config.InnerUploadAddress.AdvanceOption.SliceSize) + } + totalParts := (fileSize + chunkSize - 1) / chunkSize + // 创建分片信息组 + parts := make([]UploadPart, totalParts) + // 缓存文件 + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, fmt.Errorf("failed to cache file: %w", err) + } + defer tempFile.Close() + up(10.0) // 更新进度 + // 设置并行上传 + threadG, uploadCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + var partsMutex sync.Mutex + // 并行上传所有分片 + for partIndex := int64(0); partIndex < totalParts; partIndex++ { + if utils.IsCanceled(uploadCtx) { + break + } + partIndex := partIndex + partNumber := partIndex + 1 // 分片编号从1开始 + + threadG.Go(func(ctx context.Context) error { + // 计算此分片的大小和偏移 + offset := partIndex * chunkSize + size := chunkSize + if partIndex == totalParts-1 { + size = fileSize - offset + } + + limitedReader := driver.NewLimitedUploadStream(ctx, io.NewSectionReader(tempFile, offset, size)) + // 读取数据到内存 + data, err := io.ReadAll(limitedReader) + if err != nil { + return fmt.Errorf("failed to read part %d: %w", partNumber, err) + } + // 计算CRC32 + crc32Value := calculateCRC32(data) + // 使用_retryOperation上传分片 + var uploadPart UploadPart + if err = d._retryOperation(fmt.Sprintf("Upload part %d", partNumber), func() error { + var err error + uploadPart, err = d.uploadPart(config, uploadUrl, uploadID, partNumber, data, crc32Value) + return err + }); err != nil { + return fmt.Errorf("part %d upload failed: %w", partNumber, err) + } + // 记录成功上传的分片 + partsMutex.Lock() + parts[partIndex] = UploadPart{ + PartNumber: strconv.FormatInt(partNumber, 10), + Etag: uploadPart.Etag, + Crc32: crc32Value, + } + partsMutex.Unlock() + // 更新进度 + progress := 10.0 + 90.0*float64(threadG.Success()+1)/float64(totalParts) + up(math.Min(progress, 95.0)) + + return nil + }) + } + + if err = threadG.Wait(); err != nil { + return nil, err } + // 完成上传-分片合并 + if err = d._retryOperation("Complete multipart upload", func() error { + return d.completeMultipartUpload(config, uploadUrl, uploadID, parts) + }); err != nil { + return nil, fmt.Errorf("failed to complete multipart upload: %w", err) + } + // 提交上传 + if err = d._retryOperation("Commit upload", func() error { + return d.commitMultipartUpload(config) + }); err != nil { + return nil, fmt.Errorf("failed to commit upload: %w", err) + } + + up(98.0) // 更新到98% + // 上传节点信息 + var uploadNodeResp UploadNodeResp + + if err = d._retryOperation("Upload node", func() error { + var err error + uploadNodeResp, err = d.uploadNode(config, dstDir, file, dataType) + return err + }); err != nil { + return nil, fmt.Errorf("failed to upload node: %w", err) + } + + up(100.0) // 完成上传 + + return &model.Object{ + ID: uploadNodeResp.Data.NodeList[0].ID, + Name: uploadNodeResp.Data.NodeList[0].Name, + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +// 统一上传请求方法 +func (d *Doubao) uploadRequest(uploadUrl string, method string, storeInfo StoreInfo, callback base.ReqCallback, resp interface{}) ([]byte, error) { + client := resty.New() + client.SetTransport(&http.Transport{ + DisableKeepAlives: true, // 禁用连接复用 + ForceAttemptHTTP2: false, // 强制使用HTTP/1.1 + }) + client.SetTimeout(UploadTimeout) + + req := client.R() + req.SetHeaders(map[string]string{ + "Host": strings.Split(uploadUrl, "/")[2], + "Referer": BaseURL + "/", + "Origin": BaseURL, + "User-Agent": UserAgent, + "X-Storage-U": d.UserId, + "Authorization": storeInfo.Auth, + }) + + if method == http.MethodPost { + req.SetHeader("Content-Type", "text/plain;charset=UTF-8") + } + + if callback != nil { + callback(req) + } + + if resp != nil { + req.SetResult(resp) + } + + res, err := req.Execute(method, uploadUrl) + if err != nil && err != io.EOF { + return nil, fmt.Errorf("upload request failed: %w", err) + } + return res.Body(), nil } + +// 初始化分片上传 +func (d *Doubao) initMultipartUpload(config *UploadConfig, uploadUrl string, storeInfo StoreInfo) (uploadId string, err error) { + uploadResp := UploadResp{} + + _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "uploadmode": "part", + "phase": "init", + }) + }, &uploadResp) + + if err != nil { + return uploadId, err + } + + if uploadResp.Code != 2000 { + return uploadId, fmt.Errorf("init upload failed: %s", uploadResp.Message) + } + + return uploadResp.Data.UploadId, nil +} + +// 分片上传实现 +func (d *Doubao) uploadPart(config *UploadConfig, uploadUrl, uploadID string, partNumber int64, data []byte, crc32Value string) (resp UploadPart, err error) { + uploadResp := UploadResp{} + storeInfo := config.InnerUploadAddress.UploadNodes[0].StoreInfos[0] + + _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Content-Type": "application/octet-stream", + "Content-Crc32": crc32Value, + "Content-Length": fmt.Sprintf("%d", len(data)), + "Content-Disposition": fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI)), + }) + + req.SetQueryParams(map[string]string{ + "uploadid": uploadID, + "part_number": strconv.FormatInt(partNumber, 10), + "phase": "transfer", + }) + + req.SetBody(data) + req.SetContentLength(true) + }, &uploadResp) + + if err != nil { + return resp, err + } + + if uploadResp.Code != 2000 { + return resp, fmt.Errorf("upload part failed: %s", uploadResp.Message) + } else if uploadResp.Data.Crc32 != crc32Value { + return resp, fmt.Errorf("upload part failed: crc32 mismatch, expected %s, got %s", crc32Value, uploadResp.Data.Crc32) + } + + return uploadResp.Data, nil +} + +// 完成分片上传 +func (d *Doubao) completeMultipartUpload(config *UploadConfig, uploadUrl, uploadID string, parts []UploadPart) error { + uploadResp := UploadResp{} + + storeInfo := config.InnerUploadAddress.UploadNodes[0].StoreInfos[0] + + body := _convertUploadParts(parts) + + err := utils.Retry(MaxRetryAttempts, time.Second, func() (err error) { + _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "uploadid": uploadID, + "phase": "finish", + "uploadmode": "part", + }) + req.SetBody(body) + }, &uploadResp) + + if err != nil { + return err + } + // 检查响应状态码 2000 成功 4024 分片合并中 + if uploadResp.Code != 2000 && uploadResp.Code != 4024 { + return fmt.Errorf("finish upload failed: %s", uploadResp.Message) + } + + return err + }) + + if err != nil { + return fmt.Errorf("failed to complete multipart upload: %w", err) + } + + return nil +} + +func (d *Doubao) commitMultipartUpload(uploadConfig *UploadConfig) error { + uploadUrl := d.UploadToken.Samantha.UploadInfo.VideoHost + params := map[string]string{ + "Action": "CommitUploadInner", + "Version": "2020-11-19", + "SpaceName": d.UploadToken.Samantha.UploadInfo.SpaceName, + } + tokenType := VideoDataType + + videoCommitUploadResp := VideoCommitUploadResp{} + + jsonBytes, err := json.Marshal(base.Json{ + "SessionKey": uploadConfig.InnerUploadAddress.UploadNodes[0].SessionKey, + "Functions": []base.Json{}, + }) + if err != nil { + return fmt.Errorf("failed to marshal request data: %w", err) + } + + _, err = d.requestApi(uploadUrl, http.MethodPost, tokenType, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetQueryParams(params) + req.SetBody(jsonBytes) + + }, &videoCommitUploadResp) + if err != nil { + return err + } + + return nil +} + +// 计算CRC32 +func calculateCRC32(data []byte) string { + hash := crc32.NewIEEE() + hash.Write(data) + return hex.EncodeToString(hash.Sum(nil)) +} + +// _retryOperation 操作重试 +func (d *Doubao) _retryOperation(operation string, fn func() error) error { + return retry.Do( + fn, + retry.Attempts(MaxRetryAttempts), + retry.Delay(500*time.Millisecond), + retry.DelayType(retry.BackOffDelay), + retry.MaxJitter(200*time.Millisecond), + retry.OnRetry(func(n uint, err error) { + log.Debugf("[doubao] %s retry #%d: %v", operation, n+1, err) + }), + ) +} + +// _convertUploadParts 将分片信息转换为字符串 +func _convertUploadParts(parts []UploadPart) string { + if len(parts) == 0 { + return "" + } + + var result strings.Builder + + for i, part := range parts { + if i > 0 { + result.WriteString(",") + } + result.WriteString(fmt.Sprintf("%s:%s", part.PartNumber, part.Crc32)) + } + + return result.String() +} + +// 获取规范查询字符串 +func getCanonicalQueryString(query url.Values) string { + if len(query) == 0 { + return "" + } + + keys := make([]string, 0, len(query)) + for k := range query { + keys = append(keys, k) + } + sort.Strings(keys) + + parts := make([]string, 0, len(keys)) + for _, k := range keys { + values := query[k] + for _, v := range values { + parts = append(parts, urlEncode(k)+"="+urlEncode(v)) + } + } + + return strings.Join(parts, "&") +} + +func urlEncode(s string) string { + s = url.QueryEscape(s) + s = strings.ReplaceAll(s, "+", "%20") + return s +} + +// 获取规范头信息和已签名头列表 +func getCanonicalHeadersFromMap(headers map[string][]string) (string, string) { + // 不可签名的头部列表 + unsignableHeaders := map[string]bool{ + "authorization": true, + "content-type": true, + "content-length": true, + "user-agent": true, + "presigned-expires": true, + "expect": true, + "x-amzn-trace-id": true, + } + headerValues := make(map[string]string) + var signedHeadersList []string + + for k, v := range headers { + if len(v) == 0 { + continue + } + + lowerKey := strings.ToLower(k) + // 检查是否可签名 + if strings.HasPrefix(lowerKey, "x-amz-") || !unsignableHeaders[lowerKey] { + value := strings.TrimSpace(v[0]) + value = strings.Join(strings.Fields(value), " ") + headerValues[lowerKey] = value + signedHeadersList = append(signedHeadersList, lowerKey) + } + } + + sort.Strings(signedHeadersList) + + var canonicalHeadersStr strings.Builder + for _, key := range signedHeadersList { + canonicalHeadersStr.WriteString(key) + canonicalHeadersStr.WriteString(":") + canonicalHeadersStr.WriteString(headerValues[key]) + canonicalHeadersStr.WriteString("\n") + } + + signedHeaders := strings.Join(signedHeadersList, ";") + + return canonicalHeadersStr.String(), signedHeaders +} + +// 计算HMAC-SHA256 +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + h.Write([]byte(data)) + return h.Sum(nil) +} + +// 计算HMAC-SHA256并返回十六进制字符串 +func hmacSHA256Hex(key []byte, data string) string { + return hex.EncodeToString(hmacSHA256(key, data)) +} + +// 计算SHA256哈希并返回十六进制字符串 +func hashSHA256(data string) string { + h := sha256.New() + h.Write([]byte(data)) + return hex.EncodeToString(h.Sum(nil)) +} + +// 获取签名密钥 +func getSigningKey(secretKey, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secretKey), dateStamp) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + kSigning := hmacSHA256(kService, "aws4_request") + return kSigning +} + +// generateContentDisposition 生成符合RFC 5987标准的Content-Disposition头部 +func generateContentDisposition(filename string) string { + // 按照RFC 2047进行编码，用于filename部分 + encodedName := urlEncode(filename) + + // 按照RFC 5987进行编码，用于filename*部分 + encodedNameRFC5987 := encodeRFC5987(filename) + + return fmt.Sprintf("attachment; filename=\"%s\"; filename*=utf-8''%s", + encodedName, encodedNameRFC5987) +} + +// encodeRFC5987 按照RFC 5987规范编码字符串，适用于HTTP头部参数中的非ASCII字符 +func encodeRFC5987(s string) string { + var buf strings.Builder + for _, r := range []byte(s) { + // 根据RFC 5987，只有字母、数字和部分特殊符号可以不编码 + if (r >= 'a' && r <= 'z') || + (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || + r == '-' || r == '.' || r == '_' || r == '~' { + buf.WriteByte(r) + } else { + // 其他字符都需要百分号编码 + fmt.Fprintf(&buf, "%%%02X", r) + } + } + return buf.String() +} + +func randomString() string { + const charset = "0123456789abcdefghijklmnopqrstuvwxyz" + const length = 11 // 11位随机字符串 + + var sb strings.Builder + sb.Grow(length) + + for i := 0; i < length; i++ { + sb.WriteByte(charset[rand.Intn(len(charset))]) + } + + return sb.String() +} From 88abb323cb8e596e8053ce57f890dcc7286fe012 Mon Sep 17 00:00:00 2001 From: Lee CQ <47050568+lee-cq@users.noreply.github.com> Date: Sat, 12 Apr 2025 17:27:56 +0800 Subject: [PATCH 502/659] feat(url-tree): implement the Put interface to support adding links directly to the UrlTree on the web side (#8312) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(url-tree)支持PUT * feat(url-tree) UrlTree更新时，需要将路径和内容分割 #8303 * fix: stdpath.Join call Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Andy Hsu Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- drivers/url_tree/driver.go | 20 +++++++++++++++++++- internal/op/fs.go | 7 +++++++ internal/op/path.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/drivers/url_tree/driver.go b/drivers/url_tree/driver.go index f97d5cc5e76..049bd2db63f 100644 --- a/drivers/url_tree/driver.go +++ b/drivers/url_tree/driver.go @@ -243,7 +243,25 @@ func (d *Urls) PutURL(ctx context.Context, dstDir model.Obj, name, url string) ( } func (d *Urls) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - return errs.UploadNotSupported + if !d.Writable { + return errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + node := GetNodeFromRootByPath(d.root, dstDir.GetPath()) // parent + if node == nil { + return errs.ObjectNotFound + } + if node.isFile() { + return errs.NotFolder + } + file, err := parseFileLine(stream.GetName(), d.HeadSize) + if err != nil { + return err + } + node.Children = append(node.Children, file) + d.updateStorage() + return nil } func (d *Urls) updateStorage() { diff --git a/internal/op/fs.go b/internal/op/fs.go index 99c2fe3411e..64e993356bc 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -10,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/generic_sync" "github.com/alist-org/alist/v3/pkg/singleflight" "github.com/alist-org/alist/v3/pkg/utils" @@ -517,6 +518,12 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod log.Errorf("failed to close file streamer, %v", err) } }() + // UrlTree PUT + if storage.GetStorage().Driver == "UrlTree" { + var link string + dstDirPath, link = urlTreeSplitLineFormPath(stdpath.Join(dstDirPath, file.GetName())) + file = &stream.FileStream{Obj: &model.Object{Name: link}} + } // if file exist and size = 0, delete it dstDirPath = utils.FixAndCleanPath(dstDirPath) dstPath := stdpath.Join(dstDirPath, file.GetName()) diff --git a/internal/op/path.go b/internal/op/path.go index 27f7e183228..912a00008e1 100644 --- a/internal/op/path.go +++ b/internal/op/path.go @@ -2,6 +2,7 @@ package op import ( "github.com/alist-org/alist/v3/internal/errs" + stdpath "path" "strings" "github.com/alist-org/alist/v3/internal/driver" @@ -27,3 +28,30 @@ func GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath actualPath = utils.FixAndCleanPath(strings.TrimPrefix(rawPath, mountPath)) return } + +// urlTreeSplitLineFormPath 分割path中分割真实路径和UrlTree定义字符串 +func urlTreeSplitLineFormPath(path string) (pp string, file string) { + // url.PathUnescape 会移除 // ，手动加回去 + path = strings.Replace(path, "https:/", "https://", 1) + path = strings.Replace(path, "http:/", "http://", 1) + if strings.Contains(path, ":https:/") || strings.Contains(path, ":http:/") { + // URL-Tree模式 /url_tree_drivr/file_name[:size[:time]]:https://example.com/file + fPath := strings.SplitN(path, ":", 2)[0] + pp, _ = stdpath.Split(fPath) + file = path[len(pp):] + } else if strings.Contains(path, "/https:/") || strings.Contains(path, "/http:/") { + // URL-Tree模式 /url_tree_drivr/https://example.com/file + index := strings.Index(path, "/http://") + if index == -1 { + index = strings.Index(path, "/https://") + } + pp = path[:index] + file = path[index+1:] + } else { + pp, file = stdpath.Split(path) + } + if pp == "" { + pp = "/" + } + return +} From 0a9921fa7948ba51c5f1500cdf7e75bb4658afab Mon Sep 17 00:00:00 2001 From: Yifan Gao Date: Sat, 19 Apr 2025 14:22:12 +0800 Subject: [PATCH 503/659] fix(aliyundrive_open): resolve file duplication issues and improve path handling (#8358) * fix(aliyundrive_open): resolve file duplication issues and improve path handling 1. Fix file duplication by implementing a new removeDuplicateFiles method that cleans up duplicate files after operations 2. Change Move operation to use "ignore" for check_name_mode instead of "refuse" to allow moves when destination has same filename 3. Set Copy operation to handle duplicates by removing them after successful copy 4. Improve path handling for all file operations (Move, Rename, Put, MakeDir) by properly maintaining the full path of objects 5. Implement GetRoot interface for proper root object initialization with correct path 6. Add proper path management in List operation to ensure objects have correct paths 7. Fix path handling in error cases and improve logging of failures * refactor(aliyundrive_open): change error logging to warnings for duplicate file removal Updated the Move, Rename, and Copy methods to log warnings instead of errors when duplicate file removal fails, as the primary operations have already completed successfully. This improves the clarity of logs without affecting the functionality. * Update drivers/aliyundrive_open/util.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- drivers/aliyundrive_open/driver.go | 99 ++++++++++++++++++++++++++---- drivers/aliyundrive_open/util.go | 34 ++++++++++ 2 files changed, 121 insertions(+), 12 deletions(-) diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index a65ba05c5af..394eadb1b8c 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "path/filepath" "time" "github.com/Xhofe/rateg" @@ -14,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) type AliyundriveOpen struct { @@ -72,6 +74,18 @@ func (d *AliyundriveOpen) Drop(ctx context.Context) error { return nil } +// GetRoot implements the driver.GetRooter interface to properly set up the root object +func (d *AliyundriveOpen) GetRoot(ctx context.Context) (model.Obj, error) { + return &model.Object{ + ID: d.RootFolderID, + Path: "/", + Name: "root", + Size: 0, + Modified: d.Modified, + IsFolder: true, + }, nil +} + func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if d.limitList == nil { return nil, fmt.Errorf("driver not init") @@ -80,9 +94,17 @@ func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.Li if err != nil { return nil, err } - return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil + + objs, err := utils.SliceConvert(files, func(src File) (model.Obj, error) { + obj := fileToObj(src) + // Set the correct path for the object + if dir.GetPath() != "" { + obj.Path = filepath.Join(dir.GetPath(), obj.GetName()) + } + return obj, nil }) + + return objs, err } func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link, error) { @@ -132,7 +154,16 @@ func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirN if err != nil { return nil, err } - return fileToObj(newDir), nil + obj := fileToObj(newDir) + + // Set the correct Path for the returned directory object + if parentDir.GetPath() != "" { + obj.Path = filepath.Join(parentDir.GetPath(), dirName) + } else { + obj.Path = "/" + dirName + } + + return obj, nil } func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { @@ -142,20 +173,24 @@ func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (m "drive_id": d.DriveId, "file_id": srcObj.GetID(), "to_parent_file_id": dstDir.GetID(), - "check_name_mode": "refuse", // optional:ignore,auto_rename,refuse + "check_name_mode": "ignore", // optional:ignore,auto_rename,refuse //"new_name": "newName", // The new name to use when a file of the same name exists }).SetResult(&resp) }) if err != nil { return nil, err } - if resp.Exist { - return nil, errors.New("existence of files with the same name") - } if srcObj, ok := srcObj.(*model.ObjThumb); ok { srcObj.ID = resp.FileID srcObj.Modified = time.Now() + srcObj.Path = filepath.Join(dstDir.GetPath(), srcObj.GetName()) + + // Check for duplicate files in the destination directory + if err := d.removeDuplicateFiles(ctx, dstDir.GetPath(), srcObj.GetName(), srcObj.GetID()); err != nil { + // Only log a warning instead of returning an error since the move operation has already completed successfully + log.Warnf("Failed to remove duplicate files after move: %v", err) + } return srcObj, nil } return nil, nil @@ -173,19 +208,47 @@ func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName if err != nil { return nil, err } - return fileToObj(newFile), nil + + // Check for duplicate files in the parent directory + parentPath := filepath.Dir(srcObj.GetPath()) + if err := d.removeDuplicateFiles(ctx, parentPath, newName, newFile.FileId); err != nil { + // Only log a warning instead of returning an error since the rename operation has already completed successfully + log.Warnf("Failed to remove duplicate files after rename: %v", err) + } + + obj := fileToObj(newFile) + + // Set the correct Path for the renamed object + if parentPath != "" && parentPath != "." { + obj.Path = filepath.Join(parentPath, newName) + } else { + obj.Path = "/" + newName + } + + return obj, nil } func (d *AliyundriveOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + var resp MoveOrCopyResp _, err := d.request("/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), "to_parent_file_id": dstDir.GetID(), - "auto_rename": true, - }) + "auto_rename": false, + }).SetResult(&resp) }) - return err + if err != nil { + return err + } + + // Check for duplicate files in the destination directory + if err := d.removeDuplicateFiles(ctx, dstDir.GetPath(), srcObj.GetName(), resp.FileID); err != nil { + // Only log a warning instead of returning an error since the copy operation has already completed successfully + log.Warnf("Failed to remove duplicate files after copy: %v", err) + } + + return nil } func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { @@ -203,7 +266,18 @@ func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { } func (d *AliyundriveOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - return d.upload(ctx, dstDir, stream, up) + obj, err := d.upload(ctx, dstDir, stream, up) + + // Set the correct Path for the returned file object + if obj != nil && obj.GetPath() == "" { + if dstDir.GetPath() != "" { + if objWithPath, ok := obj.(model.SetPath); ok { + objWithPath.SetPath(filepath.Join(dstDir.GetPath(), obj.GetName())) + } + } + } + + return obj, err } func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { @@ -235,3 +309,4 @@ var _ driver.MkdirResult = (*AliyundriveOpen)(nil) var _ driver.MoveResult = (*AliyundriveOpen)(nil) var _ driver.RenameResult = (*AliyundriveOpen)(nil) var _ driver.PutResult = (*AliyundriveOpen)(nil) +var _ driver.GetRooter = (*AliyundriveOpen)(nil) diff --git a/drivers/aliyundrive_open/util.go b/drivers/aliyundrive_open/util.go index 659d7da7257..c3cda10aa88 100644 --- a/drivers/aliyundrive_open/util.go +++ b/drivers/aliyundrive_open/util.go @@ -10,6 +10,7 @@ import ( "time" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -186,3 +187,36 @@ func (d *AliyundriveOpen) getAccessToken() string { } return d.AccessToken } + +// Remove duplicate files with the same name in the given directory path, +// preserving the file with the given skipID if provided +func (d *AliyundriveOpen) removeDuplicateFiles(ctx context.Context, parentPath string, fileName string, skipID string) error { + // Handle empty path (root directory) case + if parentPath == "" { + parentPath = "/" + } + + // List all files in the parent directory + files, err := op.List(ctx, d, parentPath, model.ListArgs{}) + if err != nil { + return err + } + + // Find all files with the same name + var duplicates []model.Obj + for _, file := range files { + if file.GetName() == fileName && file.GetID() != skipID { + duplicates = append(duplicates, file) + } + } + + // Remove all duplicates files, except the file with the given ID + for _, file := range duplicates { + err := d.Remove(ctx, file) + if err != nil { + return err + } + } + + return nil +} From 477c43971f44a591b14506be3fc36f4acc7d1f9a Mon Sep 17 00:00:00 2001 From: asdfghjkl <61342682+anobodys@users.noreply.github.com> Date: Sat, 19 Apr 2025 14:22:43 +0800 Subject: [PATCH 504/659] feat(doubao_share): support doubao_share link (#8376) Co-authored-by: anobodys --- drivers/all.go | 1 + drivers/doubao_share/driver.go | 177 ++++++++ drivers/doubao_share/meta.go | 32 ++ drivers/doubao_share/types.go | 207 +++++++++ drivers/doubao_share/util.go | 744 +++++++++++++++++++++++++++++++++ 5 files changed, 1161 insertions(+) create mode 100644 drivers/doubao_share/driver.go create mode 100644 drivers/doubao_share/meta.go create mode 100644 drivers/doubao_share/types.go create mode 100644 drivers/doubao_share/util.go diff --git a/drivers/all.go b/drivers/all.go index 083d01dcd8c..0b8ce3aa61d 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -24,6 +24,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/crypt" _ "github.com/alist-org/alist/v3/drivers/doubao" + _ "github.com/alist-org/alist/v3/drivers/doubao_share" _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" diff --git a/drivers/doubao_share/driver.go b/drivers/doubao_share/driver.go new file mode 100644 index 00000000000..61076d1ed17 --- /dev/null +++ b/drivers/doubao_share/driver.go @@ -0,0 +1,177 @@ +package doubao_share + +import ( + "context" + "errors" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" + "net/http" +) + +type DoubaoShare struct { + model.Storage + Addition + RootFiles []RootFileList +} + +func (d *DoubaoShare) Config() driver.Config { + return config +} + +func (d *DoubaoShare) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *DoubaoShare) Init(ctx context.Context) error { + // 初始化 虚拟分享列表 + if err := d.initShareList(); err != nil { + return err + } + + return nil +} + +func (d *DoubaoShare) Drop(ctx context.Context) error { + return nil +} + +func (d *DoubaoShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + // 检查是否为根目录 + if dir.GetID() == "" && dir.GetPath() == "/" { + return d.listRootDirectory(ctx) + } + + // 非根目录，处理不同情况 + if fo, ok := dir.(*FileObject); ok { + if fo.ShareID == "" { + // 虚拟目录，需要列出子目录 + return d.listVirtualDirectoryContent(dir) + } else { + // 具有分享ID的目录，获取此分享下的文件 + shareId, relativePath, err := d._findShareAndPath(dir) + if err != nil { + return nil, err + } + return d.getFilesInPath(ctx, shareId, dir.GetID(), relativePath) + } + } + + // 使用通用方法 + shareId, relativePath, err := d._findShareAndPath(dir) + if err != nil { + return nil, err + } + + // 获取指定路径下的文件 + return d.getFilesInPath(ctx, shareId, dir.GetID(), relativePath) +} + +func (d *DoubaoShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var downloadUrl string + + if u, ok := file.(*FileObject); ok { + switch u.NodeType { + case VideoType, AudioType: + var r GetVideoFileUrlResp + _, err := d.request("/samantha/media/get_play_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "key": u.Key, + "share_id": u.ShareID, + "node_id": file.GetID(), + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.OriginalMediaInfo.MainURL + default: + var r GetFileUrlResp + _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{u.Key}, + "type": FileNodeType[u.NodeType], + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.FileUrls[0].MainURL + } + + // 生成标准的Content-Disposition + contentDisposition := generateContentDisposition(u.Name) + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "User-Agent": []string{UserAgent}, + "Content-Disposition": []string{contentDisposition}, + }, + }, nil + } + + return nil, errors.New("can't convert obj to URL") +} + +func (d *DoubaoShare) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + // TODO create folder, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO move obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + // TODO rename obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Remove(ctx context.Context, obj model.Obj) error { + // TODO remove obj, optional + return errs.NotImplement +} + +func (d *DoubaoShare) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // TODO upload file, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *DoubaoShare) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*DoubaoShare)(nil) diff --git a/drivers/doubao_share/meta.go b/drivers/doubao_share/meta.go new file mode 100644 index 00000000000..a749eefb975 --- /dev/null +++ b/drivers/doubao_share/meta.go @@ -0,0 +1,32 @@ +package doubao_share + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Cookie string `json:"cookie" type:"text"` + ShareIds string `json:"share_ids" type:"text" required:"true"` +} + +var config = driver.Config{ + Name: "DoubaoShare", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: true, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &DoubaoShare{} + }) +} diff --git a/drivers/doubao_share/types.go b/drivers/doubao_share/types.go new file mode 100644 index 00000000000..46f226fa5a7 --- /dev/null +++ b/drivers/doubao_share/types.go @@ -0,0 +1,207 @@ +package doubao_share + +import ( + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/internal/model" +) + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +type NodeInfoData struct { + Share ShareInfo `json:"share,omitempty"` + Creator CreatorInfo `json:"creator,omitempty"` + NodeList []File `json:"node_list,omitempty"` + NodeInfo File `json:"node_info,omitempty"` + Children []File `json:"children,omitempty"` + Path FilePath `json:"path,omitempty"` + NextCursor string `json:"next_cursor,omitempty"` + HasMore bool `json:"has_more,omitempty"` +} + +type NodeInfoResp struct { + BaseResp + NodeInfoData `json:"data"` +} + +type RootFileList struct { + ShareID string + VirtualPath string + NodeInfo NodeInfoData + Child *[]RootFileList +} + +type File struct { + ID string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` + Size int64 `json:"size"` + Source int `json:"source"` + NameReviewStatus int `json:"name_review_status"` + ContentReviewStatus int `json:"content_review_status"` + RiskReviewStatus int `json:"risk_review_status"` + ConversationID string `json:"conversation_id"` + ParentID string `json:"parent_id"` + CreateTime int64 `json:"create_time"` + UpdateTime int64 `json:"update_time"` +} + +type FileObject struct { + model.Object + ShareID string + Key string + NodeID string + NodeType int +} + +type ShareInfo struct { + ShareID string `json:"share_id"` + FirstNode struct { + ID string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` + Size int `json:"size"` + Source int `json:"source"` + Content struct { + LinkFileType string `json:"link_file_type"` + ImageWidth int `json:"image_width"` + ImageHeight int `json:"image_height"` + AiSkillStatus int `json:"ai_skill_status"` + } `json:"content"` + NameReviewStatus int `json:"name_review_status"` + ContentReviewStatus int `json:"content_review_status"` + RiskReviewStatus int `json:"risk_review_status"` + ConversationID string `json:"conversation_id"` + ParentID string `json:"parent_id"` + CreateTime int `json:"create_time"` + UpdateTime int `json:"update_time"` + } `json:"first_node"` + NodeCount int `json:"node_count"` + CreateTime int `json:"create_time"` + Channel string `json:"channel"` + InfluencerType int `json:"influencer_type"` +} + +type CreatorInfo struct { + EntityID string `json:"entity_id"` + UserName string `json:"user_name"` + NickName string `json:"nick_name"` + Avatar struct { + OriginURL string `json:"origin_url"` + TinyURL string `json:"tiny_url"` + URI string `json:"uri"` + } `json:"avatar"` +} + +type FilePath []struct { + ID string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` + Size int `json:"size"` + Source int `json:"source"` + NameReviewStatus int `json:"name_review_status"` + ContentReviewStatus int `json:"content_review_status"` + RiskReviewStatus int `json:"risk_review_status"` + ConversationID string `json:"conversation_id"` + ParentID string `json:"parent_id"` + CreateTime int `json:"create_time"` + UpdateTime int `json:"update_time"` +} + +type GetFileUrlResp struct { + BaseResp + Data struct { + FileUrls []struct { + URI string `json:"uri"` + MainURL string `json:"main_url"` + BackURL string `json:"back_url"` + } `json:"file_urls"` + } `json:"data"` +} + +type GetVideoFileUrlResp struct { + BaseResp + Data struct { + MediaType string `json:"media_type"` + MediaInfo []struct { + Meta struct { + Height string `json:"height"` + Width string `json:"width"` + Format string `json:"format"` + Duration float64 `json:"duration"` + CodecType string `json:"codec_type"` + Definition string `json:"definition"` + } `json:"meta"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"media_info"` + OriginalMediaInfo struct { + Meta struct { + Height string `json:"height"` + Width string `json:"width"` + Format string `json:"format"` + Duration float64 `json:"duration"` + CodecType string `json:"codec_type"` + Definition string `json:"definition"` + } `json:"meta"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"original_media_info"` + PosterURL string `json:"poster_url"` + PlayableStatus int `json:"playable_status"` + } `json:"data"` +} + +type CommonResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` // 错误情况下的消息 + Data json.RawMessage `json:"data,omitempty"` // 原始数据,稍后解析 + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + Locale string `json:"locale"` + } `json:"error,omitempty"` +} + +// IsSuccess 判断响应是否成功 +func (r *CommonResp) IsSuccess() bool { + return r.Code == 0 +} + +// GetError 获取错误信息 +func (r *CommonResp) GetError() error { + if r.IsSuccess() { + return nil + } + // 优先使用message字段 + errMsg := r.Message + if errMsg == "" { + errMsg = r.Msg + } + // 如果error对象存在且有详细消息,则使用error中的信息 + if r.Error != nil && r.Error.Message != "" { + errMsg = r.Error.Message + } + + return fmt.Errorf("[doubao] API error (code: %d): %s", r.Code, errMsg) +} + +// UnmarshalData 将data字段解析为指定类型 +func (r *CommonResp) UnmarshalData(v interface{}) error { + if !r.IsSuccess() { + return r.GetError() + } + + if len(r.Data) == 0 { + return nil + } + + return json.Unmarshal(r.Data, v) +} diff --git a/drivers/doubao_share/util.go b/drivers/doubao_share/util.go new file mode 100644 index 00000000000..e0fc526e9fb --- /dev/null +++ b/drivers/doubao_share/util.go @@ -0,0 +1,744 @@ +package doubao_share + +import ( + "context" + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "net/http" + "net/url" + "path" + "regexp" + "strings" + "time" +) + +const ( + DirectoryType = 1 + FileType = 2 + LinkType = 3 + ImageType = 4 + PagesType = 5 + VideoType = 6 + AudioType = 7 + MeetingMinutesType = 8 +) + +var FileNodeType = map[int]string{ + 1: "directory", + 2: "file", + 3: "link", + 4: "image", + 5: "pages", + 6: "video", + 7: "audio", + 8: "meeting_minutes", +} + +const ( + BaseURL = "https://www.doubao.com" + FileDataType = "file" + ImgDataType = "image" + VideoDataType = "video" + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" +) + +func (d *DoubaoShare) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + reqUrl := BaseURL + path + req := base.RestyClient.R() + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "User-Agent": UserAgent, + }) + + req.SetQueryParams(map[string]string{ + "version_code": "20800", + "device_platform": "web", + }) + + if callback != nil { + callback(req) + } + + var commonResp CommonResp + + res, err := req.Execute(method, reqUrl) + log.Debugln(res.String()) + if err != nil { + return nil, err + } + + body := res.Body() + // 先解析为通用响应 + if err = json.Unmarshal(body, &commonResp); err != nil { + return nil, err + } + // 检查响应是否成功 + if !commonResp.IsSuccess() { + return body, commonResp.GetError() + } + + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func (d *DoubaoShare) getFiles(dirId, nodeId, cursor string) (resp []File, err error) { + var r NodeInfoResp + + var body = base.Json{ + "share_id": dirId, + "node_id": nodeId, + } + // 如果有游标，则设置游标和大小 + if cursor != "" { + body["cursor"] = cursor + body["size"] = 50 + } else { + body["need_full_path"] = false + } + + _, err = d.request("/samantha/aispace/share/node_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &r) + if err != nil { + return nil, err + } + + if r.NodeInfoData.Children != nil { + resp = r.NodeInfoData.Children + } + + if r.NodeInfoData.NextCursor != "-1" { + // 递归获取下一页 + nextFiles, err := d.getFiles(dirId, nodeId, r.NodeInfoData.NextCursor) + if err != nil { + return nil, err + } + + resp = append(r.NodeInfoData.Children, nextFiles...) + } + + return resp, err +} + +func (d *DoubaoShare) getShareOverview(shareId, cursor string) (resp []File, err error) { + return d.getShareOverviewWithHistory(shareId, cursor, make(map[string]bool)) +} + +func (d *DoubaoShare) getShareOverviewWithHistory(shareId, cursor string, cursorHistory map[string]bool) (resp []File, err error) { + var r NodeInfoResp + + var body = base.Json{ + "share_id": shareId, + } + // 如果有游标，则设置游标和大小 + if cursor != "" { + body["cursor"] = cursor + body["size"] = 50 + } else { + body["need_full_path"] = false + } + + _, err = d.request("/samantha/aispace/share/overview", http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &r) + if err != nil { + return nil, err + } + + if r.NodeInfoData.NodeList != nil { + resp = r.NodeInfoData.NodeList + } + + if r.NodeInfoData.NextCursor != "-1" { + // 检查游标是否重复出现，防止无限循环 + if cursorHistory[r.NodeInfoData.NextCursor] { + return resp, nil + } + + // 记录当前游标 + cursorHistory[r.NodeInfoData.NextCursor] = true + + // 递归获取下一页 + nextFiles, err := d.getShareOverviewWithHistory(shareId, r.NodeInfoData.NextCursor, cursorHistory) + if err != nil { + return nil, err + } + + resp = append(resp, nextFiles...) + } + + return resp, nil +} + +func (d *DoubaoShare) initShareList() error { + if d.Addition.ShareIds == "" { + return fmt.Errorf("share_ids is empty") + } + + // 解析分享配置 + shareConfigs, rootShares, err := d._parseShareConfigs() + if err != nil { + return err + } + + // 检查路径冲突 + if err := d._detectPathConflicts(shareConfigs); err != nil { + return err + } + + // 构建树形结构 + rootMap := d._buildTreeStructure(shareConfigs, rootShares) + + // 提取顶级节点 + topLevelNodes := d._extractTopLevelNodes(rootMap, rootShares) + if len(topLevelNodes) == 0 { + return fmt.Errorf("no valid share_ids found") + } + + // 存储结果 + d.RootFiles = topLevelNodes + + return nil +} + +// 从配置中解析分享ID和路径 +func (d *DoubaoShare) _parseShareConfigs() (map[string]string, []string, error) { + shareConfigs := make(map[string]string) // 路径 -> 分享ID + rootShares := make([]string, 0) // 根目录显示的分享ID + + lines := strings.Split(strings.TrimSpace(d.Addition.ShareIds), "\n") + if len(lines) == 0 { + return nil, nil, fmt.Errorf("no share_ids found") + } + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // 解析分享ID和路径 + parts := strings.Split(line, "|") + var shareId, sharePath string + + if len(parts) == 1 { + // 无路径分享，直接在根目录显示 + shareId = _extractShareId(parts[0]) + if shareId != "" { + rootShares = append(rootShares, shareId) + } + continue + } else if len(parts) >= 2 { + shareId = _extractShareId(parts[0]) + sharePath = strings.Trim(parts[1], "/") + } + + if shareId == "" { + log.Warnf("[doubao_share] Invalid Share_id Format: %s", line) + continue + } + + // 空路径也加入根目录显示 + if sharePath == "" { + rootShares = append(rootShares, shareId) + continue + } + + // 添加到路径映射 + shareConfigs[sharePath] = shareId + } + + return shareConfigs, rootShares, nil +} + +// 检测路径冲突 +func (d *DoubaoShare) _detectPathConflicts(shareConfigs map[string]string) error { + // 检查直接路径冲突 + pathToShareIds := make(map[string][]string) + for sharePath, id := range shareConfigs { + pathToShareIds[sharePath] = append(pathToShareIds[sharePath], id) + } + + for sharePath, ids := range pathToShareIds { + if len(ids) > 1 { + return fmt.Errorf("路径冲突: 路径 '%s' 被多个不同的分享ID使用: %s", + sharePath, strings.Join(ids, ", ")) + } + } + + // 检查层次冲突 + for path1, id1 := range shareConfigs { + for path2, id2 := range shareConfigs { + if path1 == path2 || id1 == id2 { + continue + } + + // 检查前缀冲突 + if strings.HasPrefix(path2, path1+"/") || strings.HasPrefix(path1, path2+"/") { + return fmt.Errorf("路径冲突: 路径 '%s' (ID: %s) 与路径 '%s' (ID: %s) 存在层次冲突", + path1, id1, path2, id2) + } + } + } + + return nil +} + +// 构建树形结构 +func (d *DoubaoShare) _buildTreeStructure(shareConfigs map[string]string, rootShares []string) map[string]*RootFileList { + rootMap := make(map[string]*RootFileList) + + // 添加所有分享节点 + for sharePath, shareId := range shareConfigs { + children := make([]RootFileList, 0) + rootMap[sharePath] = &RootFileList{ + ShareID: shareId, + VirtualPath: sharePath, + NodeInfo: NodeInfoData{}, + Child: &children, + } + } + + // 构建父子关系 + for sharePath, node := range rootMap { + if sharePath == "" { + continue + } + + pathParts := strings.Split(sharePath, "/") + if len(pathParts) > 1 { + parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") + + // 确保所有父级路径都已创建 + _ensurePathExists(rootMap, parentPath) + + // 添加当前节点到父节点 + if parent, exists := rootMap[parentPath]; exists { + *parent.Child = append(*parent.Child, *node) + } + } + } + + return rootMap +} + +// 提取顶级节点 +func (d *DoubaoShare) _extractTopLevelNodes(rootMap map[string]*RootFileList, rootShares []string) []RootFileList { + var topLevelNodes []RootFileList + + // 添加根目录分享 + for _, shareId := range rootShares { + children := make([]RootFileList, 0) + topLevelNodes = append(topLevelNodes, RootFileList{ + ShareID: shareId, + VirtualPath: "", + NodeInfo: NodeInfoData{}, + Child: &children, + }) + } + + // 添加顶级目录 + for rootPath, node := range rootMap { + if rootPath == "" { + continue + } + + isTopLevel := true + pathParts := strings.Split(rootPath, "/") + + if len(pathParts) > 1 { + parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") + if _, exists := rootMap[parentPath]; exists { + isTopLevel = false + } + } + + if isTopLevel { + topLevelNodes = append(topLevelNodes, *node) + } + } + + return topLevelNodes +} + +// 确保路径存在，创建所有必要的中间节点 +func _ensurePathExists(rootMap map[string]*RootFileList, path string) { + if path == "" { + return + } + + // 如果路径已存在，不需要再处理 + if _, exists := rootMap[path]; exists { + return + } + + // 创建当前路径节点 + children := make([]RootFileList, 0) + rootMap[path] = &RootFileList{ + ShareID: "", + VirtualPath: path, + NodeInfo: NodeInfoData{}, + Child: &children, + } + + // 处理父路径 + pathParts := strings.Split(path, "/") + if len(pathParts) > 1 { + parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") + + // 确保父路径存在 + _ensurePathExists(rootMap, parentPath) + + // 将当前节点添加为父节点的子节点 + if parent, exists := rootMap[parentPath]; exists { + *parent.Child = append(*parent.Child, *rootMap[path]) + } + } +} + +// _extractShareId 从URL或直接ID中提取分享ID +func _extractShareId(input string) string { + input = strings.TrimSpace(input) + if strings.HasPrefix(input, "http") { + regex := regexp.MustCompile(`/drive/s/([a-zA-Z0-9]+)`) + if matches := regex.FindStringSubmatch(input); len(matches) > 1 { + return matches[1] + } + return "" + } + return input // 直接返回ID +} + +// _findRootFileByShareID 查找指定ShareID的配置 +func _findRootFileByShareID(rootFiles []RootFileList, shareID string) *RootFileList { + for i, rf := range rootFiles { + if rf.ShareID == shareID { + return &rootFiles[i] + } + if rf.Child != nil && len(*rf.Child) > 0 { + if found := _findRootFileByShareID(*rf.Child, shareID); found != nil { + return found + } + } + } + return nil +} + +// _findNodeByPath 查找指定路径的节点 +func _findNodeByPath(rootFiles []RootFileList, path string) *RootFileList { + for i, rf := range rootFiles { + if rf.VirtualPath == path { + return &rootFiles[i] + } + if rf.Child != nil && len(*rf.Child) > 0 { + if found := _findNodeByPath(*rf.Child, path); found != nil { + return found + } + } + } + return nil +} + +// _findShareByPath 根据路径查找分享和相对路径 +func _findShareByPath(rootFiles []RootFileList, path string) (*RootFileList, string) { + // 完全匹配或子路径匹配 + for i, rf := range rootFiles { + if rf.VirtualPath == path { + return &rootFiles[i], "" + } + + if rf.VirtualPath != "" && strings.HasPrefix(path, rf.VirtualPath+"/") { + relPath := strings.TrimPrefix(path, rf.VirtualPath+"/") + + // 先检查子节点 + if rf.Child != nil && len(*rf.Child) > 0 { + if child, childPath := _findShareByPath(*rf.Child, path); child != nil { + return child, childPath + } + } + + return &rootFiles[i], relPath + } + + // 递归检查子节点 + if rf.Child != nil && len(*rf.Child) > 0 { + if child, childPath := _findShareByPath(*rf.Child, path); child != nil { + return child, childPath + } + } + } + + // 检查根目录分享 + for i, rf := range rootFiles { + if rf.VirtualPath == "" && rf.ShareID != "" { + parts := strings.SplitN(path, "/", 2) + if len(parts) > 0 && parts[0] == rf.ShareID { + if len(parts) > 1 { + return &rootFiles[i], parts[1] + } + return &rootFiles[i], "" + } + } + } + + return nil, "" +} + +// _findShareAndPath 根据给定路径查找对应的ShareID和相对路径 +func (d *DoubaoShare) _findShareAndPath(dir model.Obj) (string, string, error) { + dirPath := dir.GetPath() + + // 如果是根目录，返回空值表示需要列出所有分享 + if dirPath == "/" || dirPath == "" { + return "", "", nil + } + + // 检查是否是 FileObject 类型，并获取 ShareID + if fo, ok := dir.(*FileObject); ok && fo.ShareID != "" { + // 直接使用对象中存储的 ShareID + // 计算相对路径（移除前导斜杠） + relativePath := strings.TrimPrefix(dirPath, "/") + + // 递归查找对应的 RootFile + found := _findRootFileByShareID(d.RootFiles, fo.ShareID) + if found != nil { + if found.VirtualPath != "" { + // 如果此分享配置了路径前缀，需要考虑相对路径的计算 + if strings.HasPrefix(relativePath, found.VirtualPath) { + return fo.ShareID, strings.TrimPrefix(relativePath, found.VirtualPath+"/"), nil + } + } + return fo.ShareID, relativePath, nil + } + + // 如果找不到对应的 RootFile 配置，仍然使用对象中的 ShareID + return fo.ShareID, relativePath, nil + } + + // 移除开头的斜杠 + cleanPath := strings.TrimPrefix(dirPath, "/") + + // 先检查是否有直接匹配的根目录分享 + for _, rootFile := range d.RootFiles { + if rootFile.VirtualPath == "" && rootFile.ShareID != "" { + // 检查是否匹配当前路径的第一部分 + parts := strings.SplitN(cleanPath, "/", 2) + if len(parts) > 0 && parts[0] == rootFile.ShareID { + if len(parts) > 1 { + return rootFile.ShareID, parts[1], nil + } + return rootFile.ShareID, "", nil + } + } + } + + // 查找匹配此路径的分享或虚拟目录 + share, relPath := _findShareByPath(d.RootFiles, cleanPath) + if share != nil { + return share.ShareID, relPath, nil + } + + log.Warnf("[doubao_share] No matching share path found: %s", dirPath) + return "", "", fmt.Errorf("no matching share path found: %s", dirPath) +} + +// convertToFileObject 将File转换为FileObject +func (d *DoubaoShare) convertToFileObject(file File, shareId string, relativePath string) *FileObject { + // 构建文件对象 + obj := &FileObject{ + Object: model.Object{ + ID: file.ID, + Name: file.Name, + Size: file.Size, + Modified: time.Unix(file.UpdateTime, 0), + Ctime: time.Unix(file.CreateTime, 0), + IsFolder: file.NodeType == DirectoryType, + Path: path.Join(relativePath, file.Name), + }, + ShareID: shareId, + Key: file.Key, + NodeID: file.ID, + NodeType: file.NodeType, + } + + return obj +} + +// getFilesInPath 获取指定分享和路径下的文件 +func (d *DoubaoShare) getFilesInPath(ctx context.Context, shareId, nodeId, relativePath string) ([]model.Obj, error) { + var ( + files []File + err error + ) + + // 调用overview接口获取分享链接信息 nodeId + if nodeId == "" { + files, err = d.getShareOverview(shareId, "") + if err != nil { + return nil, fmt.Errorf("failed to get share link information: %w", err) + } + + result := make([]model.Obj, 0, len(files)) + for _, file := range files { + result = append(result, d.convertToFileObject(file, shareId, "/")) + } + + return result, nil + + } else { + files, err = d.getFiles(shareId, nodeId, "") + if err != nil { + return nil, fmt.Errorf("failed to get share file: %w", err) + } + + result := make([]model.Obj, 0, len(files)) + for _, file := range files { + result = append(result, d.convertToFileObject(file, shareId, path.Join("/", relativePath))) + } + + return result, nil + } +} + +// listRootDirectory 处理根目录的内容展示 +func (d *DoubaoShare) listRootDirectory(ctx context.Context) ([]model.Obj, error) { + objects := make([]model.Obj, 0) + + // 分组处理：直接显示的分享内容 vs 虚拟目录 + var directShareIDs []string + addedDirs := make(map[string]bool) + + // 处理所有根节点 + for _, rootFile := range d.RootFiles { + if rootFile.VirtualPath == "" && rootFile.ShareID != "" { + // 无路径分享，记录ShareID以便后续获取内容 + directShareIDs = append(directShareIDs, rootFile.ShareID) + } else { + // 有路径的分享，显示第一级目录 + parts := strings.SplitN(rootFile.VirtualPath, "/", 2) + firstLevel := parts[0] + + // 避免重复添加同名目录 + if _, exists := addedDirs[firstLevel]; exists { + continue + } + + // 创建虚拟目录对象 + obj := &FileObject{ + Object: model.Object{ + ID: "", + Name: firstLevel, + Modified: time.Now(), + Ctime: time.Now(), + IsFolder: true, + Path: path.Join("/", firstLevel), + }, + ShareID: rootFile.ShareID, + Key: "", + NodeID: "", + NodeType: DirectoryType, + } + objects = append(objects, obj) + addedDirs[firstLevel] = true + } + } + + // 处理直接显示的分享内容 + for _, shareID := range directShareIDs { + shareFiles, err := d.getFilesInPath(ctx, shareID, "", "") + if err != nil { + log.Warnf("[doubao_share] Failed to get list of files in share %s: %s", shareID, err) + continue + } + objects = append(objects, shareFiles...) + } + + return objects, nil +} + +// listVirtualDirectoryContent 列出虚拟目录的内容 +func (d *DoubaoShare) listVirtualDirectoryContent(dir model.Obj) ([]model.Obj, error) { + dirPath := strings.TrimPrefix(dir.GetPath(), "/") + objects := make([]model.Obj, 0) + + // 递归查找此路径的节点 + node := _findNodeByPath(d.RootFiles, dirPath) + + if node != nil && node.Child != nil { + // 显示此节点的所有子节点 + for _, child := range *node.Child { + // 计算显示名称（取路径的最后一部分） + displayName := child.VirtualPath + if child.VirtualPath != "" { + parts := strings.Split(child.VirtualPath, "/") + displayName = parts[len(parts)-1] + } else if child.ShareID != "" { + displayName = child.ShareID + } + + obj := &FileObject{ + Object: model.Object{ + ID: "", + Name: displayName, + Modified: time.Now(), + Ctime: time.Now(), + IsFolder: true, + Path: path.Join("/", child.VirtualPath), + }, + ShareID: child.ShareID, + Key: "", + NodeID: "", + NodeType: DirectoryType, + } + objects = append(objects, obj) + } + } + + return objects, nil +} + +// generateContentDisposition 生成符合RFC 5987标准的Content-Disposition头部 +func generateContentDisposition(filename string) string { + // 按照RFC 2047进行编码，用于filename部分 + encodedName := urlEncode(filename) + + // 按照RFC 5987进行编码，用于filename*部分 + encodedNameRFC5987 := encodeRFC5987(filename) + + return fmt.Sprintf("attachment; filename=\"%s\"; filename*=utf-8''%s", + encodedName, encodedNameRFC5987) +} + +// encodeRFC5987 按照RFC 5987规范编码字符串，适用于HTTP头部参数中的非ASCII字符 +func encodeRFC5987(s string) string { + var buf strings.Builder + for _, r := range []byte(s) { + // 根据RFC 5987，只有字母、数字和部分特殊符号可以不编码 + if (r >= 'a' && r <= 'z') || + (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || + r == '-' || r == '.' || r == '_' || r == '~' { + buf.WriteByte(r) + } else { + // 其他字符都需要百分号编码 + fmt.Fprintf(&buf, "%%%02X", r) + } + } + return buf.String() +} + +func urlEncode(s string) string { + s = url.QueryEscape(s) + s = strings.ReplaceAll(s, "+", "%20") + return s +} From 28e5b5759ecade05072a2e4ee1df5c62bcfcf1d8 Mon Sep 17 00:00:00 2001 From: New Future Date: Sat, 19 Apr 2025 14:23:48 +0800 Subject: [PATCH 505/659] feat(azure_blob): implement GetRootId interface in Addition struct (#8389) fix failed get dir --- drivers/azure_blob/meta.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/azure_blob/meta.go b/drivers/azure_blob/meta.go index 8e42bdd6b6d..b1e021b86d0 100644 --- a/drivers/azure_blob/meta.go +++ b/drivers/azure_blob/meta.go @@ -12,6 +12,11 @@ type Addition struct { SignURLExpire int `json:"sign_url_expire" type:"number" default:"4" help:"The expiration time for SAS URLs, in hours."` } +// implement GetRootId interface +func (r Addition) GetRootId() string { + return r.ContainerName +} + var config = driver.Config{ Name: "Azure Blob Storage", LocalSort: true, From 52d4e8ec47ba196f7c502e6541141b82a6d29097 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sat, 19 Apr 2025 14:24:43 +0800 Subject: [PATCH 506/659] fix(lanzou): remove JavaScript comments from response data (#8386) * feat(lanzou): add RemoveJSComment function to clean JavaScript comments from HTML * feat(lanzou): remove comments from share page data in getFilesByShareUrl function * fix(lanzou): optimize RemoveJSComment function to improve comment removal logic --- drivers/lanzou/help.go | 36 ++++++++++++++++++++++++++++++++++++ drivers/lanzou/util.go | 4 ++++ 2 files changed, 40 insertions(+) diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index 81d7c567d5c..c3f5c6bb5bc 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -78,6 +78,42 @@ func RemoveNotes(html string) string { }) } +// 清理JS注释 +func RemoveJSComment(data string) string { + var result strings.Builder + inComment := false + inSingleLineComment := false + + for i := 0; i < len(data); i++ { + v := data[i] + + if inSingleLineComment && (v == '\n' || v == '\r') { + inSingleLineComment = false + result.WriteByte(v) + continue + } + if inComment && v == '*' && i+1 < len(data) && data[i+1] == '/' { + inComment = false + continue + } + if v == '/' && i+1 < len(data) { + nextChar := data[i+1] + if nextChar == '*' { + inComment = true + i++ + continue + } else if nextChar == '/' { + inSingleLineComment = true + i++ + continue + } + } + result.WriteByte(v) + } + + return result.String() +} + var findAcwScV2Reg = regexp.MustCompile(`arg1='([0-9A-Z]+)'`) // 在页面被过多访问或其他情况下，有时候会先返回一个加密的页面，其执行计算出一个acw_sc__v2后放入页面后再重新访问页面才能获得正常页面 diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index 4b9959ad53d..e66252bcc79 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -348,6 +348,10 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( file FileOrFolderByShareUrl ) + // 删除注释 + sharePageData = RemoveNotes(sharePageData) + sharePageData = RemoveJSComment(sharePageData) + // 需要密码 if strings.Contains(sharePageData, "pwdload") || strings.Contains(sharePageData, "passwddiv") { sharePageData, err := getJSFunctionByName(sharePageData, "down_p") From b449312da83c67b154668d5835e44cbed8260c74 Mon Sep 17 00:00:00 2001 From: wxnq <49645495+yanjing19989@users.noreply.github.com> Date: Sat, 19 Apr 2025 14:26:19 +0800 Subject: [PATCH 507/659] fix(docker_release): avoid duplicate occupation in docker image (#8393 close #8388) * fix(ci): modify the method of adding permissions * fix(build): modify the method of adding permissions(to keep up with ci) --- Dockerfile | 7 +++---- Dockerfile.ci | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0e2ee96fb37..f5e91bee230 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,10 +32,9 @@ RUN apk update && \ /opt/aria2/.aria2/tracker.sh ; \ rm -rf /var/cache/apk/* -COPY --from=builder /app/bin/alist ./ -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /opt/alist/alist && \ - chmod +x /entrypoint.sh && /entrypoint.sh version +COPY --chmod=755 --from=builder /app/bin/alist ./ +COPY --chmod=755 entrypoint.sh /entrypoint.sh +RUN /entrypoint.sh version ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/alist/data/ diff --git a/Dockerfile.ci b/Dockerfile.ci index 25d502a90aa..a17aae9fcfd 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -24,10 +24,9 @@ RUN apk update && \ /opt/aria2/.aria2/tracker.sh ; \ rm -rf /var/cache/apk/* -COPY /build/${TARGETPLATFORM}/alist ./ -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /opt/alist/alist && \ - chmod +x /entrypoint.sh && /entrypoint.sh version +COPY --chmod=755 /build/${TARGETPLATFORM}/alist ./ +COPY --chmod=755 entrypoint.sh /entrypoint.sh +RUN /entrypoint.sh version ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/alist/data/ From 8f89c55acaeb04a29bab59d225db13983c1082e0 Mon Sep 17 00:00:00 2001 From: Lin Tianchuan <47070449+1024th@users.noreply.github.com> Date: Sat, 19 Apr 2025 14:27:13 +0800 Subject: [PATCH 508/659] perf(local): avoid duplicate parsing of VideoThumbPos (#7812) * feat(local): support percent for video thumbnail The percentage determines the point in the video (as a percentage of the total duration) at which the thumbnail will be generated. * feat(local): support both time and percent for video thumbnail * refactor(local): avoid duplicate parsing of VideoThumbPos --- drivers/local/driver.go | 8 ++++++++ drivers/local/util.go | 16 ++++------------ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 8a804ef32a6..faa2b3bd157 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -35,6 +35,10 @@ type Local struct { // zero means no limit thumbConcurrency int thumbTokenBucket TokenBucket + + // video thumb position + videoThumbPos float64 + videoThumbPosIsPercentage bool } func (d *Local) Config() driver.Config { @@ -92,6 +96,8 @@ func (d *Local) Init(ctx context.Context) error { if val < 0 || val > 100 { return fmt.Errorf("invalid video_thumb_pos value: %s, the precentage must be a number between 0 and 100", d.VideoThumbPos) } + d.videoThumbPosIsPercentage = true + d.videoThumbPos = val / 100 } else { val, err := strconv.ParseFloat(d.VideoThumbPos, 64) if err != nil { @@ -100,6 +106,8 @@ func (d *Local) Init(ctx context.Context) error { if val < 0 { return fmt.Errorf("invalid video_thumb_pos value: %s, the time must be a positive number", d.VideoThumbPos) } + d.videoThumbPosIsPercentage = false + d.videoThumbPos = val } return nil } diff --git a/drivers/local/util.go b/drivers/local/util.go index d2fbd097b5a..802f60cf627 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -61,22 +61,14 @@ func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) } var ss string - if strings.HasSuffix(d.VideoThumbPos, "%") { - percentage, err := strconv.ParseFloat(strings.TrimSuffix(d.VideoThumbPos, "%"), 64) - if err != nil { - return nil, err - } - ss = fmt.Sprintf("%f", totalDuration*percentage/100) + if d.videoThumbPosIsPercentage { + ss = fmt.Sprintf("%f", totalDuration*d.videoThumbPos) } else { - val, err := strconv.ParseFloat(d.VideoThumbPos, 64) - if err != nil { - return nil, err - } // If the value is greater than the total duration, use the total duration - if val > totalDuration { + if d.videoThumbPos > totalDuration { ss = fmt.Sprintf("%f", totalDuration) } else { - ss = d.VideoThumbPos + ss = fmt.Sprintf("%f", d.videoThumbPos) } } From 41bdab49aa8acca9e88862c3db55cd7a8a84ba6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sam-=20Pan=EF=BC=88=E6=BD=98=E7=BB=8D=E6=A3=AE=EF=BC=89?= Date: Sat, 19 Apr 2025 14:29:12 +0800 Subject: [PATCH 509/659] fix(139): incorrect host (#8368) * fix: correct new personal cloud path for 139Driver * Update drivers/139/driver.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix bug --------- Co-authored-by: panshaosen <19802021493@139.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: j2rong4cn <253551464@qq.com> --- drivers/139/driver.go | 72 ++++++++++++++++++++++--------------------- drivers/139/types.go | 31 +++++++++++++++---- drivers/139/util.go | 72 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 129 insertions(+), 46 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 0af5a4f781a..a57609bc550 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -24,9 +24,10 @@ import ( type Yun139 struct { model.Storage Addition - cron *cron.Cron - Account string - ref *Yun139 + cron *cron.Cron + Account string + ref *Yun139 + PersonalCloudHost string } func (d *Yun139) Config() driver.Config { @@ -39,13 +40,36 @@ func (d *Yun139) GetAddition() driver.Additional { func (d *Yun139) Init(ctx context.Context) error { if d.ref == nil { - if d.Authorization == "" { + if len(d.Authorization) == 0 { return fmt.Errorf("authorization is empty") } err := d.refreshToken() if err != nil { return err } + + // Query Route Policy + var resp QueryRoutePolicyResp + _, err = d.requestRoute(base.Json{ + "userInfo": base.Json{ + "userType": 1, + "accountType": 1, + "accountName": d.Account}, + "modAddrType": 1, + }, &resp) + if err != nil { + return err + } + for _, policyItem := range resp.Data.RoutePolicyList { + if policyItem.ModName == "personal" { + d.PersonalCloudHost = policyItem.HttpsUrl + break + } + } + if len(d.PersonalCloudHost) == 0 { + return fmt.Errorf("PersonalCloudHost is empty") + } + d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { err := d.refreshToken() @@ -71,28 +95,6 @@ func (d *Yun139) Init(ctx context.Context) error { default: return errs.NotImplement } - // if d.ref != nil { - // return nil - // } - // decode, err := base64.StdEncoding.DecodeString(d.Authorization) - // if err != nil { - // return err - // } - // decodeStr := string(decode) - // splits := strings.Split(decodeStr, ":") - // if len(splits) < 2 { - // return fmt.Errorf("authorization is invalid, splits < 2") - // } - // d.Account = splits[1] - // _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ - // "qryUserExternInfoReq": base.Json{ - // "commonAccountInfo": base.Json{ - // "account": d.getAccount(), - // "accountType": 1, - // }, - // }, - // }, nil) - // return err return nil } @@ -160,7 +162,7 @@ func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin "type": "folder", "fileRenameMode": "force_rename", } - pathname := "/hcy/file/create" + pathname := "/file/create" _, err = d.personalPost(pathname, data, nil) case MetaPersonal: data := base.Json{ @@ -213,7 +215,7 @@ func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, "fileIds": []string{srcObj.GetID()}, "toParentFileId": dstDir.GetID(), } - pathname := "/hcy/file/batchMove" + pathname := "/file/batchMove" _, err := d.personalPost(pathname, data, nil) if err != nil { return nil, err @@ -290,7 +292,7 @@ func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) e "name": newName, "description": "", } - pathname := "/hcy/file/update" + pathname := "/file/update" _, err = d.personalPost(pathname, data, nil) case MetaPersonal: var data base.Json @@ -390,7 +392,7 @@ func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { "fileIds": []string{srcObj.GetID()}, "toParentFileId": dstDir.GetID(), } - pathname := "/hcy/file/batchCopy" + pathname := "/file/batchCopy" _, err := d.personalPost(pathname, data, nil) return err case MetaPersonal: @@ -430,7 +432,7 @@ func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { data := base.Json{ "fileIds": []string{obj.GetID()}, } - pathname := "/hcy/recyclebin/batchTrash" + pathname := "/recyclebin/batchTrash" _, err := d.personalPost(pathname, data, nil) return err case MetaGroup: @@ -574,7 +576,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "type": "file", "fileRenameMode": "auto_rename", } - pathname := "/hcy/file/create" + pathname := "/file/create" var resp PersonalUploadResp _, err = d.personalPost(pathname, data, &resp) if err != nil { @@ -611,7 +613,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "accountType": 1, }, } - pathname := "/hcy/file/getUploadUrl" + pathname := "/file/getUploadUrl" var moreresp PersonalUploadUrlResp _, err = d.personalPost(pathname, moredata, &moreresp) if err != nil { @@ -662,7 +664,7 @@ func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr "fileId": resp.Data.FileId, "uploadId": resp.Data.UploadId, } - _, err = d.personalPost("/hcy/file/complete", data, nil) + _, err = d.personalPost("/file/complete", data, nil) if err != nil { return err } @@ -854,7 +856,7 @@ func (d *Yun139) Other(ctx context.Context, args model.OtherArgs) (interface{}, } switch args.Method { case "video_preview": - uri = "/hcy/videoPreview/getPreviewInfo" + uri = "/videoPreview/getPreviewInfo" default: return nil, errs.NotSupport } diff --git a/drivers/139/types.go b/drivers/139/types.go index 50ae1f81242..d5f025a1672 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -285,11 +285,30 @@ type PersonalUploadUrlResp struct { } } +type QueryRoutePolicyResp struct { + Success bool `json:"success"` + Code string `json:"code"` + Message string `json:"message"` + Data struct { + RoutePolicyList []struct { + SiteID string `json:"siteID"` + SiteCode string `json:"siteCode"` + ModName string `json:"modName"` + HttpUrl string `json:"httpUrl"` + HttpsUrl string `json:"httpsUrl"` + EnvID string `json:"envID"` + ExtInfo string `json:"extInfo"` + HashName string `json:"hashName"` + ModAddrType int `json:"modAddrType"` + } `json:"routePolicyList"` + } `json:"data"` +} + type RefreshTokenResp struct { - XMLName xml.Name `xml:"root"` - Return string `xml:"return"` - Token string `xml:"token"` - Expiretime int32 `xml:"expiretime"` - AccessToken string `xml:"accessToken"` - Desc string `xml:"desc"` + XMLName xml.Name `xml:"root"` + Return string `xml:"return"` + Token string `xml:"token"` + Expiretime int32 `xml:"expiretime"` + AccessToken string `xml:"accessToken"` + Desc string `xml:"desc"` } diff --git a/drivers/139/util.go b/drivers/139/util.go index 53defef528e..4b43e7d3721 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -157,6 +157,64 @@ func (d *Yun139) request(pathname string, method string, callback base.ReqCallba } return res.Body(), nil } + +func (d *Yun139) requestRoute(data interface{}, resp interface{}) ([]byte, error) { + url := "https://user-njs.yun.139.com/user/route/qryRoutePolicy" + req := base.RestyClient.R() + randStr := random.String(16) + ts := time.Now().Format("2006-01-02 15:04:05") + callback := func(req *resty.Request) { + req.SetBody(data) + } + if callback != nil { + callback(req) + } + body, err := utils.Json.Marshal(req.Body) + if err != nil { + return nil, err + } + sign := calSign(string(body), ts, randStr) + svcType := "1" + if d.isFamily() { + svcType = "2" + } + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "CMS-DEVICE": "default", + "Authorization": "Basic " + d.getAuthorization(), + "mcloud-channel": "1000101", + "mcloud-client": "10701", + //"mcloud-route": "001", + "mcloud-sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), + //"mcloud-skey":"", + "mcloud-version": "7.14.0", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/w/", + "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", + "x-huawei-channelSrc": "10000034", + "x-inner-ntwk": "2", + "x-m4c-caller": "PC", + "x-m4c-src": "10002", + "x-SvcType": svcType, + "Inner-Hcy-Router-Https": "1", + }) + + var e BaseResp + req.SetResult(&e) + res, err := req.Execute(http.MethodPost, url) + log.Debugln(res.String()) + if !e.Success { + return nil, errors.New(e.Message) + } + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + return nil, err + } + } + return res.Body(), nil +} + func (d *Yun139) post(pathname string, data interface{}, resp interface{}) ([]byte, error) { return d.request(pathname, http.MethodPost, func(req *resty.Request) { req.SetBody(data) @@ -391,7 +449,7 @@ func unicode(str string) string { } func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - url := "https://personal-kd-njs.yun.139.com" + pathname + url := d.getPersonalCloudHost() + pathname req := base.RestyClient.R() randStr := random.String(16) ts := time.Now().Format("2006-01-02 15:04:05") @@ -417,8 +475,6 @@ func (d *Yun139) personalRequest(pathname string, method string, callback base.R "Mcloud-Route": "001", "Mcloud-Sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), "Mcloud-Version": "7.14.0", - "Origin": "https://yun.139.com", - "Referer": "https://yun.139.com/w/", "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", "x-huawei-channelSrc": "10000034", "x-inner-ntwk": "2", @@ -480,7 +536,7 @@ func (d *Yun139) personalGetFiles(fileId string) ([]model.Obj, error) { "parentFileId": fileId, } var resp PersonalListResp - _, err := d.personalPost("/hcy/file/list", data, &resp) + _, err := d.personalPost("/file/list", data, &resp) if err != nil { return nil, err } @@ -528,7 +584,7 @@ func (d *Yun139) personalGetLink(fileId string) (string, error) { data := base.Json{ "fileId": fileId, } - res, err := d.personalPost("/hcy/file/getDownloadUrl", + res, err := d.personalPost("/file/getDownloadUrl", data, nil) if err != nil { return "", err @@ -553,3 +609,9 @@ func (d *Yun139) getAccount() string { } return d.Account } +func (d *Yun139) getPersonalCloudHost() string { + if d.ref != nil { + return d.ref.getPersonalCloudHost() + } + return d.PersonalCloudHost +} From 17b42b9fa4ade8d237f407ae1e19c7cc8d7cb09d Mon Sep 17 00:00:00 2001 From: gdm257 <257@gdm.anonaddy.com> Date: Sun, 27 Apr 2025 20:56:04 +0900 Subject: [PATCH 510/659] fix(mega): use newest file for same filename (#8422 close #8344) Mega supports duplicate names but alist does not support. In `List()` method, driver will return multiple files with same name. That makes alist to use oldest version file for listing/downloading. So it is necessary to filter old same name files in a folder. After fixes, all CRUD work normally. Refs #8344 --- drivers/mega/driver.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index f76bfeefd70..dc7b220129c 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -56,13 +56,22 @@ func (d *Mega) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] if err != nil { return nil, err } - res := make([]model.Obj, 0) + fn := make(map[string]model.Obj) for i := range nodes { n := nodes[i] - if n.GetType() == mega.FILE || n.GetType() == mega.FOLDER { - res = append(res, &MegaNode{n}) + if n.GetType() != mega.FILE && n.GetType() != mega.FOLDER { + continue + } + if _, ok := fn[n.GetName()]; !ok { + fn[n.GetName()] = &MegaNode{n} + } else if sameNameObj := fn[n.GetName()]; (&MegaNode{n}).ModTime().After(sameNameObj.ModTime()) { + fn[n.GetName()] = &MegaNode{n} } } + res := make([]model.Obj, 0) + for _, v := range fn { + res = append(res, v) + } return res, nil } log.Errorf("can't convert: %+v", dir) From bf0705ec172f53c1fb08d7ebda783c41f71757d5 Mon Sep 17 00:00:00 2001 From: Mmx <36563672+Mmx233@users.noreply.github.com> Date: Sun, 27 Apr 2025 19:56:34 +0800 Subject: [PATCH 511/659] fix: shebang of entrypoint.sh (#8408) --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 28a18d7d54c..c24ed6eebf9 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh umask ${UMASK} From e532ab31efb58a9552381b19c483c65d1d7e560b Mon Sep 17 00:00:00 2001 From: Mmx <36563672+Mmx233@users.noreply.github.com> Date: Sun, 27 Apr 2025 19:58:09 +0800 Subject: [PATCH 512/659] fix: remove auth middleware for authn login (#8407) --- server/router.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/router.go b/server/router.go index 2dd6ee88601..09a0bb44faf 100644 --- a/server/router.go +++ b/server/router.go @@ -77,10 +77,10 @@ func Init(e *gin.Engine) { api.GET("/auth/sso_get_token", handles.SSOLoginCallback) // webauthn + api.GET("/authn/webauthn_begin_login", handles.BeginAuthnLogin) + api.POST("/authn/webauthn_finish_login", handles.FinishAuthnLogin) webauthn.GET("/webauthn_begin_registration", handles.BeginAuthnRegistration) webauthn.POST("/webauthn_finish_registration", handles.FinishAuthnRegistration) - webauthn.GET("/webauthn_begin_login", handles.BeginAuthnLogin) - webauthn.POST("/webauthn_finish_login", handles.FinishAuthnLogin) webauthn.POST("/delete_authn", handles.DeleteAuthnLogin) webauthn.GET("/getcredentials", handles.GetAuthnCredentials) From 6d9c554f6f14a2e1471845e37e16a4ea7b157964 Mon Sep 17 00:00:00 2001 From: bigQY <52437374+bigQY@users.noreply.github.com> Date: Sun, 27 Apr 2025 19:58:45 +0800 Subject: [PATCH 513/659] feat: add UseLargeThumbnail for 139 (#8424) --- drivers/139/meta.go | 1 + drivers/139/util.go | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/drivers/139/meta.go b/drivers/139/meta.go index 866aadb4192..c02b1347587 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -13,6 +13,7 @@ type Addition struct { CloudID string `json:"cloud_id"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` ReportRealSize bool `json:"report_real_size" type:"bool" default:"true" help:"Enable to report the real file size during upload"` + UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false" help:"Enable to use large thumbnail for images"` } var config = driver.Config{ diff --git a/drivers/139/util.go b/drivers/139/util.go index 4b43e7d3721..5adc39b4116 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -556,7 +556,15 @@ func (d *Yun139) personalGetFiles(fileId string) ([]model.Obj, error) { } else { var Thumbnails = item.Thumbnails var ThumbnailUrl string - if len(Thumbnails) > 0 { + if d.UseLargeThumbnail { + for _, thumb := range Thumbnails { + if strings.Contains(thumb.Style, "Large") { + ThumbnailUrl = thumb.Url + break + } + } + } + if ThumbnailUrl == "" && len(Thumbnails) > 0 { ThumbnailUrl = Thumbnails[len(Thumbnails)-1].Url } f = &model.ObjThumb{ From f541489d7d733c30f62f13ecf5cbf70783059e2a Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sun, 27 Apr 2025 19:59:30 +0800 Subject: [PATCH 514/659] fix(netease_music): change ListResp size fields from string to int64 (#8417) --- drivers/netease_music/types.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/netease_music/types.go b/drivers/netease_music/types.go index 12afeb7a67e..93ecdf702ff 100644 --- a/drivers/netease_music/types.go +++ b/drivers/netease_music/types.go @@ -2,13 +2,13 @@ package netease_music import ( "context" - "github.com/alist-org/alist/v3/internal/driver" "io" "net/http" "strconv" "strings" "time" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/http_range" @@ -28,8 +28,8 @@ type SongResp struct { } type ListResp struct { - Size string `json:"size"` - MaxSize string `json:"maxSize"` + Size int64 `json:"size"` + MaxSize int64 `json:"maxSize"` Data []struct { AddTime int64 `json:"addTime"` FileName string `json:"fileName"` From b2b91a92814487f52125d17691963bd19bfb6713 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sun, 27 Apr 2025 20:00:25 +0800 Subject: [PATCH 515/659] feat(doubao): add get_download_info API and download_api option (#8428) --- drivers/doubao/driver.go | 64 +++++++++++++++++++++++++--------------- drivers/doubao/meta.go | 1 + drivers/doubao/types.go | 14 ++++++++- 3 files changed, 55 insertions(+), 24 deletions(-) diff --git a/drivers/doubao/driver.go b/drivers/doubao/driver.go index a066feee1a7..0d421946099 100644 --- a/drivers/doubao/driver.go +++ b/drivers/doubao/driver.go @@ -3,6 +3,11 @@ package doubao import ( "context" "errors" + "net/http" + "strconv" + "strings" + "time" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -10,10 +15,6 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" "github.com/google/uuid" - "net/http" - "strconv" - "strings" - "time" ) type Doubao struct { @@ -97,33 +98,50 @@ func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) var downloadUrl string if u, ok := file.(*Object); ok { - switch u.NodeType { - case VideoType, AudioType: - var r GetVideoFileUrlResp - _, err := d.request("/samantha/media/get_play_info", http.MethodPost, func(req *resty.Request) { + switch d.DownloadApi { + case "get_download_info": + var r GetDownloadInfoResp + _, err := d.request("/samantha/aispace/get_download_info", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ - "key": u.Key, - "node_id": file.GetID(), + "requests": []base.Json{{"node_id": file.GetID()}}, }) }, &r) if err != nil { return nil, err } - downloadUrl = r.Data.OriginalMediaInfo.MainURL - default: - var r GetFileUrlResp - _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { - req.SetBody(base.Json{ - "uris": []string{u.Key}, - "type": FileNodeType[u.NodeType], - }) - }, &r) - if err != nil { - return nil, err + downloadUrl = r.Data.DownloadInfos[0].MainURL + case "get_file_url": + switch u.NodeType { + case VideoType, AudioType: + var r GetVideoFileUrlResp + _, err := d.request("/samantha/media/get_play_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "key": u.Key, + "node_id": file.GetID(), + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.OriginalMediaInfo.MainURL + default: + var r GetFileUrlResp + _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{u.Key}, + "type": FileNodeType[u.NodeType], + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.FileUrls[0].MainURL } - - downloadUrl = r.Data.FileUrls[0].MainURL + default: + return nil, errs.NotImplement } // 生成标准的Content-Disposition diff --git a/drivers/doubao/meta.go b/drivers/doubao/meta.go index c3d8eb34624..7735e5ff0b0 100644 --- a/drivers/doubao/meta.go +++ b/drivers/doubao/meta.go @@ -12,6 +12,7 @@ type Addition struct { // define other Cookie string `json:"cookie" type:"text"` UploadThread string `json:"upload_thread" default:"3"` + DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"` } var config = driver.Config{ diff --git a/drivers/doubao/types.go b/drivers/doubao/types.go index 4264eb7d83b..ae747f887cf 100644 --- a/drivers/doubao/types.go +++ b/drivers/doubao/types.go @@ -3,8 +3,9 @@ package doubao import ( "encoding/json" "fmt" - "github.com/alist-org/alist/v3/internal/model" "time" + + "github.com/alist-org/alist/v3/internal/model" ) type BaseResp struct { @@ -38,6 +39,17 @@ type File struct { UpdateTime int64 `json:"update_time"` } +type GetDownloadInfoResp struct { + BaseResp + Data struct { + DownloadInfos []struct { + NodeID string `json:"node_id"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"download_infos"` + } `json:"data"` +} + type GetFileUrlResp struct { BaseResp Data struct { From 11e7284824e2dfa5eb4517cd0ee5a61923b1746c Mon Sep 17 00:00:00 2001 From: yoclo <147054286+yclw@users.noreply.github.com> Date: Tue, 29 Apr 2025 23:14:16 +0800 Subject: [PATCH 516/659] fix: prevent guest user from updating profile (#8447) --- server/handles/auth.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/handles/auth.go b/server/handles/auth.go index e1f512c4dc1..7a2c0fb5376 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -113,6 +113,10 @@ func UpdateCurrent(c *gin.Context) { return } user := c.MustGet("user").(*model.User) + if user.IsGuest() { + common.ErrorStrResp(c, "Guest user can not update profile", 403) + return + } user.Username = req.Username if req.Password != "" { user.SetPassword(req.Password) From bc5117fa4f5f8d46a4ba200fb6f9f0b2c18010e7 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Fri, 2 May 2025 16:53:39 +0800 Subject: [PATCH 517/659] fix(115_open): add delay in MakeDir function to handle rate limiting --- drivers/115_open/driver.go | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 0eb943ac067..3826c78ff52 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -117,6 +117,7 @@ func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri if err != nil { return nil, err } + time.Sleep(800 * time.Millisecond) return &Obj{ Fid: resp.FileID, Pid: parentDir.GetID(), From 630cf30af5544359a7ccb98ff6c844c643049df0 Mon Sep 17 00:00:00 2001 From: Andy Hsu Date: Sun, 11 May 2025 13:39:32 +0800 Subject: [PATCH 518/659] feat(115_open): implement rate limiting for API requests --- drivers/115_open/driver.go | 39 ++++++++++++++++++++++++++++++++++++-- drivers/115_open/meta.go | 7 ++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go index 3826c78ff52..6121d3b2ed8 100644 --- a/drivers/115_open/driver.go +++ b/drivers/115_open/driver.go @@ -16,12 +16,14 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" sdk "github.com/xhofe/115-sdk-go" + "golang.org/x/time/rate" ) type Open115 struct { model.Storage Addition - client *sdk.Client + client *sdk.Client + limiter *rate.Limiter } func (d *Open115) Config() driver.Config { @@ -47,6 +49,16 @@ func (d *Open115) Init(ctx context.Context) error { if err != nil { return err } + if d.Addition.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) + } + return nil +} + +func (d *Open115) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } return nil } @@ -59,6 +71,9 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) pageSize := int64(200) offset := int64(0) for { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ CID: dir.GetID(), Limit: pageSize, @@ -84,6 +99,9 @@ func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) } func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } var ua string if args.Header != nil { ua = args.Header.Get("User-Agent") @@ -113,11 +131,13 @@ func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } resp, err := d.client.Mkdir(ctx, parentDir.GetID(), dirName) if err != nil { return nil, err } - time.Sleep(800 * time.Millisecond) return &Obj{ Fid: resp.FileID, Pid: parentDir.GetID(), @@ -130,6 +150,9 @@ func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri } func (d *Open115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } _, err := d.client.Move(ctx, &sdk.MoveReq{ FileIDs: srcObj.GetID(), ToCid: dstDir.GetID(), @@ -141,6 +164,9 @@ func (d *Open115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj } func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ FileID: srcObj.GetID(), FileNma: newName, @@ -156,6 +182,9 @@ func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *Open115) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } _, err := d.client.Copy(ctx, &sdk.CopyReq{ PID: dstDir.GetID(), FileID: srcObj.GetID(), @@ -168,6 +197,9 @@ func (d *Open115) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj } func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } _obj, ok := obj.(*Obj) if !ok { return fmt.Errorf("can't convert obj") @@ -183,6 +215,9 @@ func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { } func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } tempF, err := file.CacheFullInTempFile() if err != nil { return err diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go index 7e26e0ddbc2..66b956c0a9d 100644 --- a/drivers/115_open/meta.go +++ b/drivers/115_open/meta.go @@ -9,9 +9,10 @@ type Addition struct { // Usually one of two driver.RootID // define other - RefreshToken string `json:"refresh_token" required:"true"` - OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + RefreshToken string `json:"refresh_token" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` AccessToken string } From ffa03bfda11aa18bb899afc1f29e8690fcea1036 Mon Sep 17 00:00:00 2001 From: MadDogOwner Date: Sat, 24 May 2025 13:38:43 +0800 Subject: [PATCH 519/659] feat(cloudreve_v4): add Cloudreve V4 driver (#8470 closes #8328 #8467) * feat(cloudreve_v4): add Cloudreve V4 driver implementation * fix(cloudreve_v4): update request handling to prevent token refresh loop * feat(onedrive): implement retry logic for upload failures * feat(cloudreve): implement retry logic for upload failures * feat(cloudreve_v4): support cloud sorting * fix(cloudreve_v4): improve token handling in Init method * feat(cloudreve_v4): support share * feat(cloudreve): support reference * feat(cloudreve_v4): support version upload * fix(cloudreve_v4): add SetBody in upLocal * fix(cloudreve_v4): update URL structure in Link and FileUrlResp --- drivers/all.go | 1 + drivers/cloudreve/driver.go | 11 + drivers/cloudreve/util.go | 157 ++++++++--- drivers/cloudreve_v4/driver.go | 305 +++++++++++++++++++++ drivers/cloudreve_v4/meta.go | 44 +++ drivers/cloudreve_v4/types.go | 164 ++++++++++++ drivers/cloudreve_v4/util.go | 476 +++++++++++++++++++++++++++++++++ drivers/onedrive/util.go | 33 ++- drivers/onedrive_app/util.go | 33 ++- 9 files changed, 1158 insertions(+), 66 deletions(-) create mode 100644 drivers/cloudreve_v4/driver.go create mode 100644 drivers/cloudreve_v4/meta.go create mode 100644 drivers/cloudreve_v4/types.go create mode 100644 drivers/cloudreve_v4/util.go diff --git a/drivers/all.go b/drivers/all.go index 0b8ce3aa61d..224fb8ddb4b 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -22,6 +22,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_share" _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" + _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" _ "github.com/alist-org/alist/v3/drivers/doubao" _ "github.com/alist-org/alist/v3/drivers/doubao_share" diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 8c2321b8f40..dcde58c638d 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -18,6 +18,7 @@ import ( type Cloudreve struct { model.Storage Addition + ref *Cloudreve } func (d *Cloudreve) Config() driver.Config { @@ -37,8 +38,18 @@ func (d *Cloudreve) Init(ctx context.Context) error { return d.login() } +func (d *Cloudreve) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*Cloudreve) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + func (d *Cloudreve) Drop(ctx context.Context) error { d.Cookie = "" + d.ref = nil return nil } diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index 196d7303337..5054de6cb56 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -4,12 +4,14 @@ import ( "bytes" "context" "encoding/base64" + "encoding/json" "errors" "fmt" "io" "net/http" "strconv" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" @@ -19,7 +21,6 @@ import ( "github.com/alist-org/alist/v3/pkg/cookie" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - json "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go" ) @@ -35,6 +36,9 @@ func (d *Cloudreve) getUA() string { } func (d *Cloudreve) request(method string, path string, callback base.ReqCallback, out interface{}) error { + if d.ref != nil { + return d.ref.request(method, path, callback, out) + } u := d.Address + "/api/v3" + path req := base.RestyClient.R() req.SetHeaders(map[string]string{ @@ -79,11 +83,11 @@ func (d *Cloudreve) request(method string, path string, callback base.ReqCallbac } if out != nil && r.Data != nil { var marshal []byte - marshal, err = json.Marshal(r.Data) + marshal, err = jsoniter.Marshal(r.Data) if err != nil { return err } - err = json.Unmarshal(marshal, out) + err = jsoniter.Unmarshal(marshal, out) if err != nil { return err } @@ -187,12 +191,9 @@ func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u Up if utils.IsCanceled(ctx) { return ctx.Err() } - utils.Log.Debugf("[Cloudreve-Local] upload: %d", finish) - var byteSize = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Cloudreve-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) utils.Log.Debug(err, n) @@ -205,9 +206,26 @@ func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u Up req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) req.SetHeader("User-Agent", d.getUA()) req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + req.AddRetryCondition(func(r *resty.Response, err error) bool { + if err != nil { + return true + } + if r.IsError() { + return true + } + var retryResp Resp + jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) + if jErr != nil { + return true + } + if retryResp.Code != 0 { + return true + } + return false + }) }, nil) if err != nil { - break + return err } finish += byteSize up(float64(finish) * 100 / float64(stream.GetSize())) @@ -222,16 +240,15 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U var finish int64 = 0 var chunk int = 0 DEFAULT := int64(u.ChunkSize) + retryCount := 0 + maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } - utils.Log.Debugf("[Cloudreve-Remote] upload: %d", finish) - var byteSize = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) utils.Log.Debug(err, n) @@ -248,14 +265,43 @@ func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u U // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Authorization", fmt.Sprint(credential)) req.Header.Set("User-Agent", d.getUA()) - finish += byteSize - res, err := base.HttpClient.Do(req) - if err != nil { - return err + err = func() error { + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }() + if err == nil { + retryCount = 0 + finish += byteSize + up(float64(finish) * 100 / float64(stream.GetSize())) + chunk++ + } else { + retryCount++ + if retryCount > maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< 0 { + src.Size = ds.FolderSummary.Size + } + } + var thumb model.Thumbnail + if d.EnableThumb && src.Type == 0 { + var t FileThumbResp + err := d.request(http.MethodGet, "/file/thumb", func(req *resty.Request) { + req.SetQueryParam("uri", src.Path) + }, &t) + if err == nil && t.URL != "" { + thumb = model.Thumbnail{ + Thumbnail: t.URL, + } + } + } + return &model.ObjThumb{ + Object: model.Object{ + ID: src.ID, + Path: src.Path, + Name: src.Name, + Size: src.Size, + Modified: src.UpdatedAt, + Ctime: src.CreatedAt, + IsFolder: src.Type == 1, + }, + Thumbnail: thumb, + }, nil + }) +} + +func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var url FileUrlResp + err := d.request(http.MethodPost, "/file/url", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{file.GetPath()}, + "download": true, + }) + }, &url) + if err != nil { + return nil, err + } + if len(url.Urls) == 0 { + return nil, errors.New("server returns no url") + } + exp := time.Until(url.Expires) + return &model.Link{ + URL: url.Urls[0].URL, + Expiration: &exp, + }, nil +} + +func (d *CloudreveV4) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "type": "folder", + "uri": parentDir.GetPath() + "/" + dirName, + "error_on_conflict": true, + }) + }, nil) +} + +func (d *CloudreveV4) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{srcObj.GetPath()}, + "dst": dstDir.GetPath(), + "copy": false, + }) + }, nil) +} + +func (d *CloudreveV4) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "new_name": newName, + "uri": srcObj.GetPath(), + }) + }, nil) + +} + +func (d *CloudreveV4) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{srcObj.GetPath()}, + "dst": dstDir.GetPath(), + "copy": true, + }) + }, nil) +} + +func (d *CloudreveV4) Remove(ctx context.Context, obj model.Obj) error { + return d.request(http.MethodDelete, "/file", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{obj.GetPath()}, + "unlink": false, + "skip_soft_delete": true, + }) + }, nil) +} + +func (d *CloudreveV4) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if file.GetSize() == 0 { + // 空文件使用新建文件方法，避免上传卡锁 + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "type": "file", + "uri": dstDir.GetPath() + "/" + file.GetName(), + "error_on_conflict": true, + }) + }, nil) + } + var p StoragePolicy + var r FileResp + var u FileUploadResp + var err error + params := map[string]string{ + "page_size": "10", + "uri": dstDir.GetPath(), + "order_by": "created_at", + "order_direction": "asc", + "page": "0", + } + err = d.request(http.MethodGet, "/file", func(req *resty.Request) { + req.SetQueryParams(params) + }, &r) + if err != nil { + return err + } + p = r.StoragePolicy + body := base.Json{ + "uri": dstDir.GetPath() + "/" + file.GetName(), + "size": file.GetSize(), + "policy_id": p.ID, + "last_modified": file.ModTime().UnixMilli(), + "mime_type": "", + } + if d.EnableVersionUpload { + body["entity_type"] = "version" + } + err = d.request(http.MethodPut, "/file/upload", func(req *resty.Request) { + req.SetBody(body) + }, &u) + if err != nil { + return err + } + if u.StoragePolicy.Relay { + err = d.upLocal(ctx, file, u, up) + } else { + switch u.StoragePolicy.Type { + case "local": + err = d.upLocal(ctx, file, u, up) + case "remote": + err = d.upRemote(ctx, file, u, up) + case "onedrive": + err = d.upOneDrive(ctx, file, u, up) + case "s3": + err = d.upS3(ctx, file, u, up) + default: + return errs.NotImplement + } + } + if err != nil { + // 删除失败的会话 + _ = d.request(http.MethodDelete, "/file/upload", func(req *resty.Request) { + req.SetBody(base.Json{ + "id": u.SessionID, + "uri": u.URI, + }) + }, nil) + return err + } + return nil +} + +func (d *CloudreveV4) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *CloudreveV4) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*CloudreveV4)(nil) diff --git a/drivers/cloudreve_v4/meta.go b/drivers/cloudreve_v4/meta.go new file mode 100644 index 00000000000..bfaa14f81e4 --- /dev/null +++ b/drivers/cloudreve_v4/meta.go @@ -0,0 +1,44 @@ +package cloudreve_v4 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // driver.RootID + // define other + Address string `json:"address" required:"true"` + Username string `json:"username"` + Password string `json:"password"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + CustomUA string `json:"custom_ua"` + EnableFolderSize bool `json:"enable_folder_size"` + EnableThumb bool `json:"enable_thumb"` + EnableVersionUpload bool `json:"enable_version_upload"` + OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at" default:"name" required:"true"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc" required:"true"` +} + +var config = driver.Config{ + Name: "Cloudreve V4", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "cloudreve://my", + CheckStatus: true, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &CloudreveV4{} + }) +} diff --git a/drivers/cloudreve_v4/types.go b/drivers/cloudreve_v4/types.go new file mode 100644 index 00000000000..e81226d3da5 --- /dev/null +++ b/drivers/cloudreve_v4/types.go @@ -0,0 +1,164 @@ +package cloudreve_v4 + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Object struct { + model.Object + StoragePolicy StoragePolicy +} + +type Resp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data any `json:"data"` +} + +type BasicConfigResp struct { + InstanceID string `json:"instance_id"` + // Title string `json:"title"` + // Themes string `json:"themes"` + // DefaultTheme string `json:"default_theme"` + User struct { + ID string `json:"id"` + // Nickname string `json:"nickname"` + // CreatedAt time.Time `json:"created_at"` + // Anonymous bool `json:"anonymous"` + Group struct { + ID string `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` + } `json:"group"` + } `json:"user"` + // Logo string `json:"logo"` + // LogoLight string `json:"logo_light"` + // CaptchaReCaptchaKey string `json:"captcha_ReCaptchaKey"` + CaptchaType string `json:"captcha_type"` // support 'normal' only + // AppPromotion bool `json:"app_promotion"` +} + +type SiteLoginConfigResp struct { + LoginCaptcha bool `json:"login_captcha"` + Authn bool `json:"authn"` +} + +type PrepareLoginResp struct { + WebauthnEnabled bool `json:"webauthn_enabled"` + PasswordEnabled bool `json:"password_enabled"` +} + +type CaptchaResp struct { + Image string `json:"image"` + Ticket string `json:"ticket"` +} + +type Token struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + AccessExpires time.Time `json:"access_expires"` + RefreshExpires time.Time `json:"refresh_expires"` +} + +type TokenResponse struct { + User struct { + ID string `json:"id"` + // Email string `json:"email"` + // Nickname string `json:"nickname"` + Status string `json:"status"` + // CreatedAt time.Time `json:"created_at"` + Group struct { + ID string `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` + // DirectLinkBatchSize int `json:"direct_link_batch_size"` + // TrashRetention int `json:"trash_retention"` + } `json:"group"` + // Language string `json:"language"` + } `json:"user"` + Token Token `json:"token"` +} + +type File struct { + Type int `json:"type"` // 0: file, 1: folder + ID string `json:"id"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Size int64 `json:"size"` + Metadata interface{} `json:"metadata"` + Path string `json:"path"` + Capability string `json:"capability"` + Owned bool `json:"owned"` + PrimaryEntity string `json:"primary_entity"` +} + +type StoragePolicy struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + MaxSize int64 `json:"max_size"` + Relay bool `json:"relay,omitempty"` +} + +type Pagination struct { + Page int `json:"page"` + PageSize int `json:"page_size"` + IsCursor bool `json:"is_cursor"` + NextToken string `json:"next_token,omitempty"` +} + +type Props struct { + Capability string `json:"capability"` + MaxPageSize int `json:"max_page_size"` + OrderByOptions []string `json:"order_by_options"` + OrderDirectionOptions []string `json:"order_direction_options"` +} + +type FileResp struct { + Files []File `json:"files"` + Parent File `json:"parent"` + Pagination Pagination `json:"pagination"` + Props Props `json:"props"` + ContextHint string `json:"context_hint"` + MixedType bool `json:"mixed_type"` + StoragePolicy StoragePolicy `json:"storage_policy"` +} + +type FileUrlResp struct { + Urls []struct { + URL string `json:"url"` + } `json:"urls"` + Expires time.Time `json:"expires"` +} + +type FileUploadResp struct { + // UploadID string `json:"upload_id"` + SessionID string `json:"session_id"` + ChunkSize int64 `json:"chunk_size"` + Expires int64 `json:"expires"` + StoragePolicy StoragePolicy `json:"storage_policy"` + URI string `json:"uri"` + CompleteURL string `json:"completeURL,omitempty"` // for S3-like + CallbackSecret string `json:"callback_secret,omitempty"` // for S3-like, OneDrive + UploadUrls []string `json:"upload_urls,omitempty"` // for not-local + Credential string `json:"credential,omitempty"` // for local +} + +type FileThumbResp struct { + URL string `json:"url"` + Expires time.Time `json:"expires"` +} + +type FolderSummaryResp struct { + File + FolderSummary struct { + Size int64 `json:"size"` + Files int64 `json:"files"` + Folders int64 `json:"folders"` + Completed bool `json:"completed"` + CalculatedAt time.Time `json:"calculated_at"` + } `json:"folder_summary"` +} diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go new file mode 100644 index 00000000000..cf2337f279b --- /dev/null +++ b/drivers/cloudreve_v4/util.go @@ -0,0 +1,476 @@ +package cloudreve_v4 + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + jsoniter "github.com/json-iterator/go" +) + +// do others that not defined in Driver interface + +func (d *CloudreveV4) getUA() string { + if d.CustomUA != "" { + return d.CustomUA + } + return base.UserAgent +} + +func (d *CloudreveV4) request(method string, path string, callback base.ReqCallback, out any) error { + if d.ref != nil { + return d.ref.request(method, path, callback, out) + } + u := d.Address + "/api/v4" + path + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "User-Agent": d.getUA(), + }) + if d.AccessToken != "" { + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + } + + var r Resp + req.SetResult(&r) + + if callback != nil { + callback(req) + } + + resp, err := req.Execute(method, u) + if err != nil { + return err + } + if !resp.IsSuccess() { + return errors.New(resp.String()) + } + + if r.Code != 0 { + if r.Code == 401 && d.RefreshToken != "" && path != "/session/token/refresh" { + // try to refresh token + err = d.refreshToken() + if err != nil { + return err + } + return d.request(method, path, callback, out) + } + return errors.New(r.Msg) + } + + if out != nil && r.Data != nil { + var marshal []byte + marshal, err = json.Marshal(r.Data) + if err != nil { + return err + } + err = json.Unmarshal(marshal, out) + if err != nil { + return err + } + } + + return nil +} + +func (d *CloudreveV4) login() error { + var siteConfig SiteLoginConfigResp + err := d.request(http.MethodGet, "/site/config/login", nil, &siteConfig) + if err != nil { + return err + } + if !siteConfig.Authn { + return errors.New("authn not support") + } + var prepareLogin PrepareLoginResp + err = d.request(http.MethodGet, "/session/prepare?email="+d.Addition.Username, nil, &prepareLogin) + if err != nil { + return err + } + if !prepareLogin.PasswordEnabled { + return errors.New("password not enabled") + } + if prepareLogin.WebauthnEnabled { + return errors.New("webauthn not support") + } + for range 5 { + err = d.doLogin(siteConfig.LoginCaptcha) + if err == nil { + break + } + if err.Error() != "CAPTCHA not match." { + break + } + } + return err +} + +func (d *CloudreveV4) doLogin(needCaptcha bool) error { + var err error + loginBody := base.Json{ + "email": d.Username, + "password": d.Password, + } + if needCaptcha { + var config BasicConfigResp + err = d.request(http.MethodGet, "/site/config/basic", nil, &config) + if err != nil { + return err + } + if config.CaptchaType != "normal" { + return fmt.Errorf("captcha type %s not support", config.CaptchaType) + } + var captcha CaptchaResp + err = d.request(http.MethodGet, "/site/captcha", nil, &captcha) + if err != nil { + return err + } + if !strings.HasPrefix(captcha.Image, "data:image/png;base64,") { + return errors.New("can not get captcha") + } + loginBody["ticket"] = captcha.Ticket + i := strings.Index(captcha.Image, ",") + dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(captcha.Image[i+1:])) + vRes, err := base.RestyClient.R().SetMultipartField( + "image", "validateCode.png", "image/png", dec). + Post(setting.GetStr(conf.OcrApi)) + if err != nil { + return err + } + if jsoniter.Get(vRes.Body(), "status").ToInt() != 200 { + return errors.New("ocr error:" + jsoniter.Get(vRes.Body(), "msg").ToString()) + } + captchaCode := jsoniter.Get(vRes.Body(), "result").ToString() + if captchaCode == "" { + return errors.New("ocr error: empty result") + } + loginBody["captcha"] = captchaCode + } + var token TokenResponse + err = d.request(http.MethodPost, "/session/token", func(req *resty.Request) { + req.SetBody(loginBody) + }, &token) + if err != nil { + return err + } + d.AccessToken, d.RefreshToken = token.Token.AccessToken, token.Token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *CloudreveV4) refreshToken() error { + var token Token + if token.RefreshToken == "" { + if d.Username != "" { + err := d.login() + if err != nil { + return fmt.Errorf("cannot login to get refresh token, error: %s", err) + } + } + return nil + } + err := d.request(http.MethodPost, "/session/token/refresh", func(req *resty.Request) { + req.SetBody(base.Json{ + "refresh_token": d.RefreshToken, + }) + }, &token) + if err != nil { + return err + } + d.AccessToken, d.RefreshToken = token.AccessToken, token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *CloudreveV4) upLocal(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + if DEFAULT == 0 { + // support relay + DEFAULT = file.GetSize() + } + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[CloudreveV4-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { + req.SetHeader("Content-Type", "application/octet-stream") + req.SetContentLength(true) + req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) + req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + req.AddRetryCondition(func(r *resty.Response, err error) bool { + if err != nil { + return true + } + if r.IsError() { + return true + } + var retryResp Resp + jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) + if jErr != nil { + return true + } + if retryResp.Code != 0 { + return true + } + return false + }) + }, nil) + if err != nil { + return err + } + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + chunk++ + } + return nil +} + +func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { + uploadUrl := u.UploadUrls[0] + credential := u.Credential + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + retryCount := 0 + maxRetries := 3 + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + err = func() error { + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }() + if err == nil { + retryCount = 0 + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + chunk++ + } else { + retryCount++ + if retryCount > maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors", maxRetries) + } + backoff := time.Duration(1<") + for i, etag := range etags { + bodyBuilder.WriteString(fmt.Sprintf( + `%d%s`, + i+1, // PartNumber 从 1 开始 + etag, + )) + } + bodyBuilder.WriteString("") + req, err := http.NewRequest( + "POST", + u.CompleteURL, + strings.NewReader(bodyBuilder.String()), + ) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/xml") + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body)) + } + + // 上传成功发送回调请求 + return d.request(http.MethodPost, "/callback/s3/"+u.SessionID+"/"+u.CallbackSecret, func(req *resty.Request) { + req.SetBody("{}") + }, nil) +} diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index e256b7ae262..28ed5ccc3cc 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -8,6 +8,7 @@ import ( "io" "net/http" stdpath "path" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -17,7 +18,6 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" - log "github.com/sirupsen/logrus" ) var onedriveHostMap = map[string]Host{ @@ -204,19 +204,18 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 DEFAULT := d.ChunkSize * 1024 * 1024 + retryCount := 0 + maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } - log.Debugf("upload: %d", finish) - var byteSize int64 = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) - log.Debug(err, n) + utils.Log.Debug(err, n) if err != nil { return err } @@ -228,19 +227,31 @@ func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.Fil req.ContentLength = byteSize // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - finish += byteSize res, err := base.HttpClient.Do(req) if err != nil { return err } // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { + switch { + case res.StatusCode >= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< Date: Wed, 4 Jun 2025 15:04:06 +0000 Subject: [PATCH 520/659] feat(local): add options to use ffmpeg to generate thumbnail --- drivers/local/driver.go | 15 ++++++ drivers/local/meta.go | 2 + drivers/local/util.go | 114 +++++++++++++++++++++++++++++++++------- 3 files changed, 113 insertions(+), 18 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index faa2b3bd157..0e469cb190e 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -39,6 +39,10 @@ type Local struct { // video thumb position videoThumbPos float64 videoThumbPosIsPercentage bool + thumbPixel int + + // use ffmpeg + useFFmpeg bool } func (d *Local) Config() driver.Config { @@ -65,6 +69,9 @@ func (d *Local) Init(ctx context.Context) error { } d.Addition.RootFolderPath = abs } + + d.useFFmpeg = d.UseFFmpeg + if d.ThumbCacheFolder != "" && !utils.Exists(d.ThumbCacheFolder) { err := os.MkdirAll(d.ThumbCacheFolder, os.FileMode(d.mkdirPerm)) if err != nil { @@ -78,6 +85,14 @@ func (d *Local) Init(ctx context.Context) error { } d.thumbConcurrency = int(v) } + if d.ThumbPixel != "" { + v, err := strconv.ParseUint(d.ThumbPixel, 10, 32) + if err != nil { + return err + } + d.thumbPixel = int(v) + } + if d.thumbConcurrency == 0 { d.thumbTokenBucket = NewNopTokenBucket() } else { diff --git a/drivers/local/meta.go b/drivers/local/meta.go index 14b0404f784..70ce090db5f 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -8,8 +8,10 @@ import ( type Addition struct { driver.RootPath Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` + UseFFmpeg bool `json:"use_ffmpeg" required:"true" help:"use ffmpeg to generate thumbnail"` ThumbCacheFolder string `json:"thumb_cache_folder"` ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` + ThumbPixel string `json:"thumb_pixel" default:"320" required:"false" help:"Specifies the target width for image thumbnails in pixels. The height of the thumbnail will be calculated automatically to maintain the original aspect ratio of the image."` VideoThumbPos string `json:"video_thumb_pos" default:"20%" required:"false" help:"The position of the video thumbnail. If the value is a number (integer ot floating point), it represents the time in seconds. If the value ends with '%', it represents the percentage of the video duration."` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` MkdirPerm string `json:"mkdir_perm" default:"777"` diff --git a/drivers/local/util.go b/drivers/local/util.go index 802f60cf627..a473b709dd8 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -36,6 +36,87 @@ func isSymlinkDir(f fs.FileInfo, path string) bool { return false } +// resizeImageToBufferWithFFmpegGo 使用 ffmpeg-go 调整图片大小并输出到内存缓冲区 +func resizeImageToBufferWithFFmpegGo(inputFile string, width int, outputFormat string /* e.g., "image2pipe", "png_pipe", "mjpeg" */) (*bytes.Buffer, error) { + outBuffer := bytes.NewBuffer(nil) + + // Determine codec based on desired output format for piping + // For generic image piping, 'image2' is often used with -f image2pipe + // For specific formats to buffer, you might specify the codec directly + var vcodec string + switch outputFormat { + case "png_pipe": // if you want to ensure PNG format in buffer + vcodec = "png" + case "mjpeg": // if you want to ensure JPEG format in buffer + vcodec = "mjpeg" + // default or "image2pipe" could leave codec choice more to ffmpeg or require -c:v later + } + + outputArgs := ffmpeg.KwArgs{ + "vf": fmt.Sprintf("scale=%d:-1:flags=lanczos,format=yuv444p", width), + "vframes": "1", + "f": outputFormat, // Format for piping (e.g., image2pipe, png_pipe) + } + if vcodec != "" { + outputArgs["vcodec"] = vcodec + } + if outputFormat == "mjpeg" { + outputArgs["q:v"] = "3" + } + + err := ffmpeg.Input(inputFile). + Output("pipe:", outputArgs). // Output to pipe (stdout) + GlobalArgs("-loglevel", "error"). + Silent(true). // Suppress ffmpeg's own console output + WithOutput(outBuffer, os.Stderr). // Capture stdout to outBuffer, stderr to os.Stderr + // ErrorToStdOut(). // Alternative: send ffmpeg's stderr to Go's stdout + Run() + + if err != nil { + return nil, fmt.Errorf("ffmpeg-go failed to resize image %s to buffer: %w", inputFile, err) + } + if outBuffer.Len() == 0 { + return nil, fmt.Errorf("ffmpeg-go produced empty buffer for %s", inputFile) + } + + return outBuffer, nil +} + +func generateThumbnailWithImagingOptimized(imagePath string, targetWidth int, quality int) (*bytes.Buffer, error) { + + file, err := os.Open(imagePath) + if err != nil { + return nil, fmt.Errorf("failed to open image: %w", err) + } + defer file.Close() + + img, err := imaging.Decode(file, imaging.AutoOrientation(true)) + if err != nil { + return nil, fmt.Errorf("failed to decode image: %w", err) + } + + thumbImg := imaging.Resize(img, targetWidth, 0, imaging.Lanczos) + img = nil + + var buf bytes.Buffer + // imaging.Encode + // imaging.PNG, imaging.JPEG, imaging.GIF, imaging.BMP, imaging.TIFF + outputFormat := imaging.JPEG + encodeOptions := []imaging.EncodeOption{imaging.JPEGQuality(quality)} + + // outputFormat := imaging.PNG + // encodeOptions := []imaging.EncodeOption{} + + err = imaging.Encode(&buf, thumbImg, outputFormat, encodeOptions...) + if err != nil { + return nil, fmt.Errorf("failed to encode thumbnail: %w", err) + } + + thumbImg = nil + + return &buf, nil +} + // Get the snapshot of the video func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) { // Run ffprobe to get the video duration @@ -80,7 +161,7 @@ func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) // The "noaccurate_seek" option prevents this error and would also speed up // the seek process. stream := ffmpeg.Input(videoPath, ffmpeg.KwArgs{"ss": ss, "noaccurate_seek": ""}). - Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg"}). + Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg", "vf": fmt.Sprintf("scale=%d:-1:flags=lanczos", d.thumbPixel)}). GlobalArgs("-loglevel", "error").Silent(true). WithOutput(srcBuf, os.Stdout) if err = stream.Run(); err != nil { @@ -125,29 +206,26 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { } srcBuf = videoBuf } else { - imgData, err := os.ReadFile(fullPath) - if err != nil { - return nil, nil, err + if d.useFFmpeg { + imgData, err := resizeImageToBufferWithFFmpegGo(fullPath, d.thumbPixel, "image2pipe") + srcBuf = imgData + if err != nil { + return nil, nil, err + } + } else { + imgData, err := generateThumbnailWithImagingOptimized(fullPath, d.thumbPixel, 70) + srcBuf = imgData + if err != nil { + return nil, nil, err + } } - imgBuf := bytes.NewBuffer(imgData) - srcBuf = imgBuf } - image, err := imaging.Decode(srcBuf, imaging.AutoOrientation(true)) - if err != nil { - return nil, nil, err - } - thumbImg := imaging.Resize(image, 144, 0, imaging.Lanczos) - var buf bytes.Buffer - err = imaging.Encode(&buf, thumbImg, imaging.PNG) - if err != nil { - return nil, nil, err - } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0666) + err := os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), srcBuf.Bytes(), 0666) if err != nil { return nil, nil, err } } - return &buf, nil, nil + return srcBuf, nil, nil } From 7aeb0ab078c78dad7e1acb531f3cd0fdf78f2d62 Mon Sep 17 00:00:00 2001 From: AlistDev Date: Fri, 27 Jun 2025 16:28:09 +0800 Subject: [PATCH 521/659] fix: update documentation links to point to the new domain And fix 189pc getToken fail --- .github/FUNDING.yml | 2 +- .github/ISSUE_TEMPLATE/bug_report.yml | 8 ++++---- .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- README.md | 8 ++++---- README_cn.md | 10 +++++----- README_ja.md | 10 +++++----- cmd/root.go | 2 +- drivers/189pc/utils.go | 2 +- drivers/onedrive/meta.go | 2 +- internal/bootstrap/data/setting.go | 2 +- internal/model/user.go | 2 +- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index a336406b355..f9e80a5aada 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -10,4 +10,4 @@ liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry -custom: ['https://alist.nn.ci/guide/sponsor.html'] +custom: ['https://alistgo.com/guide/sponsor.html'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 07a8338e5ef..f5cfaedacf8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -16,14 +16,14 @@ body: 您必须勾选以下所有内容，否则您的issue可能会被直接关闭。或者您可以去[讨论区](https://github.com/alist-org/alist/discussions) options: - label: | - I have read the [documentation](https://alist.nn.ci). - 我已经阅读了[文档](https://alist.nn.ci)。 + I have read the [documentation](https://alistgo.com). + 我已经阅读了[文档](https://alistgo.com)。 - label: | I'm sure there are no duplicate issues or discussions. 我确定没有重复的issue或讨论。 - label: | - I'm sure it's due to `AList` and not something else(such as [Network](https://alist.nn.ci/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host) ,`Dependencies` or `Operational`). - 我确定是`AList`的问题，而不是其他原因（例如[网络](https://alist.nn.ci/zh/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host)，`依赖`或`操作`）。 + I'm sure it's due to `AList` and not something else(such as [Network](https://alistgo.com/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host) ,`Dependencies` or `Operational`). + 我确定是`AList`的问题，而不是其他原因（例如[网络](https://alistgo.com/zh/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host)，`依赖`或`操作`）。 - label: | I'm sure this issue is not fixed in the latest version. 我确定这个问题在最新版本中没有被修复。 diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index a16c8f98d24..a118992ce0b 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -7,7 +7,7 @@ body: label: Please make sure of the following things description: You may select more than one, even select all. options: - - label: I have read the [documentation](https://alist.nn.ci). + - label: I have read the [documentation](https://alistgo.com). - label: I'm sure there are no duplicate issues or discussions. - label: I'm sure this feature is not implemented. - label: I'm sure it's a reasonable and popular requirement. diff --git a/README.md b/README.md index 1261839e429..2bd7e812890 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@
- logo + logo

🗂️A file list program that supports multiple storages, powered by Gin and Solidjs.

@@ -88,7 +88,7 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] Dark mode - [x] I18n - [x] Protected routes (password protection and authentication) -- [x] WebDav (see https://alist.nn.ci/guide/webdav.html for details) +- [x] WebDav (see https://alistgo.com/guide/webdav.html for details) - [x] [Docker Deploy](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare Workers proxy - [x] File/Folder package download @@ -112,7 +112,7 @@ Please go to our [discussion forum](https://github.com/alist-org/alist/discussio ## Sponsor AList is an open-source software, if you happen to like this project and want me to keep going, please consider sponsoring me or providing a single donation! Thanks for all the love and support: -https://alist.nn.ci/guide/sponsor.html +https://alistgo.com/guide/sponsor.html ### Special sponsors diff --git a/README_cn.md b/README_cn.md index 5c71ccce4c3..9052e79b0ea 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,5 +1,5 @@
- logo + logo

🗂一个支持多存储的文件列表程序，使用 Gin 和 Solidjs。

@@ -86,7 +86,7 @@ - [x] 黑暗模式 - [x] 国际化 - [x] 受保护的路由（密码保护和身份验证） -- [x] WebDav (具体见 https://alist.nn.ci/zh/guide/webdav.html) +- [x] WebDav (具体见 https://alistgo.com/zh/guide/webdav.html) - [x] [Docker 部署](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare workers 中转 - [x] 文件/文件夹打包下载 @@ -97,7 +97,7 @@ ## 文档 - + ## Demo @@ -109,7 +109,7 @@ ## 赞助 -AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我继续下去，请考虑赞助我或提供一个单一的捐款！感谢所有的爱和支持：https://alist.nn.ci/zh/guide/sponsor.html +AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我继续下去，请考虑赞助我或提供一个单一的捐款！感谢所有的爱和支持：https://alistgo.com/zh/guide/sponsor.html ### 特别赞助 diff --git a/README_ja.md b/README_ja.md index cd4446fab8e..4dcdfd203bf 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,5 +1,5 @@
- logo + logo

🗂️Gin と Solidjs による、複数のストレージをサポートするファイルリストプログラム。

@@ -87,7 +87,7 @@ - [x] ダークモード - [x] 国際化 - [x] 保護されたルート (パスワード保護と認証) -- [x] WebDav (詳細は https://alist.nn.ci/guide/webdav.html を参照) +- [x] WebDav (詳細は https://alistgo.com/guide/webdav.html を参照) - [x] [Docker デプロイ](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare ワーカープロキシ - [x] ファイル/フォルダパッケージのダウンロード @@ -98,7 +98,7 @@ ## ドキュメント - + ## デモ @@ -111,7 +111,7 @@ ## スポンサー AList はオープンソースのソフトウェアです。もしあなたがこのプロジェクトを気に入ってくださり、続けて欲しいと思ってくださるなら、ぜひスポンサーになってくださるか、1口でも寄付をしてくださるようご検討ください！すべての愛とサポートに感謝します: -https://alist.nn.ci/guide/sponsor.html +https://alistgo.com/guide/sponsor.html ### スペシャルスポンサー diff --git a/cmd/root.go b/cmd/root.go index 59eb989c3a0..cd50529728b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ var RootCmd = &cobra.Command{ Short: "A file list program that supports multiple storage.", Long: `A file list program that supports multiple storage, built with love by Xhofe and friends in Go/Solid.js. -Complete documentation is available at https://alist.nn.ci/`, +Complete documentation is available at https://alistgo.com/`, } func Execute() { diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index c391f7e676f..a8b444cb4b7 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -324,7 +324,7 @@ func (y *Cloud189PC) login() (err error) { _, err = y.client.R(). SetResult(&tokenInfo).SetError(&erron). SetQueryParams(clientSuffix()). - SetQueryParam("redirectURL", url.QueryEscape(loginresp.ToUrl)). + SetQueryParam("redirectURL", loginresp.ToUrl). Post(API_URL + "/getSessionForPC.action") if err != nil { return diff --git a/drivers/onedrive/meta.go b/drivers/onedrive/meta.go index a60e5f33a93..54a7340a942 100644 --- a/drivers/onedrive/meta.go +++ b/drivers/onedrive/meta.go @@ -11,7 +11,7 @@ type Addition struct { IsSharepoint bool `json:"is_sharepoint"` ClientID string `json:"client_id" required:"true"` ClientSecret string `json:"client_secret" required:"true"` - RedirectUri string `json:"redirect_uri" required:"true" default:"https://alist.nn.ci/tool/onedrive/callback"` + RedirectUri string `json:"redirect_uri" required:"true" default:"https://alistgo.com/tool/onedrive/callback"` RefreshToken string `json:"refresh_token" required:"true"` SiteId string `json:"site_id"` ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 407a5c64e17..fe1d9219a08 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -155,7 +155,7 @@ func InitialSettings() []model.SettingItem { ([[:xdigit:]]{1,4}(?::[[:xdigit:]]{1,4}){7}|::|:(?::[[:xdigit:]]{1,4}){1,6}|[[:xdigit:]]{1,4}:(?::[[:xdigit:]]{1,4}){1,5}|(?:[[:xdigit:]]{1,4}:){2}(?::[[:xdigit:]]{1,4}){1,4}|(?:[[:xdigit:]]{1,4}:){3}(?::[[:xdigit:]]{1,4}){1,3}|(?:[[:xdigit:]]{1,4}:){4}(?::[[:xdigit:]]{1,4}){1,2}|(?:[[:xdigit:]]{1,4}:){5}:[[:xdigit:]]{1,4}|(?:[[:xdigit:]]{1,4}:){1,6}:) (?U)access_token=(.*)&`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, - {Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, + {Key: conf.OcrApi, Value: "https://api.alistgo.com/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL}, {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, diff --git a/internal/model/user.go b/internal/model/user.go index eaa0fed9d09..0f7d3af5064 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -177,5 +177,5 @@ func (u *User) WebAuthnCredentials() []webauthn.Credential { } func (u *User) WebAuthnIcon() string { - return "https://alist.nn.ci/logo.svg" + return "https://alistgo.com/logo.svg" } From b1586612ca75e5a9e55ed79b829a76988584c004 Mon Sep 17 00:00:00 2001 From: Alone Date: Fri, 27 Jun 2025 23:39:23 +0800 Subject: [PATCH 522/659] feat: add ghcr docker image (#8524) --- .github/workflows/release_docker.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index 7cd05549f18..1c31b2fd20f 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -18,6 +18,7 @@ env: REGISTRY: 'xhofe/alist' REGISTRY_USERNAME: 'xhofe' REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + GITHUB_CR_REPO: ghcr.io/${{ github.repository }} ARTIFACT_NAME: 'binaries_docker_release' RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64' IMAGE_PUSH: ${{ github.event_name == 'push' }} @@ -114,11 +115,21 @@ jobs: username: ${{ env.REGISTRY_USERNAME }} password: ${{ env.REGISTRY_PASSWORD }} + - name: Login to GHCR + uses: docker/login-action@v3 + with: + logout: true + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Docker meta id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }} + images: | + ${{ env.REGISTRY }} + ${{ env.GITHUB_CR_REPO }} tags: ${{ env.IMAGE_IS_PROD == 'true' && '' || env.IMAGE_TAGS_BETA }} flavor: | ${{ env.IMAGE_IS_PROD == 'true' && 'latest=true' || '' }} From 51eeb224657972c6c8d910a6b4368cd702b8c2b4 Mon Sep 17 00:00:00 2001 From: alistgo Date: Fri, 27 Jun 2025 23:58:52 +0800 Subject: [PATCH 523/659] fix: dead link --- .github/workflows/beta_release.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- build.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml index 485942c4a9b..27e0142ea3f 100644 --- a/.github/workflows/beta_release.yml +++ b/.github/workflows/beta_release.yml @@ -119,7 +119,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 with: - repository: alist-org/desktop-release + repository: AlistGo/desktop-release ref: main persist-credentials: false fetch-depth: 0 @@ -135,4 +135,4 @@ jobs: with: github_token: ${{ secrets.MY_TOKEN }} branch: main - repository: alist-org/desktop-release \ No newline at end of file + repository: AlistGo/desktop-release \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1d42019ad06..2257826b064 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: - name: Checkout repo uses: actions/checkout@v4 with: - repository: alist-org/desktop-release + repository: AlistGo/desktop-release ref: main persist-credentials: false fetch-depth: 0 @@ -89,4 +89,4 @@ jobs: with: github_token: ${{ secrets.MY_TOKEN }} branch: main - repository: alist-org/desktop-release \ No newline at end of file + repository: AlistGo/desktop-release \ No newline at end of file diff --git a/build.sh b/build.sh index 2dee8e20773..4045820adfd 100644 --- a/build.sh +++ b/build.sh @@ -93,7 +93,7 @@ BuildDocker() { PrepareBuildDockerMusl() { mkdir -p build/musl-libs - BASE="https://musl.cc/" + BASE="https://github.com/go-cross/musl-toolchain-archive/releases/latest/download/" FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross riscv64-linux-musl-cross powerpc64le-linux-musl-cross) for i in "${FILES[@]}"; do url="${BASE}${i}.tgz" @@ -245,7 +245,7 @@ BuildReleaseFreeBSD() { cgo_cc="clang --target=${CGO_ARGS[$i]} --sysroot=/opt/freebsd/${os_arch}" echo building for freebsd-${os_arch} sudo mkdir -p "/opt/freebsd/${os_arch}" - wget -q https://download.freebsd.org/releases/${os_arch}/14.1-RELEASE/base.txz + wget -q https://download.freebsd.org/releases/${os_arch}/14.3-RELEASE/base.txz sudo tar -xf ./base.txz -C /opt/freebsd/${os_arch} rm base.txz export GOOS=freebsd From fd411866797e404222bf2cecc8129f0231c9a7fa Mon Sep 17 00:00:00 2001 From: AlistDev Date: Mon, 14 Jul 2025 23:04:40 +0800 Subject: [PATCH 524/659] fix: update DriveId assignment to use DeviceID from Addition struct --- drivers/aliyundrive/driver.go | 2 +- drivers/aliyundrive/meta.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index 105e28b2e98..606ff385e81 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -55,7 +55,7 @@ func (d *AliDrive) Init(ctx context.Context) error { if err != nil { return err } - d.DriveId = utils.Json.Get(res, "default_drive_id").ToString() + d.DriveId = d.Addition.DeviceID d.UserID = utils.Json.Get(res, "user_id").ToString() d.cron = cron.NewCron(time.Hour * 2) d.cron.Do(func() { diff --git a/drivers/aliyundrive/meta.go b/drivers/aliyundrive/meta.go index 9aee856908d..a0ae8a5917d 100644 --- a/drivers/aliyundrive/meta.go +++ b/drivers/aliyundrive/meta.go @@ -7,8 +7,8 @@ import ( type Addition struct { driver.RootID - RefreshToken string `json:"refresh_token" required:"true"` - //DeviceID string `json:"device_id" required:"true"` + RefreshToken string `json:"refresh_token" required:"true"` + DeviceID string `json:"device_id" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` RapidUpload bool `json:"rapid_upload"` From 13ea1c14053163206ab37209959b88f6842fcd51 Mon Sep 17 00:00:00 2001 From: AlistDev Date: Wed, 16 Jul 2025 20:39:05 +0800 Subject: [PATCH 525/659] fix: restore user-agent header in HTTP requests --- drivers/123/util.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/123/util.go b/drivers/123/util.go index 7e5a23970c6..b85c9afbac8 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -161,12 +161,12 @@ func (d *Pan123) login() error { } res, err := base.RestyClient.R(). SetHeaders(map[string]string{ - "origin": "https://www.123pan.com", - "referer": "https://www.123pan.com/", - "user-agent": "Dart/2.19(dart:io)-alist", + "origin": "https://www.123pan.com", + "referer": "https://www.123pan.com/", + //"user-agent": "Dart/2.19(dart:io)-alist", "platform": "web", "app-version": "3", - //"user-agent": base.UserAgent, + "user-agent": base.UserAgent, }). SetBody(body).Post(SignIn) if err != nil { @@ -202,7 +202,7 @@ do: "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, - "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) alist-client", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", "platform": "web", "app-version": "3", //"user-agent": base.UserAgent, From 5e15a360b7aa47b7f8eab273523dfbccf9956b14 Mon Sep 17 00:00:00 2001 From: Sakana Date: Thu, 24 Jul 2025 15:30:12 +0800 Subject: [PATCH 526/659] feat(github_releases): concurrently request the GitHub API (#9211) --- drivers/github_releases/driver.go | 189 ++++++++++++++++++------------ drivers/github_releases/meta.go | 11 +- drivers/github_releases/types.go | 2 +- drivers/github_releases/util.go | 4 +- 4 files changed, 125 insertions(+), 81 deletions(-) diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go index b35aa57a41c..3268dc2fd88 100644 --- a/drivers/github_releases/driver.go +++ b/drivers/github_releases/driver.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "sync" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -36,88 +37,130 @@ func (d *GithubReleases) Drop(ctx context.Context) error { return nil } -func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files := make([]File, 0) - path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) +// processPoint 处理单个挂载点的文件列表 +func (d *GithubReleases) processPoint(point *MountPoint, path string, args model.ListArgs) []File { + var pointFiles []File - for i := range d.points { - point := &d.points[i] + if !d.Addition.ShowAllVersion { // latest + point.RequestLatestRelease(d.GetRequest, args.Refresh) + pointFiles = d.processLatestVersion(point, path) + } else { // all version + point.RequestReleases(d.GetRequest, args.Refresh) + pointFiles = d.processAllVersions(point, path) + } - if !d.Addition.ShowAllVersion { // latest - point.RequestRelease(d.GetRequest, args.Refresh) + return pointFiles +} - if point.Point == path { // 与仓库路径相同 - files = append(files, point.GetLatestRelease()...) - if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) - } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 - nextDir := GetNextDir(point.Point, path) - if nextDir == "" { - continue - } +// processLatestVersion 处理最新版本的逻辑 +func (d *GithubReleases) processLatestVersion(point *MountPoint, path string) []File { + var pointFiles []File - hasSameDir := false - for index := range files { - if files[index].GetName() == nextDir { - hasSameDir = true - files[index].Size += point.GetLatestSize() - break - } - } - if !hasSameDir { - files = append(files, File{ - Path: path + "/" + nextDir, - FileName: nextDir, - Size: point.GetLatestSize(), - UpdateAt: point.Release.PublishedAt, - CreateAt: point.Release.CreatedAt, - Type: "dir", - Url: "", - }) - } + if point.Point == path { // 与仓库路径相同 + pointFiles = append(pointFiles, point.GetLatestRelease()...) + if d.Addition.ShowReadme { + files := point.GetOtherFile(d.GetRequest, false) + pointFiles = append(pointFiles, files...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir != "" { + dirFile := File{ + Path: path + "/" + nextDir, + FileName: nextDir, + Size: point.GetLatestSize(), + UpdateAt: point.Release.PublishedAt, + CreateAt: point.Release.CreatedAt, + Type: "dir", + Url: "", } - } else { // all version - point.RequestReleases(d.GetRequest, args.Refresh) + pointFiles = append(pointFiles, dirFile) + } + } - if point.Point == path { // 与仓库路径相同 - files = append(files, point.GetAllVersion()...) - if d.Addition.ShowReadme { - files = append(files, point.GetOtherFile(d.GetRequest, args.Refresh)...) - } - } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 - nextDir := GetNextDir(point.Point, path) - if nextDir == "" { - continue - } + return pointFiles +} - hasSameDir := false - for index := range files { - if files[index].GetName() == nextDir { - hasSameDir = true - files[index].Size += point.GetAllVersionSize() - break - } - } - if !hasSameDir { - files = append(files, File{ - FileName: nextDir, - Path: path + "/" + nextDir, - Size: point.GetAllVersionSize(), - UpdateAt: (*point.Releases)[0].PublishedAt, - CreateAt: (*point.Releases)[0].CreatedAt, - Type: "dir", - Url: "", - }) - } - } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 - tagName := GetNextDir(path, point.Point) - if tagName == "" { - continue - } +// processAllVersions 处理所有版本的逻辑 +func (d *GithubReleases) processAllVersions(point *MountPoint, path string) []File { + var pointFiles []File - files = append(files, point.GetReleaseByTagName(tagName)...) + if point.Point == path { // 与仓库路径相同 + pointFiles = append(pointFiles, point.GetAllVersion()...) + if d.Addition.ShowReadme { + files := point.GetOtherFile(d.GetRequest, false) + pointFiles = append(pointFiles, files...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir != "" { + dirFile := File{ + FileName: nextDir, + Path: path + "/" + nextDir, + Size: point.GetAllVersionSize(), + UpdateAt: (*point.Releases)[0].PublishedAt, + CreateAt: (*point.Releases)[0].CreatedAt, + Type: "dir", + Url: "", } + pointFiles = append(pointFiles, dirFile) + } + } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 + tagName := GetNextDir(path, point.Point) + if tagName != "" { + pointFiles = append(pointFiles, point.GetReleaseByTagName(tagName)...) + } + } + + return pointFiles +} + +// mergeFiles 合并文件列表，处理重复目录 +func (d *GithubReleases) mergeFiles(files *[]File, newFiles []File) { + for _, newFile := range newFiles { + if newFile.Type == "dir" { + hasSameDir := false + for index := range *files { + if (*files)[index].GetName() == newFile.GetName() && (*files)[index].Type == "dir" { + hasSameDir = true + (*files)[index].Size += newFile.Size + break + } + } + if !hasSameDir { + *files = append(*files, newFile) + } + } else { + *files = append(*files, newFile) + } + } +} + +func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]File, 0) + path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) + + if d.Addition.ConcurrentRequests && d.Addition.Token != "" { // 并发处理 + var mu sync.Mutex + var wg sync.WaitGroup + + for i := range d.points { + wg.Add(1) + go func(point *MountPoint) { + defer wg.Done() + pointFiles := d.processPoint(point, path, args) + + mu.Lock() + d.mergeFiles(&files, pointFiles) + mu.Unlock() + }(&d.points[i]) + } + wg.Wait() + } else { // 串行处理 + for i := range d.points { + point := &d.points[i] + pointFiles := d.processPoint(point, path, args) + d.mergeFiles(&files, pointFiles) } } diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go index 47b84d37927..b54cb3cc608 100644 --- a/drivers/github_releases/meta.go +++ b/drivers/github_releases/meta.go @@ -7,11 +7,12 @@ import ( type Addition struct { driver.RootID - RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"` - ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` - Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` - ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` - GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` + RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"` + ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` + Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` + ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` + ConcurrentRequests bool `json:"concurrent_requests" type:"bool" default:"false" help:"To concurrently request the GitHub API, you must enter a GitHub token"` + GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` } var config = driver.Config{ diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go index b0a9ee619e0..b4562056185 100644 --- a/drivers/github_releases/types.go +++ b/drivers/github_releases/types.go @@ -18,7 +18,7 @@ type MountPoint struct { } // 请求最新版本 -func (m *MountPoint) RequestRelease(get func(url string) (*resty.Response, error), refresh bool) { +func (m *MountPoint) RequestLatestRelease(get func(url string) (*resty.Response, error), refresh bool) { if m.Repo == "" { return } diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go index df846e8a109..097295bf408 100644 --- a/drivers/github_releases/util.go +++ b/drivers/github_releases/util.go @@ -6,8 +6,8 @@ import ( "strings" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - log "github.com/sirupsen/logrus" ) // 发送 GET 请求 @@ -23,7 +23,7 @@ func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) { return nil, err } if res.StatusCode() != 200 { - log.Warn("failed to get request: ", res.StatusCode(), res.String()) + utils.Log.Warnf("failed to get request: %s %d %s", url, res.StatusCode(), res.String()) } return res, nil } From 00120cba273a131d5e916895e1d4ee324095cdf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 26 Jul 2025 09:51:59 +0800 Subject: [PATCH 527/659] feat: enhance permission control and label management (#9215) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 标签管理 * pr检查优化 * feat(role): Implement role management functionality - Add role management routes in `server/router.go` for listing, getting, creating, updating, and deleting roles - Introduce `initRoles()` in `internal/bootstrap/data/data.go` for initializing roles during bootstrap - Create `internal/op/role.go` to handle role operations including caching and singleflight - Implement role handler functions in `server/handles/role.go` for API responses - Define database operations for roles in `internal/db/role.go` - Extend `internal/db/db.go` for role model auto-migration - Design `internal/model/role.go` to represent role structure with ID, name, description, base path, and permissions - Initialize default roles (`admin` and `guest`) in `internal/bootstrap/data/role.go` during startup * refactor(user roles): Support multiple roles for users - Change the `Role` field type from `int` to `[]int` in `drivers/alist_v3/types.go` and `drivers/quqi/types.go`. - Update the `Role` field in `internal/model/user.go` to use a new `Roles` type with JSON and database support. - Modify `IsGuest` and `IsAdmin` methods to check for roles using `Contains` method. - Update `GetUserByRole` method in `internal/db/user.go` to handle multiple roles. - Add `roles.go` to define a new `Roles` type with JSON marshalling and scanning capabilities. - Adjust code in `server/handles/user.go` to compare roles with `utils.SliceEqual`. - Change role initialization for users in `internal/bootstrap/data/dev.go` and `internal/bootstrap/data/user.go`. - Update `Role` handling in `server/handles/task.go`, `server/handles/ssologin.go`, and `server/handles/ldap_login.go`. * feat(user/role): Add path limit check for user and role permissions - Add new permission bit for checking path limits in `user.go` - Implement `CheckPathLimit` method in `User` struct to validate path access - Modify `JoinPath` method in `User` to enforce path limit checks - Update `role.go` to include path limit logic in `Role` struct - Document new permission bit in `Role` and `User` comments for clarity * feat(permission): Add role-based permission handling - Introduce `role_perm.go` for managing user permissions based on roles. - Implement `HasPermission` and `MergeRolePermissions` functions. - Update `webdav.go` to utilize role-based permissions instead of direct user checks. - Modify `fsup.go` to integrate `CanAccessWithRoles` function. - Refactor `fsread.go` to use `common.HasPermission` for permission validation. - Adjust `fsmanage.go` for role-based access control checks. - Enhance `ftp.go` and `sftp.go` to manage FTP access via roles. - Update `fsbatch.go` to employ `MergeRolePermissions` for batch operations. - Replace direct user permission checks with role-based permission handling across various modules. * refactor(user): Replace integer role values with role IDs - Change `GetAdmin()` and `GetGuest()` functions to retrieve role by name and use role ID. - Add patch for version `v3.45.2` to convert legacy integer roles to role IDs. - Update `dev.go` and `user.go` to use role IDs instead of integer values for roles. - Remove redundant code in `role.go` related to guest role creation. - Modify `ssologin.go` and `ldap_login.go` to set user roles to nil instead of using integer roles. - Introduce `convert_roles.go` to handle conversion of legacy roles and ensure role existence in the database. * feat(role_perm): implement support for multiple base paths for roles - Modify role permission checks to support multiple base paths - Update role creation and update functions to handle multiple base paths - Add migration script to convert old base_path to base_paths - Define new Paths type for handling multiple paths in the model - Adjust role model to replace BasePath with BasePaths - Update existing patches to handle roles with multiple base paths - Update bootstrap data to reflect the new base_paths field * feat(role): Restrict modifications to default roles (admin and guest) - Add validation to prevent changes to "admin" and "guest" roles in `UpdateRole` and `DeleteRole` functions. - Introduce `ErrChangeDefaultRole` error in `internal/errs/role.go` to standardize error messaging. - Update role-related API handlers in `server/handles/role.go` to enforce the new restriction. - Enhance comments in `internal/bootstrap/data/role.go` to clarify the significance of default roles. - Ensure consistent error responses for unauthorized role modifications across the application. * 🔄 **refactor(role): Enhance role permission handling** - Replaced `BasePaths` with `PermissionPaths` in `Role` struct for better permission granularity. - Introduced JSON serialization for `PermissionPaths` using `RawPermission` field in `Role` struct. - Implemented `BeforeSave` and `AfterFind` GORM hooks for handling `PermissionPaths` serialization. - Refactored permission calculation logic in `role_perm.go` to work with `PermissionPaths`. - Updated role creation logic to initialize `PermissionPaths` for `admin` and `guest` roles. - Removed deprecated `CheckPathLimit` method from `Role` struct. * fix(model/user/role): update permission settings for admin and role - Change `RawPermission` field in `role.go` to hide JSON representation - Update `Permission` field in `user.go` to `0xFFFF` for full access - Modify `PermissionScopes` in `role.go` to `0xFFFF` for enhanced permissions * 🔒 feat(role-permissions): Enhance role-based access control - Introduce `canReadPathByRole` function in `role_perm.go` to verify path access based on user roles - Modify `CanAccessWithRoles` to include role-based path read check - Add `RoleNames` and `Permissions` to `UserResp` struct in `auth.go` for enhanced user role and permission details - Implement role details aggregation in `auth.go` to populate `RoleNames` and `Permissions` - Update `User` struct in `user.go` to include `RolesDetail` for more detailed role information - Enhance middleware in `auth.go` to load and verify detailed role information for users - Move `guest` user initialization logic in `user.go` to improve code organization and avoid repetition * 🔒 fix(permissions): Add permission checks for archive operations - Add `MergeRolePermissions` and `HasPermission` checks to validate user access for reading archives - Ensure users have `PermReadArchives` before proceeding with `GetNearestMeta` in specific archive paths - Implement permission checks for decompress operations, requiring `PermDecompress` for source paths - Return `PermissionDenied` errors with 403 status if user lacks necessary permissions * 🔒 fix(server): Add permission check for offline download - Add permission merging logic for user roles - Check user has permission for offline download addition - Return error response with "permission denied" if check fails * ✨ feat(role-permission): Implement path-based role permission checks - Add `CheckPathLimitWithRoles` function to validate access based on `PermPathLimit` permission. - Integrate `CheckPathLimitWithRoles` in `offline_download` to enforce path-based access control. - Apply `CheckPathLimitWithRoles` across file system management operations (e.g., creation, movement, deletion). - Ensure `CheckPathLimitWithRoles` is invoked for batch operations and archive-related actions. - Update error handling to return `PermissionDenied` if the path validation fails. - Import `errs` package in `offline_download` for consistent error responses. * ✨ feat(role-permission): Implement path-based role permission checks - Add `CheckPathLimitWithRoles` function to validate access based on `PermPathLimit` permission. - Integrate `CheckPathLimitWithRoles` in `offline_download` to enforce path-based access control. - Apply `CheckPathLimitWithRoles` across file system management operations (e.g., creation, movement, deletion). - Ensure `CheckPathLimitWithRoles` is invoked for batch operations and archive-related actions. - Update error handling to return `PermissionDenied` if the path validation fails. - Import `errs` package in `offline_download` for consistent error responses. * ♻️ refactor(access-control): Update access control logic to use role-based checks - Remove deprecated logic from `CanAccess` function in `check.go`, replacing it with `CanAccessWithRoles` for improved role-based access control. - Modify calls in `search.go` to use `CanAccessWithRoles` for more precise handling of permissions. - Update `fsread.go` to utilize `CanAccessWithRoles`, ensuring accurate access validation based on user roles. - Simplify import statements in `check.go` by removing unused packages to clean up the codebase. * ✨ feat(fs): Improve visibility logic for hidden files - Import `server/common` package to handle permissions more robustly - Update `whetherHide` function to use `MergeRolePermissions` for user-specific path permissions - Replace direct user checks with `HasPermission` for `PermSeeHides` - Enhance logic to ensure `nil` user cases are handled explicitly * 标签管理 * feat(db/auth/user): Enhance role handling and clean permission paths - Comment out role modification checks in `server/handles/user.go` to allow flexible role changes. - Improve permission path handling in `server/handles/auth.go` by normalizing and deduplicating paths. - Introduce `addedPaths` map in `CurrentUser` to prevent duplicate permissions. * feat(storage/db): Implement role permissions path prefix update - Add `UpdateRolePermissionsPathPrefix` function in `role.go` to update role permissions paths. - Modify `storage.go` to call the new function when the mount path is renamed. - Introduce path cleaning and prefix matching logic for accurate path updates. - Ensure roles are updated only if their permission scopes are modified. - Handle potential errors with informative messages during database operations. * feat(role-migration): Implement role conversion and introduce NEWGENERAL role - Add `NEWGENERAL` to the roles enumeration in `user.go` - Create new file `convert_role.go` for migrating legacy roles to new model - Implement `ConvertLegacyRoles` function to handle role conversion with permission scopes - Add `convert_role.go` patch to `all.go` under version `v3.46.0` * feat(role/auth): Add role retrieval by user ID and update path prefixes - Add `GetRolesByUserID` function for efficient role retrieval by user ID - Implement `UpdateUserBasePathPrefix` to update user base paths - Modify `UpdateRolePermissionsPathPrefix` to return modified role IDs - Update `auth.go` middleware to use the new role retrieval function - Refresh role and user caches upon path prefix updates to maintain consistency --------- Co-authored-by: Leslie-Xy <540049476@qq.com> --- drivers/alist_v3/driver.go | 2 +- drivers/alist_v3/types.go | 2 +- drivers/quqi/types.go | 2 +- internal/bootstrap/data/data.go | 1 + internal/bootstrap/data/dev.go | 2 +- internal/bootstrap/data/role.go | 52 ++++++ internal/bootstrap/data/user.go | 50 +++--- internal/bootstrap/patch/all.go | 7 + .../bootstrap/patch/v3_46_0/convert_role.go | 129 ++++++++++++++ internal/db/db.go | 2 +- internal/db/label.go | 79 +++++++++ internal/db/label_file_binding.go | 56 ++++++ internal/db/obj_file.go | 31 ++++ internal/db/role.go | 79 +++++++++ internal/db/user.go | 48 +++++- internal/errs/role.go | 7 + internal/fs/list.go | 10 +- internal/model/label.go | 12 ++ internal/model/label_file_binding.go | 11 ++ internal/model/obj_file.go | 18 ++ internal/model/paths.go | 27 +++ internal/model/role.go | 52 ++++++ internal/model/roles.go | 36 ++++ internal/model/user.go | 38 +++-- internal/op/label.go | 24 +++ internal/op/label_file_binding.go | 159 ++++++++++++++++++ internal/op/role.go | 121 +++++++++++++ internal/op/storage.go | 15 ++ internal/op/user.go | 12 +- server/common/check.go | 33 +--- server/common/role_perm.go | 108 ++++++++++++ server/ftp.go | 4 +- server/ftp/fsmanage.go | 11 +- server/ftp/fsread.go | 6 +- server/ftp/fsup.go | 6 +- server/handles/archive.go | 43 +++-- server/handles/auth.go | 31 +++- server/handles/fsbatch.go | 44 +++-- server/handles/fsmanage.go | 80 ++++++--- server/handles/fsread.go | 66 +++++--- server/handles/label.go | 99 +++++++++++ server/handles/label_file_binding.go | 103 ++++++++++++ server/handles/ldap_login.go | 2 +- server/handles/offline_download.go | 14 +- server/handles/role.go | 101 +++++++++++ server/handles/search.go | 2 +- server/handles/ssologin.go | 2 +- server/handles/task.go | 4 +- server/handles/user.go | 8 +- server/middlewares/auth.go | 23 +++ server/middlewares/fsup.go | 4 +- server/router.go | 20 +++ server/sftp.go | 10 +- server/webdav.go | 31 +++- server/webdav/file.go | 6 +- 55 files changed, 1762 insertions(+), 183 deletions(-) create mode 100644 internal/bootstrap/data/role.go create mode 100644 internal/bootstrap/patch/v3_46_0/convert_role.go create mode 100644 internal/db/label.go create mode 100644 internal/db/label_file_binding.go create mode 100644 internal/db/obj_file.go create mode 100644 internal/db/role.go create mode 100644 internal/errs/role.go create mode 100644 internal/model/label.go create mode 100644 internal/model/label_file_binding.go create mode 100644 internal/model/obj_file.go create mode 100644 internal/model/paths.go create mode 100644 internal/model/role.go create mode 100644 internal/model/roles.go create mode 100644 internal/op/label.go create mode 100644 internal/op/label_file_binding.go create mode 100644 internal/op/role.go create mode 100644 server/common/role_perm.go create mode 100644 server/handles/label.go create mode 100644 server/handles/label_file_binding.go create mode 100644 server/handles/role.go diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index ac7e16a1d16..56f9c01e124 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -56,7 +56,7 @@ func (d *AListV3) Init(ctx context.Context) error { if err != nil { return err } - if resp.Data.Role == model.GUEST { + if utils.SliceContains(resp.Data.Role, model.GUEST) { u := d.Address + "/api/public/settings" res, err := base.RestyClient.R().Get(u) if err != nil { diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go index 1ae7926e078..83ecde8be17 100644 --- a/drivers/alist_v3/types.go +++ b/drivers/alist_v3/types.go @@ -76,7 +76,7 @@ type MeResp struct { Username string `json:"username"` Password string `json:"password"` BasePath string `json:"base_path"` - Role int `json:"role"` + Role []int `json:"role"` Disabled bool `json:"disabled"` Permission int `json:"permission"` SsoId string `json:"sso_id"` diff --git a/drivers/quqi/types.go b/drivers/quqi/types.go index 32557361532..cade93de885 100644 --- a/drivers/quqi/types.go +++ b/drivers/quqi/types.go @@ -83,7 +83,7 @@ type Group struct { Type int `json:"type"` Name string `json:"name"` IsAdministrator int `json:"is_administrator"` - Role int `json:"role"` + Role []int `json:"role"` Avatar string `json:"avatar_url"` IsStick int `json:"is_stick"` Nickname string `json:"nickname"` diff --git a/internal/bootstrap/data/data.go b/internal/bootstrap/data/data.go index c2170d2f479..1f0a5909a58 100644 --- a/internal/bootstrap/data/data.go +++ b/internal/bootstrap/data/data.go @@ -3,6 +3,7 @@ package data import "github.com/alist-org/alist/v3/cmd/flags" func InitData() { + initRoles() initUser() initSettings() initTasks() diff --git a/internal/bootstrap/data/dev.go b/internal/bootstrap/data/dev.go index f6296c9e96a..74097dbd8b7 100644 --- a/internal/bootstrap/data/dev.go +++ b/internal/bootstrap/data/dev.go @@ -26,7 +26,7 @@ func initDevData() { Username: "Noah", Password: "hsu", BasePath: "/data", - Role: 0, + Role: nil, Permission: 512, }) if err != nil { diff --git a/internal/bootstrap/data/role.go b/internal/bootstrap/data/role.go new file mode 100644 index 00000000000..a82fa2afcc3 --- /dev/null +++ b/internal/bootstrap/data/role.go @@ -0,0 +1,52 @@ +package data + +// initRoles creates the default admin and guest roles if missing. +// These roles are essential and must not be modified or removed. + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +func initRoles() { + guestRole, err := op.GetRoleByName("guest") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + guestRole = &model.Role{ + ID: uint(model.GUEST), + Name: "guest", + Description: "Guest", + PermissionScopes: []model.PermissionEntry{ + {Path: "/", Permission: 0}, + }, + } + if err := op.CreateRole(guestRole); err != nil { + utils.Log.Fatalf("[init role] Failed to create guest role: %v", err) + } + } else { + utils.Log.Fatalf("[init role] Failed to get guest role: %v", err) + } + } + + _, err = op.GetRoleByName("admin") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + adminRole := &model.Role{ + ID: uint(model.ADMIN), + Name: "admin", + Description: "Administrator", + PermissionScopes: []model.PermissionEntry{ + {Path: "/", Permission: 0xFFFF}, + }, + } + if err := op.CreateRole(adminRole); err != nil { + utils.Log.Fatalf("[init role] Failed to create admin role: %v", err) + } + } else { + utils.Log.Fatalf("[init role] Failed to get admin role: %v", err) + } + } +} diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 9c3f8962ad3..6851118668b 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -1,10 +1,10 @@ package data import ( + "github.com/alist-org/alist/v3/internal/db" "os" "github.com/alist-org/alist/v3/cmd/flags" - "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -14,6 +14,28 @@ import ( ) func initUser() { + guest, err := op.GetGuest() + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + salt := random.String(16) + guestRole, _ := op.GetRoleByName("guest") + guest = &model.User{ + Username: "guest", + PwdHash: model.TwoHashPwd("guest", salt), + Salt: salt, + Role: model.Roles{int(guestRole.ID)}, + BasePath: "/", + Permission: 0, + Disabled: true, + Authn: "[]", + } + if err := db.CreateUser(guest); err != nil { + utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) + } + } else { + utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) + } + } admin, err := op.GetAdmin() adminPassword := random.String(8) envpass := os.Getenv("ALIST_ADMIN_PASSWORD") @@ -25,15 +47,16 @@ func initUser() { if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) + adminRole, _ := op.GetRoleByName("admin") admin = &model.User{ Username: "admin", Salt: salt, PwdHash: model.TwoHashPwd(adminPassword, salt), - Role: model.ADMIN, + Role: model.Roles{int(adminRole.ID)}, BasePath: "/", Authn: "[]", // 0(can see hidden) - 7(can remove) & 12(can read archives) - 13(can decompress archives) - Permission: 0x30FF, + Permission: 0xFFFF, } if err := op.CreateUser(admin); err != nil { panic(err) @@ -44,25 +67,4 @@ func initUser() { utils.Log.Fatalf("[init user] Failed to get admin user: %v", err) } } - guest, err := op.GetGuest() - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - salt := random.String(16) - guest = &model.User{ - Username: "guest", - PwdHash: model.TwoHashPwd("guest", salt), - Salt: salt, - Role: model.GUEST, - BasePath: "/", - Permission: 0, - Disabled: true, - Authn: "[]", - } - if err := db.CreateUser(guest); err != nil { - utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) - } - } else { - utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) - } - } } diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go index b363d12981d..eb679147ec9 100644 --- a/internal/bootstrap/patch/all.go +++ b/internal/bootstrap/patch/all.go @@ -4,6 +4,7 @@ import ( "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_24_0" "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_32_0" "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_41_0" + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_46_0" ) type VersionPatches struct { @@ -32,4 +33,10 @@ var UpgradePatches = []VersionPatches{ v3_41_0.GrantAdminPermissions, }, }, + { + Version: "v3.46.0", + Patches: []func(){ + v3_46_0.ConvertLegacyRoles, + }, + }, } diff --git a/internal/bootstrap/patch/v3_46_0/convert_role.go b/internal/bootstrap/patch/v3_46_0/convert_role.go new file mode 100644 index 00000000000..43799485c12 --- /dev/null +++ b/internal/bootstrap/patch/v3_46_0/convert_role.go @@ -0,0 +1,129 @@ +package v3_46_0 + +import ( + "errors" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "gorm.io/gorm" +) + +// ConvertLegacyRoles migrates old integer role values to a new role model with permission scopes. +func ConvertLegacyRoles() { + guestRole, err := op.GetRoleByName("guest") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + guestRole = &model.Role{ + ID: uint(model.GUEST), + Name: "guest", + Description: "Guest", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0, + }, + }, + } + if err = op.CreateRole(guestRole); err != nil { + utils.Log.Errorf("[convert roles] failed to create guest role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed to get guest role: %v", err) + return + } + } + + adminRole, err := op.GetRoleByName("admin") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + adminRole = &model.Role{ + ID: uint(model.ADMIN), + Name: "admin", + Description: "Administrator", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0x33FF, + }, + }, + } + if err = op.CreateRole(adminRole); err != nil { + utils.Log.Errorf("[convert roles] failed to create admin role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed to get admin role: %v", err) + return + } + } + + generalRole, err := op.GetRoleByName("general") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + generalRole = &model.Role{ + ID: uint(model.NEWGENERAL), + Name: "general", + Description: "General User", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0, + }, + }, + } + if err = op.CreateRole(generalRole); err != nil { + utils.Log.Errorf("[convert roles] failed create general role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed get general role: %v", err) + return + } + } + + users, _, err := op.GetUsers(1, -1) + if err != nil { + utils.Log.Errorf("[convert roles] failed to get users: %v", err) + return + } + + for i := range users { + user := users[i] + if user.Role == nil { + continue + } + changed := false + var roles model.Roles + for _, r := range user.Role { + switch r { + case model.ADMIN: + roles = append(roles, int(adminRole.ID)) + if int(adminRole.ID) != r { + changed = true + } + case model.GUEST: + roles = append(roles, int(guestRole.ID)) + if int(guestRole.ID) != r { + changed = true + } + case model.GENERAL: + roles = append(roles, int(generalRole.ID)) + if int(generalRole.ID) != r { + changed = true + } + default: + roles = append(roles, r) + } + } + if changed { + user.Role = roles + if err := db.UpdateUser(&user); err != nil { + utils.Log.Errorf("[convert roles] failed to update user %s: %v", user.Username, err) + } + } + } + + utils.Log.Infof("[convert roles] completed role conversion for %d users", len(users)) +} diff --git a/internal/db/db.go b/internal/db/db.go index 2cd18050da9..c6491dc984f 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinDing), new(model.ObjFile)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/label.go b/internal/db/label.go new file mode 100644 index 00000000000..fd9842d680c --- /dev/null +++ b/internal/db/label.go @@ -0,0 +1,79 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" + "time" +) + +// GetLabels Get all label from database order by id +func GetLabels(pageIndex, pageSize int) ([]model.Label, int64, error) { + labelDB := db.Model(&model.Label{}) + var count int64 + if err := labelDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get label count") + } + var labels []model.Label + if err := labelDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&labels).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return labels, count, nil +} + +// GetLabelById Get Label by id, used to update label usually +func GetLabelById(id uint) (*model.Label, error) { + var label model.Label + label.ID = id + if err := db.First(&label).Error; err != nil { + return nil, errors.WithStack(err) + } + return &label, nil +} + +// CreateLabel just insert label to database +func CreateLabel(label model.Label) (uint, error) { + label.CreateTime = time.Now() + err := errors.WithStack(db.Create(&label).Error) + if err != nil { + return label.ID, errors.WithMessage(err, "failed create label in database") + } + return label.ID, nil +} + +// UpdateLabel just update storage in database +func UpdateLabel(label *model.Label) (*model.Label, error) { + label.CreateTime = time.Now() + _, err := GetLabelById(label.ID) + if err != nil { + return nil, errors.WithMessage(err, "failed get old label") + } + err = errors.WithStack(db.Save(label).Error) + if err != nil { + return nil, errors.WithMessage(err, "failed create label in database") + } + return label, nil +} + +// DeleteLabelById just delete label from database by id +func DeleteLabelById(id uint) error { + return errors.WithStack(db.Delete(&model.Label{}, id).Error) +} + +// GetLabelByIds Get label from database order by ids +func GetLabelByIds(ids []uint) ([]model.Label, error) { + labelDB := db.Model(&model.Label{}) + var labels []model.Label + if err := labelDB.Where(ids).Find(&labels).Error; err != nil { + return nil, errors.WithStack(err) + } + return labels, nil +} + +// GetLabelByName Get Label by name +func GetLabelByName(name string) bool { + var label model.Label + result := db.Where("name = ?", name).First(&label) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} diff --git a/internal/db/label_file_binding.go b/internal/db/label_file_binding.go new file mode 100644 index 00000000000..ec722efb979 --- /dev/null +++ b/internal/db/label_file_binding.go @@ -0,0 +1,56 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" + "time" +) + +// GetLabelIds Get all label_ids from database order by file_name +func GetLabelIds(userId uint, fileName string) ([]uint, error) { + labelFileBinDingDB := db.Model(&model.LabelFileBinDing{}) + var labelIds []uint + if err := labelFileBinDingDB.Where("file_name = ?", fileName).Where("user_id = ?", userId).Pluck("label_id", &labelIds).Error; err != nil { + return nil, errors.WithStack(err) + } + return labelIds, nil +} + +func CreateLabelFileBinDing(fileName string, labelId, userId uint) error { + var labelFileBinDing model.LabelFileBinDing + labelFileBinDing.UserId = userId + labelFileBinDing.LabelId = labelId + labelFileBinDing.FileName = fileName + labelFileBinDing.CreateTime = time.Now() + err := errors.WithStack(db.Create(&labelFileBinDing).Error) + if err != nil { + return errors.WithMessage(err, "failed create label in database") + } + return nil +} + +// GetLabelFileBinDingByLabelIdExists Get Label by label_id, used to del label usually +func GetLabelFileBinDingByLabelIdExists(labelId, userId uint) bool { + var labelFileBinDing model.LabelFileBinDing + result := db.Where("label_id = ?", labelId).Where("user_id = ?", userId).First(&labelFileBinDing) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} + +// DelLabelFileBinDingByFileName used to del usually +func DelLabelFileBinDingByFileName(userId uint, fileName string) error { + return errors.WithStack(db.Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinDing{}).Error) +} + +// DelLabelFileBinDingById used to del usually +func DelLabelFileBinDingById(labelId, userId uint, fileName string) error { + return errors.WithStack(db.Where("label_id = ?", labelId).Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinDing{}).Error) +} + +func GetLabelFileBinDingByLabelId(labelIds []uint, userId uint) (result []model.LabelFileBinDing, err error) { + if err := db.Where("label_id in (?)", labelIds).Where("user_id = ?", userId).Find(&result).Error; err != nil { + return nil, errors.WithStack(err) + } + return result, nil +} diff --git a/internal/db/obj_file.go b/internal/db/obj_file.go new file mode 100644 index 00000000000..2bbce9e6dd6 --- /dev/null +++ b/internal/db/obj_file.go @@ -0,0 +1,31 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// GetFileByNameExists Get file by name +func GetFileByNameExists(name string) bool { + var label model.ObjFile + result := db.Where("name = ?", name).First(&label) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} + +// GetFileByName Get file by name +func GetFileByName(name string, userId uint) (objFile model.ObjFile, err error) { + if err = db.Where("name = ?", name).Where("user_id = ?", userId).First(&objFile).Error; err != nil { + return objFile, errors.WithStack(err) + } + return objFile, nil +} + +func CreateObjFile(obj model.ObjFile) error { + err := errors.WithStack(db.Create(&obj).Error) + if err != nil { + return errors.WithMessage(err, "failed create file in database") + } + return nil +} diff --git a/internal/db/role.go b/internal/db/role.go new file mode 100644 index 00000000000..e6d0d9568b4 --- /dev/null +++ b/internal/db/role.go @@ -0,0 +1,79 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "path" + "strings" +) + +func GetRole(id uint) (*model.Role, error) { + var r model.Role + if err := db.First(&r, id).Error; err != nil { + return nil, errors.Wrapf(err, "failed get role") + } + return &r, nil +} + +func GetRoleByName(name string) (*model.Role, error) { + r := model.Role{Name: name} + if err := db.Where(r).First(&r).Error; err != nil { + return nil, errors.Wrapf(err, "failed get role") + } + return &r, nil +} + +func GetRoles(pageIndex, pageSize int) (roles []model.Role, count int64, err error) { + roleDB := db.Model(&model.Role{}) + if err = roleDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get roles count") + } + if err = roleDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&roles).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get find roles") + } + return roles, count, nil +} + +func CreateRole(r *model.Role) error { + return errors.WithStack(db.Create(r).Error) +} + +func UpdateRole(r *model.Role) error { + return errors.WithStack(db.Save(r).Error) +} + +func DeleteRole(id uint) error { + return errors.WithStack(db.Delete(&model.Role{}, id).Error) +} + +func UpdateRolePermissionsPathPrefix(oldPath, newPath string) ([]uint, error) { + var roles []model.Role + var modifiedRoleIDs []uint + + if err := db.Find(&roles).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load roles") + } + + for _, role := range roles { + updated := false + for i, entry := range role.PermissionScopes { + entryPath := path.Clean(entry.Path) + oldPathClean := path.Clean(oldPath) + + if entryPath == oldPathClean { + role.PermissionScopes[i].Path = newPath + updated = true + } else if strings.HasPrefix(entryPath, oldPathClean+"/") { + role.PermissionScopes[i].Path = newPath + entryPath[len(oldPathClean):] + updated = true + } + } + if updated { + if err := UpdateRole(&role); err != nil { + return nil, errors.WithMessagef(err, "failed to update role ID %d", role.ID) + } + modifiedRoleIDs = append(modifiedRoleIDs, role.ID) + } + } + return modifiedRoleIDs, nil +} diff --git a/internal/db/user.go b/internal/db/user.go index 8c9641b2c55..f2b6635a983 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -2,19 +2,26 @@ package db import ( "encoding/base64" - "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-webauthn/webauthn/webauthn" "github.com/pkg/errors" + "gorm.io/gorm" + "path" + "strings" ) func GetUserByRole(role int) (*model.User, error) { - user := model.User{Role: role} - if err := db.Where(user).Take(&user).Error; err != nil { + var users []model.User + if err := db.Find(&users).Error; err != nil { return nil, err } - return &user, nil + for i := range users { + if users[i].Role.Contains(role) { + return &users[i], nil + } + } + return nil, gorm.ErrRecordNotFound } func GetUserByName(username string) (*model.User, error) { @@ -100,3 +107,36 @@ func RemoveAuthn(u *model.User, id string) error { } return UpdateAuthn(u.ID, string(res)) } + +func UpdateUserBasePathPrefix(oldPath, newPath string) ([]string, error) { + var users []model.User + var modifiedUsernames []string + + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load users") + } + + oldPathClean := path.Clean(oldPath) + + for _, user := range users { + basePath := path.Clean(user.BasePath) + updated := false + + if basePath == oldPathClean { + user.BasePath = newPath + updated = true + } else if strings.HasPrefix(basePath, oldPathClean+"/") { + user.BasePath = newPath + basePath[len(oldPathClean):] + updated = true + } + + if updated { + if err := UpdateUser(&user); err != nil { + return nil, errors.WithMessagef(err, "failed to update user ID %d", user.ID) + } + modifiedUsernames = append(modifiedUsernames, user.Username) + } + } + + return modifiedUsernames, nil +} diff --git a/internal/errs/role.go b/internal/errs/role.go new file mode 100644 index 00000000000..fbd67404822 --- /dev/null +++ b/internal/errs/role.go @@ -0,0 +1,7 @@ +package errs + +import "errors" + +var ( + ErrChangeDefaultRole = errors.New("cannot modify admin or guest role") +) diff --git a/internal/fs/list.go b/internal/fs/list.go index d4f59cb829f..927b6ead1a4 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -6,6 +6,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -45,8 +46,13 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) } func whetherHide(user *model.User, meta *model.Meta, path string) bool { - // if is admin, don't hide - if user == nil || user.CanSeeHides() { + // if user is nil, don't hide + if user == nil { + return false + } + perm := common.MergeRolePermissions(user, path) + // if user has see-hides permission, don't hide + if common.HasPermission(perm, common.PermSeeHides) { return false } // if meta is nil, don't hide diff --git a/internal/model/label.go b/internal/model/label.go new file mode 100644 index 00000000000..b397542f5c3 --- /dev/null +++ b/internal/model/label.go @@ -0,0 +1,12 @@ +package model + +import "time" + +type Label struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + Type int `json:"type"` // use to type + Name string `json:"name"` // use to name + Description string `json:"description"` // use to description + BgColor string `json:"bg_color"` // use to bg_color + CreateTime time.Time `json:"create_time"` +} diff --git a/internal/model/label_file_binding.go b/internal/model/label_file_binding.go new file mode 100644 index 00000000000..3f9ea3b2271 --- /dev/null +++ b/internal/model/label_file_binding.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type LabelFileBinDing struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + UserId uint `json:"user_id"` // use to user_id + LabelId uint `json:"label_id"` // use to label_id + FileName string `json:"file_name"` // use to file_name + CreateTime time.Time `json:"create_time"` +} diff --git a/internal/model/obj_file.go b/internal/model/obj_file.go new file mode 100644 index 00000000000..0fccd6b5cba --- /dev/null +++ b/internal/model/obj_file.go @@ -0,0 +1,18 @@ +package model + +import "time" + +type ObjFile struct { + Id string `json:"id"` + UserId uint `json:"user_id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` +} diff --git a/internal/model/paths.go b/internal/model/paths.go new file mode 100644 index 00000000000..8403de8e6a2 --- /dev/null +++ b/internal/model/paths.go @@ -0,0 +1,27 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Paths []string + +func (p Paths) Value() (driver.Value, error) { + return json.Marshal([]string(p)) +} + +func (p *Paths) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, (*[]string)(p)) + case string: + return json.Unmarshal([]byte(v), (*[]string)(p)) + case nil: + *p = nil + return nil + default: + return fmt.Errorf("cannot scan %T", value) + } +} diff --git a/internal/model/role.go b/internal/model/role.go new file mode 100644 index 00000000000..ecc9aee29d1 --- /dev/null +++ b/internal/model/role.go @@ -0,0 +1,52 @@ +package model + +import ( + "encoding/json" + + "gorm.io/gorm" +) + +// PermissionEntry defines permission bitmask for a specific path. +type PermissionEntry struct { + Path string `json:"path"` // path prefix, e.g. "/admin" + Permission int32 `json:"permission"` // bitmask permissions +} + +// Role represents a permission template which can be bound to users. +type Role struct { + ID uint `json:"id" gorm:"primaryKey"` + Name string `json:"name" gorm:"unique" binding:"required"` + Description string `json:"description"` + // PermissionScopes stores structured permission list and is ignored by gorm. + PermissionScopes []PermissionEntry `json:"permission_scopes" gorm:"-"` + // RawPermission is the JSON representation of PermissionScopes stored in DB. + RawPermission string `json:"-" gorm:"type:text"` +} + +// BeforeSave GORM hook serializes PermissionScopes into RawPermission. +func (r *Role) BeforeSave(tx *gorm.DB) error { + if len(r.PermissionScopes) == 0 { + r.RawPermission = "" + return nil + } + bs, err := json.Marshal(r.PermissionScopes) + if err != nil { + return err + } + r.RawPermission = string(bs) + return nil +} + +// AfterFind GORM hook deserializes RawPermission into PermissionScopes. +func (r *Role) AfterFind(tx *gorm.DB) error { + if r.RawPermission == "" { + r.PermissionScopes = nil + return nil + } + var scopes []PermissionEntry + if err := json.Unmarshal([]byte(r.RawPermission), &scopes); err != nil { + return err + } + r.PermissionScopes = scopes + return nil +} diff --git a/internal/model/roles.go b/internal/model/roles.go new file mode 100644 index 00000000000..eb626cb93f7 --- /dev/null +++ b/internal/model/roles.go @@ -0,0 +1,36 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Roles []int + +func (r Roles) Value() (driver.Value, error) { + return json.Marshal([]int(r)) +} + +func (r *Roles) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, (*[]int)(r)) + case string: + return json.Unmarshal([]byte(v), (*[]int)(r)) + case nil: + *r = nil + return nil + default: + return fmt.Errorf("cannot scan %T", value) + } +} + +func (r Roles) Contains(role int) bool { + for _, v := range r { + if v == role { + return true + } + } + return false +} diff --git a/internal/model/user.go b/internal/model/user.go index 0f7d3af5064..0b9e576a484 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -17,20 +17,22 @@ const ( GENERAL = iota GUEST // only one exists ADMIN + NEWGENERAL ) const StaticHashSalt = "https://github.com/alist-org/alist" type User struct { - ID uint `json:"id" gorm:"primaryKey"` // unique key - Username string `json:"username" gorm:"unique" binding:"required"` // username - PwdHash string `json:"-"` // password hash - PwdTS int64 `json:"-"` // password timestamp - Salt string `json:"-"` // unique salt - Password string `json:"password"` // password - BasePath string `json:"base_path"` // base path - Role int `json:"role"` // user's role - Disabled bool `json:"disabled"` + ID uint `json:"id" gorm:"primaryKey"` // unique key + Username string `json:"username" gorm:"unique" binding:"required"` // username + PwdHash string `json:"-"` // password hash + PwdTS int64 `json:"-"` // password timestamp + Salt string `json:"-"` // unique salt + Password string `json:"password"` // password + BasePath string `json:"base_path"` // base path + Role Roles `json:"role" gorm:"type:text"` // user's roles + RolesDetail []Role `json:"-" gorm:"-"` + Disabled bool `json:"disabled"` // Determine permissions by bit // 0: can see hidden files // 1: can access without password @@ -46,6 +48,7 @@ type User struct { // 11: ftp/sftp write // 12: can read archives // 13: can decompress archives + // 14: check path limit Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -53,11 +56,11 @@ type User struct { } func (u *User) IsGuest() bool { - return u.Role == GUEST + return u.Role.Contains(GUEST) } func (u *User) IsAdmin() bool { - return u.Role == ADMIN + return u.Role.Contains(ADMIN) } func (u *User) ValidateRawPassword(password string) error { @@ -137,8 +140,19 @@ func (u *User) CanDecompress() bool { return (u.Permission>>13)&1 == 1 } +func (u *User) CheckPathLimit() bool { + return (u.Permission>>14)&1 == 1 +} + func (u *User) JoinPath(reqPath string) (string, error) { - return utils.JoinBasePath(u.BasePath, reqPath) + path, err := utils.JoinBasePath(u.BasePath, reqPath) + if err != nil { + return "", err + } + if u.CheckPathLimit() && !utils.IsSubPath(u.BasePath, path) { + return "", errs.PermissionDenied + } + return path, nil } func StaticHash(password string) string { diff --git a/internal/op/label.go b/internal/op/label.go new file mode 100644 index 00000000000..7e913edfa8d --- /dev/null +++ b/internal/op/label.go @@ -0,0 +1,24 @@ +package op + +import ( + "context" + "github.com/alist-org/alist/v3/internal/db" + "github.com/pkg/errors" +) + +func DeleteLabelById(ctx context.Context, id, userId uint) error { + _, err := db.GetLabelById(id) + if err != nil { + return errors.WithMessage(err, "failed get label") + } + + if db.GetLabelFileBinDingByLabelIdExists(id, userId) { + return errors.New("label have binding relationships") + } + + // delete the label in the database + if err := db.DeleteLabelById(id); err != nil { + return errors.WithMessage(err, "failed delete label in database") + } + return nil +} diff --git a/internal/op/label_file_binding.go b/internal/op/label_file_binding.go new file mode 100644 index 00000000000..79137ed38af --- /dev/null +++ b/internal/op/label_file_binding.go @@ -0,0 +1,159 @@ +package op + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "strconv" + "strings" + "time" +) + +type CreateLabelFileBinDingReq struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + LabelIds string `json:"label_ids"` +} + +type ObjLabelResp struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + LabelList []model.Label `json:"label_list"` +} + +func GetLabelByFileName(userId uint, fileName string) ([]model.Label, error) { + labelIds, err := db.GetLabelIds(userId, fileName) + if err != nil { + return nil, errors.WithMessage(err, "failed get label_file_binding") + } + var labels []model.Label + if len(labelIds) > 0 { + if labels, err = db.GetLabelByIds(labelIds); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + } + return labels, nil +} + +func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { + if err := db.DelLabelFileBinDingByFileName(userId, req.Name); err != nil { + return errors.WithMessage(err, "failed del label_file_bin_ding in database") + } + if req.LabelIds == "" { + return nil + } + labelMap := strings.Split(req.LabelIds, ",") + for _, value := range labelMap { + labelId, err := strconv.ParseUint(value, 10, 64) + if err != nil { + return fmt.Errorf("invalid label ID '%s': %v", value, err) + } + if err = db.CreateLabelFileBinDing(req.Name, uint(labelId), userId); err != nil { + return errors.WithMessage(err, "failed labels in database") + } + } + if !db.GetFileByNameExists(req.Name) { + objFile := model.ObjFile{ + Id: req.Id, + UserId: userId, + Path: req.Path, + Name: req.Name, + Size: req.Size, + IsDir: req.IsDir, + Modified: req.Modified, + Created: req.Created, + Sign: req.Sign, + Thumb: req.Thumb, + Type: req.Type, + HashInfoStr: req.HashInfoStr, + } + err := db.CreateObjFile(objFile) + if err != nil { + return errors.WithMessage(err, "failed file in database") + } + } + return nil +} + +func GetFileByLabel(userId uint, labelId string) (result []ObjLabelResp, err error) { + labelMap := strings.Split(labelId, ",") + var labelIds []uint + var labelsFile []model.LabelFileBinDing + var labels []model.Label + var labelsFileMap = make(map[string][]model.Label) + var labelsMap = make(map[uint]model.Label) + if labelIds, err = StringSliceToUintSlice(labelMap); err != nil { + return nil, errors.WithMessage(err, "failed string to uint err") + } + //查询标签信息 + if labels, err = db.GetLabelByIds(labelIds); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + for _, val := range labels { + labelsMap[val.ID] = val + } + //查询标签对应文件名列表 + if labelsFile, err = db.GetLabelFileBinDingByLabelId(labelIds, userId); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + for _, value := range labelsFile { + var labelTemp model.Label + labelTemp = labelsMap[value.LabelId] + labelsFileMap[value.FileName] = append(labelsFileMap[value.FileName], labelTemp) + } + for index, v := range labelsFileMap { + objFile, err := db.GetFileByName(index, userId) + if err != nil { + return nil, errors.WithMessage(err, "failed GetFileByName in database") + } + objLabel := ObjLabelResp{ + Id: objFile.Id, + Path: objFile.Path, + Name: objFile.Name, + Size: objFile.Size, + IsDir: objFile.IsDir, + Modified: objFile.Modified, + Created: objFile.Created, + Sign: objFile.Sign, + Thumb: objFile.Thumb, + Type: objFile.Type, + HashInfoStr: objFile.HashInfoStr, + LabelList: v, + } + result = append(result, objLabel) + } + return result, nil +} + +func StringSliceToUintSlice(strSlice []string) ([]uint, error) { + uintSlice := make([]uint, len(strSlice)) + for i, str := range strSlice { + // 使用strconv.ParseUint将字符串转换为uint64 + uint64Value, err := strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, err // 如果转换失败，返回错误 + } + // 将uint64值转换为uint（注意：这里可能存在精度损失，如果uint64值超出了uint的范围） + uintSlice[i] = uint(uint64Value) + } + return uintSlice, nil +} diff --git a/internal/op/role.go b/internal/op/role.go new file mode 100644 index 00000000000..64502f98df4 --- /dev/null +++ b/internal/op/role.go @@ -0,0 +1,121 @@ +package op + +import ( + "fmt" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/singleflight" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var roleCache = cache.NewMemCache[*model.Role](cache.WithShards[*model.Role](2)) +var roleG singleflight.Group[*model.Role] + +func GetRole(id uint) (*model.Role, error) { + key := fmt.Sprint(id) + if r, ok := roleCache.Get(key); ok { + return r, nil + } + r, err, _ := roleG.Do(key, func() (*model.Role, error) { + _r, err := db.GetRole(id) + if err != nil { + return nil, err + } + roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + return r, err +} + +func GetRoleByName(name string) (*model.Role, error) { + if r, ok := roleCache.Get(name); ok { + return r, nil + } + r, err, _ := roleG.Do(name, func() (*model.Role, error) { + _r, err := db.GetRoleByName(name) + if err != nil { + return nil, err + } + roleCache.Set(name, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + return r, err +} + +func GetRolesByUserID(userID uint) ([]model.Role, error) { + user, err := GetUserById(userID) + if err != nil { + return nil, err + } + + var roles []model.Role + for _, roleID := range user.Role { + key := fmt.Sprint(roleID) + + if r, ok := roleCache.Get(key); ok { + roles = append(roles, *r) + continue + } + + r, err, _ := roleG.Do(key, func() (*model.Role, error) { + _r, err := db.GetRole(uint(roleID)) + if err != nil { + return nil, err + } + roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + if err != nil { + return nil, err + } + roles = append(roles, *r) + } + + return roles, nil +} + +func GetRoles(pageIndex, pageSize int) ([]model.Role, int64, error) { + return db.GetRoles(pageIndex, pageSize) +} + +func CreateRole(r *model.Role) error { + for i := range r.PermissionScopes { + r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) + } + roleCache.Del(fmt.Sprint(r.ID)) + roleCache.Del(r.Name) + return db.CreateRole(r) +} + +func UpdateRole(r *model.Role) error { + old, err := db.GetRole(r.ID) + if err != nil { + return err + } + if old.Name == "admin" || old.Name == "guest" { + return errs.ErrChangeDefaultRole + } + for i := range r.PermissionScopes { + r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) + } + roleCache.Del(fmt.Sprint(r.ID)) + roleCache.Del(r.Name) + return db.UpdateRole(r) +} + +func DeleteRole(id uint) error { + old, err := db.GetRole(id) + if err != nil { + return err + } + if old.Name == "admin" || old.Name == "guest" { + return errs.ErrChangeDefaultRole + } + roleCache.Del(fmt.Sprint(id)) + roleCache.Del(old.Name) + return db.DeleteRole(id) +} diff --git a/internal/op/storage.go b/internal/op/storage.go index f957f95b596..2ec68aae5e6 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -216,6 +216,21 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) + modifiedRoleIDs, err := db.UpdateRolePermissionsPathPrefix(oldStorage.MountPath, storage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed to update role permissions") + } + for _, id := range modifiedRoleIDs { + roleCache.Del(fmt.Sprint(id)) + } + + modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldStorage.MountPath, storage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed to update user base path") + } + for _, name := range modifiedUsernames { + userCache.Del(name) + } } if err != nil { return errors.WithMessage(err, "failed get storage driver") diff --git a/internal/op/user.go b/internal/op/user.go index 79e73db86ce..e775df63e93 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -18,7 +18,11 @@ var adminUser *model.User func GetAdmin() (*model.User, error) { if adminUser == nil { - user, err := db.GetUserByRole(model.ADMIN) + role, err := GetRoleByName("admin") + if err != nil { + return nil, err + } + user, err := db.GetUserByRole(int(role.ID)) if err != nil { return nil, err } @@ -29,7 +33,11 @@ func GetAdmin() (*model.User, error) { func GetGuest() (*model.User, error) { if guestUser == nil { - user, err := db.GetUserByRole(model.GUEST) + role, err := GetRoleByName("guest") + if err != nil { + return nil, err + } + user, err := db.GetUserByRole(int(role.ID)) if err != nil { return nil, err } diff --git a/server/common/check.go b/server/common/check.go index 78051f4ee1e..34aaa41d93a 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -1,15 +1,11 @@ package common import ( - "path" - "strings" - "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" - "github.com/dlclark/regexp2" ) func IsStorageSignEnabled(rawPath string) bool { @@ -32,30 +28,11 @@ func IsApply(metaPath, reqPath string, applySub bool) bool { } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { - // if the reqPath is in hide (only can check the nearest meta) and user can't see hides, can't access - if meta != nil && !user.CanSeeHides() && meta.Hide != "" && - IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path - for _, hide := range strings.Split(meta.Hide, "\n") { - re := regexp2.MustCompile(hide, regexp2.None) - if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { - return false - } - } - } - // if is not guest and can access without password - if user.CanAccessWithoutPassword() { - return true - } - // if meta is nil or password is empty, can access - if meta == nil || meta.Password == "" { - return true - } - // if meta doesn't apply to sub_folder, can access - if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { - return true - } - // validate password - return meta.Password == password + // Deprecated: CanAccess is kept for backward compatibility. + // The logic has been moved to CanAccessWithRoles which performs the + // necessary checks based on role permissions. This wrapper ensures + // older calls still work without relying on user permission bits. + return CanAccessWithRoles(user, meta, reqPath, password) } // ShouldProxy TODO need optimize diff --git a/server/common/role_perm.go b/server/common/role_perm.go new file mode 100644 index 00000000000..1e539d966c5 --- /dev/null +++ b/server/common/role_perm.go @@ -0,0 +1,108 @@ +package common + +import ( + "path" + "strings" + + "github.com/dlclark/regexp2" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + PermSeeHides = iota + PermAccessWithoutPassword + PermAddOfflineDownload + PermWrite + PermRename + PermMove + PermCopy + PermRemove + PermWebdavRead + PermWebdavManage + PermFTPAccess + PermFTPManage + PermReadArchives + PermDecompress + PermPathLimit +) + +func HasPermission(perm int32, bit uint) bool { + return (perm>>bit)&1 == 1 +} + +func MergeRolePermissions(u *model.User, reqPath string) int32 { + if u == nil { + return 0 + } + var perm int32 + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(entry.Path, reqPath) { + perm |= entry.Permission + } + } + } + return perm +} + +func CanAccessWithRoles(u *model.User, meta *model.Meta, reqPath, password string) bool { + if !canReadPathByRole(u, reqPath) { + return false + } + perm := MergeRolePermissions(u, reqPath) + if meta != nil && !HasPermission(perm, PermSeeHides) && meta.Hide != "" && + IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { + for _, hide := range strings.Split(meta.Hide, "\n") { + re := regexp2.MustCompile(hide, regexp2.None) + if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { + return false + } + } + } + if HasPermission(perm, PermAccessWithoutPassword) { + return true + } + if meta == nil || meta.Password == "" { + return true + } + if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { + return true + } + return meta.Password == password +} + +func canReadPathByRole(u *model.User, reqPath string) bool { + if u == nil { + return false + } + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(entry.Path, reqPath) { + return true + } + } + } + return false +} + +// CheckPathLimitWithRoles checks whether the path is allowed when the user has +// the `PermPathLimit` permission for the target path. When the user does not +// have this permission, the check passes by default. +func CheckPathLimitWithRoles(u *model.User, reqPath string) bool { + perm := MergeRolePermissions(u, reqPath) + if HasPermission(perm, PermPathLimit) { + return canReadPathByRole(u, reqPath) + } + return true +} diff --git a/server/ftp.go b/server/ftp.go index 4d507b684b4..d41063731bf 100644 --- a/server/ftp.go +++ b/server/ftp.go @@ -11,6 +11,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/ftp" "math/rand" "net" @@ -130,7 +131,8 @@ func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) return nil, err } } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via FTP") } diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go index fb03c1b95cb..83e7bae1733 100644 --- a/server/ftp/fsmanage.go +++ b/server/ftp/fsmanage.go @@ -18,7 +18,8 @@ func Mkdir(ctx context.Context, path string) error { if err != nil { return err } - if !user.CanWrite() || !user.CanFTPManage() { + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) || !common.HasPermission(perm, common.PermFTPManage) { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -34,7 +35,8 @@ func Mkdir(ctx context.Context, path string) error { func Remove(ctx context.Context, path string) error { user := ctx.Value("user").(*model.User) - if !user.CanRemove() || !user.CanFTPManage() { + perm := common.MergeRolePermissions(user, path) + if !common.HasPermission(perm, common.PermRemove) || !common.HasPermission(perm, common.PermFTPManage) { return errs.PermissionDenied } reqPath, err := user.JoinPath(path) @@ -56,13 +58,14 @@ func Rename(ctx context.Context, oldPath, newPath string) error { } srcDir, srcBase := stdpath.Split(srcPath) dstDir, dstBase := stdpath.Split(dstPath) + permSrc := common.MergeRolePermissions(user, srcPath) if srcDir == dstDir { - if !user.CanRename() || !user.CanFTPManage() { + if !common.HasPermission(permSrc, common.PermRename) || !common.HasPermission(permSrc, common.PermFTPManage) { return errs.PermissionDenied } return fs.Rename(ctx, srcPath, dstBase) } else { - if !user.CanFTPManage() || !user.CanMove() || (srcBase != dstBase && !user.CanRename()) { + if !common.HasPermission(permSrc, common.PermFTPManage) || !common.HasPermission(permSrc, common.PermMove) || (srcBase != dstBase && !common.HasPermission(permSrc, common.PermRename)) { return errs.PermissionDenied } if err = fs.Move(ctx, srcPath, dstDir); err != nil { diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go index c051a19db21..2ba8cb82abc 100644 --- a/server/ftp/fsread.go +++ b/server/ftp/fsread.go @@ -30,7 +30,7 @@ func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownl } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } @@ -125,7 +125,7 @@ func Stat(ctx context.Context, path string) (os.FileInfo, error) { } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) @@ -148,7 +148,7 @@ func List(ctx context.Context, path string) ([]os.FileInfo, error) { } } ctx = context.WithValue(ctx, "meta", meta) - if !common.CanAccess(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { return nil, errs.PermissionDenied } objs, err := fs.List(ctx, reqPath, &fs.ListArgs{}) diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go index ee38b1bfb07..9610eea7588 100644 --- a/server/ftp/fsup.go +++ b/server/ftp/fsup.go @@ -35,8 +35,10 @@ func uploadAuth(ctx context.Context, path string) error { return err } } - if !(common.CanAccess(user, meta, path, ctx.Value("meta_pass").(string)) && - ((user.CanFTPManage() && user.CanWrite()) || common.CanWrite(meta, stdpath.Dir(path)))) { + perm := common.MergeRolePermissions(user, path) + if !(common.CanAccessWithRoles(user, meta, path, ctx.Value("meta_pass").(string)) && + ((common.HasPermission(perm, common.PermFTPManage) && common.HasPermission(perm, common.PermWrite)) || + common.CanWrite(meta, stdpath.Dir(path)))) { return errs.PermissionDenied } return nil diff --git a/server/handles/archive.go b/server/handles/archive.go index 550bc3cec9c..0bb8d94a728 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -78,15 +78,20 @@ func FsArchiveMeta(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanReadArchives() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermReadArchives) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -156,15 +161,20 @@ func FsArchiveList(c *gin.Context) { } req.Validate() user := c.MustGet("user").(*model.User) - if !user.CanReadArchives() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermReadArchives) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -242,10 +252,6 @@ func FsArchiveDecompress(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanDecompress() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcPaths := make([]string, 0, len(req.Name)) for _, name := range req.Name { srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) @@ -253,6 +259,10 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } srcPaths = append(srcPaths, srcPath) } dstDir, err := user.JoinPath(req.DstDir) @@ -260,8 +270,17 @@ func FsArchiveDecompress(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } tasks := make([]task.TaskExtensionInfo, 0, len(srcPaths)) for _, srcPath := range srcPaths { + perm := common.MergeRolePermissions(user, srcPath) + if !common.HasPermission(perm, common.PermDecompress) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } t, e := fs.ArchiveDecompress(c, srcPath, dstDir, model.ArchiveDecompressArgs{ ArchiveInnerArgs: model.ArchiveInnerArgs{ ArchiveArgs: model.ArchiveArgs{ diff --git a/server/handles/auth.go b/server/handles/auth.go index 7a2c0fb5376..96a9ba9e20d 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/base64" "image/png" + "path" + "strings" "time" "github.com/Xhofe/go-cache" @@ -89,13 +91,16 @@ func loginHash(c *gin.Context, req *LoginReq) { type UserResp struct { model.User - Otp bool `json:"otp"` + Otp bool `json:"otp"` + RoleNames []string `json:"role_names"` + Permissions []model.PermissionEntry `json:"permissions"` } // CurrentUser get current user by token // if token is empty, return guest user func CurrentUser(c *gin.Context) { user := c.MustGet("user").(*model.User) + userResp := UserResp{ User: *user, } @@ -103,6 +108,30 @@ func CurrentUser(c *gin.Context) { if userResp.OtpSecret != "" { userResp.Otp = true } + + var roleNames []string + permMap := map[string]int32{} + addedPaths := map[string]bool{} + + for _, role := range user.RolesDetail { + roleNames = append(roleNames, role.Name) + for _, entry := range role.PermissionScopes { + cleanPath := path.Clean("/" + strings.TrimPrefix(entry.Path, "/")) + permMap[cleanPath] |= entry.Permission + } + } + userResp.RoleNames = roleNames + + for fullPath, perm := range permMap { + if !addedPaths[fullPath] { + userResp.Permissions = append(userResp.Permissions, model.PermissionEntry{ + Path: fullPath, + Permission: perm, + }) + addedPaths[fullPath] = true + } + } + common.SuccessResp(c, userResp) } diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 3841bff5a34..7ff07c6df28 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -29,20 +29,29 @@ func FsRecursiveMove(c *gin.Context) { } user := c.MustGet("user").(*model.User) - if !user.CanMove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermMove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(srcDir) if err != nil { @@ -149,16 +158,20 @@ func FsBatchRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } - reqPath, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(reqPath) if err != nil { @@ -194,14 +207,19 @@ func FsRegexRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { + reqPath, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, reqPath) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - reqPath, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) return } diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index c527464e2e4..87be6e4106b 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -35,7 +35,12 @@ func FsMkdir(c *gin.Context) { common.ErrorResp(c, err, 403) return } - if !user.CanWrite() { + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -73,20 +78,29 @@ func FsMove(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanMove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + permMove := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(permMove, common.PermMove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { for _, name := range req.Names { if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { @@ -116,20 +130,29 @@ func FsCopy(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanCopy() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermCopy) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { for _, name := range req.Names { if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { @@ -167,15 +190,20 @@ func FsRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } if !req.Overwrite { dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) if dstPath != reqPath { @@ -208,15 +236,20 @@ func FsRemove(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRemove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqDir, err := user.JoinPath(req.Dir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqDir) + if !common.HasPermission(perm, common.PermRemove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } for _, name := range req.Names { err := fs.Remove(c, stdpath.Join(reqDir, name)) if err != nil { @@ -240,15 +273,20 @@ func FsRemoveEmptyDirectory(c *gin.Context) { } user := c.MustGet("user").(*model.User) - if !user.CanRemove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermRemove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(srcDir) if err != nil { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 73bde23b6de..b49f0b646b9 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -48,12 +48,28 @@ type ObjResp struct { } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjLabelResp `json:"content"` + Total int64 `json:"total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` +} + +type ObjLabelResp struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + LabelList []model.Label `json:"label_list"` } func FsList(c *gin.Context) { @@ -77,11 +93,12 @@ func FsList(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } - if !user.CanWrite() && !common.CanWrite(meta, reqPath) && req.Refresh { + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) && !common.CanWrite(meta, reqPath) && req.Refresh { common.ErrorStrResp(c, "Refresh without permission", 403) return } @@ -97,11 +114,11 @@ func FsList(c *gin.Context) { provider = storage.GetStorage().Driver } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath), user.ID), Total: int64(total), Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), + Write: common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, reqPath), Provider: provider, }) } @@ -135,7 +152,7 @@ func FsDirs(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } @@ -207,11 +224,15 @@ func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { return total, objs[start:end] } -func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { - var resp []ObjResp +func toObjsResp(objs []model.Obj, parent string, encrypt bool, userId uint) []ObjLabelResp { + var resp []ObjLabelResp for _, obj := range objs { + var labels []model.Label + if obj.IsDir() == false { + labels, _ = op.GetLabelByFileName(userId, obj.GetName()) + } thumb, _ := model.GetThumb(obj) - resp = append(resp, ObjResp{ + resp = append(resp, ObjLabelResp{ Id: obj.GetID(), Path: obj.GetPath(), Name: obj.GetName(), @@ -224,6 +245,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { Sign: common.Sign(obj, parent, encrypt), Thumb: thumb, Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + LabelList: labels, }) } return resp @@ -236,11 +258,11 @@ type FsGetReq struct { type FsGetResp struct { ObjResp - RawURL string `json:"raw_url"` - Readme string `json:"readme"` - Header string `json:"header"` - Provider string `json:"provider"` - Related []ObjResp `json:"related"` + RawURL string `json:"raw_url"` + Readme string `json:"readme"` + Header string `json:"header"` + Provider string `json:"provider"` + Related []ObjLabelResp `json:"related"` } func FsGet(c *gin.Context) { @@ -263,7 +285,7 @@ func FsGet(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } @@ -347,7 +369,7 @@ func FsGet(c *gin.Context) { Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, - Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), + Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath), user.ID), }) } @@ -391,7 +413,7 @@ func FsOther(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, req.Path, req.Password) { + if !common.CanAccessWithRoles(user, meta, req.Path, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } diff --git a/server/handles/label.go b/server/handles/label.go new file mode 100644 index 00000000000..4631124ecbf --- /dev/null +++ b/server/handles/label.go @@ -0,0 +1,99 @@ +package handles + +import ( + "errors" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + "strconv" +) + +func ListLabel(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + log.Debugf("%+v", req) + labels, total, err := db.GetLabels(req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, common.PageResp{ + Content: labels, + Total: total, + }) +} + +func GetLabel(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + label, err := db.GetLabelById(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, label) +} + +func CreateLabel(c *gin.Context) { + var req model.Label + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if db.GetLabelByName(req.Name) { + common.ErrorResp(c, errors.New("label name is exists"), 401) + return + } + if id, err := db.CreateLabel(req); err != nil { + common.ErrorWithDataResp(c, err, 500, gin.H{ + "id": id, + }, true) + } else { + common.SuccessResp(c, gin.H{ + "id": id, + }) + } +} + +func UpdateLabel(c *gin.Context) { + var req model.Label + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if label, err := db.UpdateLabel(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c, label) + } +} + +func DeleteLabel(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + if err = op.DeleteLabelById(c, uint(id), userObj.ID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/label_file_binding.go b/server/handles/label_file_binding.go new file mode 100644 index 00000000000..78af929b34e --- /dev/null +++ b/server/handles/label_file_binding.go @@ -0,0 +1,103 @@ +package handles + +import ( + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "strconv" +) + +type DelLabelFileBinDingReq struct { + FileName string `json:"file_name"` + LabelId string `json:"label_id"` +} + +func GetLabelByFileName(c *gin.Context) { + fileName := c.Query("file_name") + if fileName == "" { + common.ErrorResp(c, errors.New("file_name must not empty"), 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + labels, err := op.GetLabelByFileName(userObj.ID, fileName) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, labels) +} + +func CreateLabelFileBinDing(c *gin.Context) { + var req op.CreateLabelFileBinDingReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.IsDir == true { + common.ErrorStrResp(c, "Unable to bind folder", 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + if err := op.CreateLabelFileBinDing(req, userObj.ID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } else { + common.SuccessResp(c, gin.H{ + "msg": "添加成功！", + }) + } +} + +func DelLabelByFileName(c *gin.Context) { + var req DelLabelFileBinDingReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + labelId, err := strconv.ParseUint(req.LabelId, 10, 64) + if err != nil { + common.ErrorResp(c, fmt.Errorf("invalid label ID '%s': %v", req.LabelId, err), 500, true) + return + } + if err = db.DelLabelFileBinDingById(uint(labelId), userObj.ID, req.FileName); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func GetFileByLabel(c *gin.Context) { + labelId := c.Query("label_id") + if labelId == "" { + common.ErrorResp(c, errors.New("file_name must not empty"), 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + fileList, err := op.GetFileByLabel(userObj.ID, labelId) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, fileList) +} diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go index cf3148291b1..2a85dc03d0b 100644 --- a/server/handles/ldap_login.go +++ b/server/handles/ldap_login.go @@ -131,7 +131,7 @@ func ladpRegister(username string) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), BasePath: setting.GetStr(conf.LdapDefaultDir), - Role: 0, + Role: nil, Disabled: false, } if err := db.CreateUser(user); err != nil { diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 24ff7a05369..8aade9eae57 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -5,6 +5,7 @@ import ( "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" @@ -253,10 +254,6 @@ type AddOfflineDownloadReq struct { func AddOfflineDownload(c *gin.Context) { user := c.MustGet("user").(*model.User) - if !user.CanAddOfflineDownloadTasks() { - common.ErrorStrResp(c, "permission denied", 403) - return - } var req AddOfflineDownloadReq if err := c.ShouldBind(&req); err != nil { @@ -268,6 +265,15 @@ func AddOfflineDownload(c *gin.Context) { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermAddOfflineDownload) { + common.ErrorStrResp(c, "permission denied", 403) + return + } var tasks []task.TaskExtensionInfo for _, url := range req.Urls { t, err := tool.AddURL(c, &tool.AddURLArgs{ diff --git a/server/handles/role.go b/server/handles/role.go new file mode 100644 index 00000000000..1bf7d4996bd --- /dev/null +++ b/server/handles/role.go @@ -0,0 +1,101 @@ +package handles + +import ( + "strconv" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" +) + +func ListRoles(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + log.Debugf("%+v", req) + roles, total, err := op.GetRoles(req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, common.PageResp{Content: roles, Total: total}) +} + +func GetRole(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, role) +} + +func CreateRole(c *gin.Context) { + var req model.Role + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.CreateRole(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c) + } +} + +func UpdateRole(c *gin.Context) { + var req model.Role + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(req.ID) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) + return + } + if err := op.UpdateRole(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c) + } +} + +func DeleteRole(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) + return + } + if err := op.DeleteRole(uint(id)); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/search.go b/server/handles/search.go index 8881731bd60..7d421a21e59 100644 --- a/server/handles/search.go +++ b/server/handles/search.go @@ -57,7 +57,7 @@ func Search(c *gin.Context) { if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { continue } - if !common.CanAccess(user, meta, path.Join(node.Parent, node.Name), req.Password) { + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), req.Password) { continue } filteredNodes = append(filteredNodes, node) diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 62bd4aaa2bf..eb6599e7a4d 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -154,7 +154,7 @@ func autoRegister(username, userID string, err error) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.SSODefaultPermission, 0)), BasePath: setting.GetStr(conf.SSODefaultDir), - Role: 0, + Role: nil, Disabled: false, SsoID: userID, } diff --git a/server/handles/task.go b/server/handles/task.go index af7974a9c29..6d49f9e5027 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -18,7 +18,7 @@ type TaskInfo struct { ID string `json:"id"` Name string `json:"name"` Creator string `json:"creator"` - CreatorRole int `json:"creator_role"` + CreatorRole model.Roles `json:"creator_role"` State tache.State `json:"state"` Status string `json:"status"` Progress float64 `json:"progress"` @@ -39,7 +39,7 @@ func getTaskInfo[T task.TaskExtensionInfo](task T) TaskInfo { progress = 100 } creatorName := "" - creatorRole := -1 + var creatorRole model.Roles if task.GetCreator() != nil { creatorName = task.GetCreator().Username creatorRole = task.GetCreator().Role diff --git a/server/handles/user.go b/server/handles/user.go index 4d404a4c652..50eaf969773 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -60,10 +60,10 @@ func UpdateUser(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if user.Role != req.Role { - common.ErrorStrResp(c, "role can not be changed", 400) - return - } + //if !utils.SliceEqual(user.Role, req.Role) { + // common.ErrorStrResp(c, "role can not be changed", 400) + // return + //} if req.Password == "" { req.PwdHash = user.PwdHash req.Salt = user.Salt diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index d65d1ad648a..47e7c0566c9 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -2,6 +2,7 @@ package middlewares import ( "crypto/subtle" + "fmt" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" @@ -68,6 +69,15 @@ func Auth(c *gin.Context) { c.Abort() return } + if len(user.Role) > 0 { + roles, err := op.GetRolesByUserID(user.ID) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("Fail to load roles: %v", err), 500) + c.Abort() + return + } + user.RolesDetail = roles + } c.Set("user", user) log.Debugf("use login token: %+v", user) c.Next() @@ -122,6 +132,19 @@ func Authn(c *gin.Context) { c.Abort() return } + if len(user.Role) > 0 { + var roles []model.Role + for _, roleID := range user.Role { + role, err := op.GetRole(uint(roleID)) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("load role %d failed", roleID), 500) + c.Abort() + return + } + roles = append(roles, *role) + } + user.RolesDetail = roles + } c.Set("user", user) log.Debugf("use login token: %+v", user) c.Next() diff --git a/server/middlewares/fsup.go b/server/middlewares/fsup.go index 2aa7fca6d03..243c22e4131 100644 --- a/server/middlewares/fsup.go +++ b/server/middlewares/fsup.go @@ -35,7 +35,9 @@ func FsUp(c *gin.Context) { return } } - if !(common.CanAccess(user, meta, path, password) && (user.CanWrite() || common.CanWrite(meta, stdpath.Dir(path)))) { + perm := common.MergeRolePermissions(user, path) + if !(common.CanAccessWithRoles(user, meta, path, password) && + (common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, stdpath.Dir(path)))) { common.ErrorResp(c, errs.PermissionDenied, 403) c.Abort() return diff --git a/server/router.go b/server/router.go index 09a0bb44faf..bf43a6258f4 100644 --- a/server/router.go +++ b/server/router.go @@ -120,6 +120,13 @@ func admin(g *gin.RouterGroup) { user.GET("/sshkey/list", handles.ListPublicKeys) user.POST("/sshkey/delete", handles.DeletePublicKey) + role := g.Group("/role") + role.GET("/list", handles.ListRoles) + role.GET("/get", handles.GetRole) + role.POST("/create", handles.CreateRole) + role.POST("/update", handles.UpdateRole) + role.POST("/delete", handles.DeleteRole) + storage := g.Group("/storage") storage.GET("/list", handles.ListStorages) storage.GET("/get", handles.GetStorage) @@ -161,6 +168,19 @@ func admin(g *gin.RouterGroup) { index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) + + label := g.Group("/label") + label.GET("/list", handles.ListLabel) + label.GET("/get", handles.GetLabel) + label.POST("/create", handles.CreateLabel) + label.POST("/update", handles.UpdateLabel) + label.POST("/delete", handles.DeleteLabel) + + labelFileBinding := g.Group("/label_file_binding") + labelFileBinding.GET("/get", handles.GetLabelByFileName) + labelFileBinding.GET("/get_file_by_label", handles.GetFileByLabel) + labelFileBinding.POST("/create", handles.CreateLabelFileBinDing) + labelFileBinding.POST("/delete", handles.DelLabelByFileName) } func _fs(g *gin.RouterGroup) { diff --git a/server/sftp.go b/server/sftp.go index 42c676e8c17..7d8c7212e9a 100644 --- a/server/sftp.go +++ b/server/sftp.go @@ -8,6 +8,7 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/ftp" "github.com/alist-org/alist/v3/server/sftp" "github.com/pkg/errors" @@ -78,7 +79,8 @@ func (d *SftpDriver) NoClientAuth(conn ssh.ConnMetadata) (*ssh.Permissions, erro if err != nil { return nil, err } - if guest.Disabled || !guest.CanFTPAccess() { + permGuest := common.MergeRolePermissions(guest, guest.BasePath) + if guest.Disabled || !common.HasPermission(permGuest, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } return nil, nil @@ -89,7 +91,8 @@ func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh. if err != nil { return nil, err } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } passHash := model.StaticHash(string(password)) @@ -104,7 +107,8 @@ func (d *SftpDriver) PublicKeyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*s if err != nil { return nil, err } - if userObj.Disabled || !userObj.CanFTPAccess() { + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { return nil, errors.New("user is not allowed to access via SFTP") } keys, _, err := op.GetSSHPublicKeyByUserId(userObj.ID, 1, -1) diff --git a/server/webdav.go b/server/webdav.go index a735e285527..a65896dfc79 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -3,16 +3,19 @@ package server import ( "context" "crypto/subtle" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/server/middlewares" "net/http" + "net/url" "path" "strings" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/middlewares" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/webdav" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -92,7 +95,19 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if user.Disabled || !user.CanWebdavRead() { + reqPath := c.Param("path") + if reqPath == "" { + reqPath = "/" + } + reqPath, _ = url.PathUnescape(reqPath) + reqPath, err = user.JoinPath(reqPath) + if err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + perm := common.MergeRolePermissions(user, reqPath) + if user.Disabled || !common.HasPermission(perm, common.PermWebdavRead) { if c.Request.Method == "OPTIONS" { c.Set("user", guest) c.Next() @@ -102,27 +117,27 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!user.CanWebdavManage() || !user.CanWrite()) { + if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermWrite)) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "MOVE" && (!user.CanWebdavManage() || (!user.CanMove() && !user.CanRename())) { + if c.Request.Method == "MOVE" && (!common.HasPermission(perm, common.PermWebdavManage) || (!common.HasPermission(perm, common.PermMove) && !common.HasPermission(perm, common.PermRename))) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "COPY" && (!user.CanWebdavManage() || !user.CanCopy()) { + if c.Request.Method == "COPY" && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermCopy)) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "DELETE" && (!user.CanWebdavManage() || !user.CanRemove()) { + if c.Request.Method == "DELETE" && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermRemove)) { c.Status(http.StatusForbidden) c.Abort() return } - if c.Request.Method == "PROPPATCH" && !user.CanWebdavManage() { + if c.Request.Method == "PROPPATCH" && !common.HasPermission(perm, common.PermWebdavManage) { c.Status(http.StatusForbidden) c.Abort() return diff --git a/server/webdav/file.go b/server/webdav/file.go index ac8f5c1cbfb..ab78d26105d 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -14,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" ) // slashClean is equivalent to but slightly more efficient than @@ -34,10 +35,11 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int srcName := path.Base(src) dstName := path.Base(dst) user := ctx.Value("user").(*model.User) - if srcDir != dstDir && !user.CanMove() { + perm := common.MergeRolePermissions(user, src) + if srcDir != dstDir && !common.HasPermission(perm, common.PermMove) { return http.StatusForbidden, nil } - if srcName != dstName && !user.CanRename() { + if srcName != dstName && !common.HasPermission(perm, common.PermRename) { return http.StatusForbidden, nil } if srcDir == dstDir { From f61d13d4330348493e4fa64a3b595999e9b6b100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 26 Jul 2025 15:20:08 +0800 Subject: [PATCH 528/659] refactor(convert_role): Improve role conversion logic for legacy formats (#9219) - Add new imports: `database/sql`, `encoding/json`, and `conf` package in `convert_role.go`. - Simplify permission entry initialization by removing redundant struct formatting. - Update error logging messages for better clarity. - Replace `op.GetUsers` with direct database access for fetching user roles. - Implement role update logic using `rawDb` and handle legacy int role conversion. - Count the number of users whose roles are updated and log completion. - Introduce `IsLegacyRoleDetected` function to check for legacy role formats. - Modify `cmd/common.go` to invoke role conversion if legacy format is detected. --- cmd/common.go | 7 ++ .../bootstrap/patch/v3_46_0/convert_role.go | 107 ++++++++++++++---- 2 files changed, 89 insertions(+), 25 deletions(-) diff --git a/cmd/common.go b/cmd/common.go index 8a73f9b0582..d88a86eb09d 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_46_0" "os" "path/filepath" "strconv" @@ -16,6 +17,12 @@ func Init() { bootstrap.InitConfig() bootstrap.Log() bootstrap.InitDB() + + if v3_46_0.IsLegacyRoleDetected() { + utils.Log.Warnf("Detected legacy role format, executing ConvertLegacyRoles patch early...") + v3_46_0.ConvertLegacyRoles() + } + data.InitData() bootstrap.InitStreamLimit() bootstrap.InitIndex() diff --git a/internal/bootstrap/patch/v3_46_0/convert_role.go b/internal/bootstrap/patch/v3_46_0/convert_role.go index 43799485c12..3aac95b691c 100644 --- a/internal/bootstrap/patch/v3_46_0/convert_role.go +++ b/internal/bootstrap/patch/v3_46_0/convert_role.go @@ -1,7 +1,10 @@ package v3_46_0 import ( + "database/sql" + "encoding/json" "errors" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" @@ -83,47 +86,101 @@ func ConvertLegacyRoles() { } } - users, _, err := op.GetUsers(1, -1) + rawDb := db.GetDb() + table := conf.Conf.Database.TablePrefix + "users" + rows, err := rawDb.Table(table).Select("id, username, role").Rows() if err != nil { utils.Log.Errorf("[convert roles] failed to get users: %v", err) return } + defer rows.Close() - for i := range users { - user := users[i] - if user.Role == nil { + var updatedCount int + for rows.Next() { + var id uint + var username string + var rawRole []byte + + if err := rows.Scan(&id, &username, &rawRole); err != nil { + utils.Log.Warnf("[convert roles] skip user scan err: %v", err) continue } - changed := false - var roles model.Roles - for _, r := range user.Role { + + utils.Log.Debugf("[convert roles] user: %s raw role: %s", username, string(rawRole)) + + if len(rawRole) == 0 { + continue + } + + var oldRoles []int + wasSingleInt := false + if err := json.Unmarshal(rawRole, &oldRoles); err != nil { + var single int + if err := json.Unmarshal(rawRole, &single); err != nil { + utils.Log.Warnf("[convert roles] user %s has invalid role: %s", username, string(rawRole)) + continue + } + oldRoles = []int{single} + wasSingleInt = true + } + + var newRoles model.Roles + for _, r := range oldRoles { switch r { case model.ADMIN: - roles = append(roles, int(adminRole.ID)) - if int(adminRole.ID) != r { - changed = true - } + newRoles = append(newRoles, int(adminRole.ID)) case model.GUEST: - roles = append(roles, int(guestRole.ID)) - if int(guestRole.ID) != r { - changed = true - } + newRoles = append(newRoles, int(guestRole.ID)) case model.GENERAL: - roles = append(roles, int(generalRole.ID)) - if int(generalRole.ID) != r { - changed = true - } + newRoles = append(newRoles, int(generalRole.ID)) default: - roles = append(roles, r) + newRoles = append(newRoles, r) } } - if changed { - user.Role = roles - if err := db.UpdateUser(&user); err != nil { - utils.Log.Errorf("[convert roles] failed to update user %s: %v", user.Username, err) + + if wasSingleInt { + err := rawDb.Table(table).Where("id = ?", id).Update("role", newRoles).Error + if err != nil { + utils.Log.Errorf("[convert roles] failed to update user %s: %v", username, err) + } else { + updatedCount++ + utils.Log.Infof("[convert roles] updated user %s: %v → %v", username, oldRoles, newRoles) } } } - utils.Log.Infof("[convert roles] completed role conversion for %d users", len(users)) + utils.Log.Infof("[convert roles] completed role conversion for %d users", updatedCount) +} + +func IsLegacyRoleDetected() bool { + rawDb := db.GetDb() + table := conf.Conf.Database.TablePrefix + "users" + rows, err := rawDb.Table(table).Select("role").Rows() + if err != nil { + utils.Log.Errorf("[role check] failed to scan user roles: %v", err) + return false + } + defer rows.Close() + + for rows.Next() { + var raw sql.RawBytes + if err := rows.Scan(&raw); err != nil { + continue + } + if len(raw) == 0 { + continue + } + + var roles []int + if err := json.Unmarshal(raw, &roles); err == nil { + continue + } + + var single int + if err := json.Unmarshal(raw, &single); err == nil { + utils.Log.Infof("[role check] detected legacy int role: %d", single) + return true + } + } + return false } From 91cc7529a035d7f78d332c4cf54f501ffb65c2cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sun, 27 Jul 2025 22:25:45 +0800 Subject: [PATCH 529/659] feat(user/role/storage): enhance user and storage operations with additional validations (#9223) - Update `CreateUser` to adjust `BasePath` based on user roles and clean paths. - Modify `UpdateUser` to incorporate role-based path changes. - Add validation in `CreateStorage` and `UpdateStorage` to prevent root mount path. - Prevent changes to admin user's role and username in user handler. - Update `UpdateRole` to modify user base paths when role paths change, and clear user cache accordingly. - Import `errors` package to handle error messages. --- internal/op/role.go | 15 +++++++++++++++ internal/op/storage.go | 8 ++++++++ internal/op/user.go | 31 ++++++++++++++++++++++++++++++- server/handles/user.go | 17 +++++++++++++---- 4 files changed, 66 insertions(+), 5 deletions(-) diff --git a/internal/op/role.go b/internal/op/role.go index 64502f98df4..e0f2dc7592c 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -2,6 +2,7 @@ package op import ( "fmt" + "github.com/pkg/errors" "time" "github.com/Xhofe/go-cache" @@ -102,6 +103,20 @@ func UpdateRole(r *model.Role) error { for i := range r.PermissionScopes { r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) } + if len(old.PermissionScopes) > 0 && len(r.PermissionScopes) > 0 && + old.PermissionScopes[0].Path != r.PermissionScopes[0].Path { + + oldPath := old.PermissionScopes[0].Path + newPath := r.PermissionScopes[0].Path + modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath) + if err != nil { + return errors.WithMessage(err, "failed to update user base path when role updated") + } + + for _, name := range modifiedUsernames { + userCache.Del(name) + } + } roleCache.Del(fmt.Sprint(r.ID)) roleCache.Del(r.Name) return db.UpdateRole(r) diff --git a/internal/op/storage.go b/internal/op/storage.go index 2ec68aae5e6..2833afa84ca 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -46,6 +46,11 @@ func GetStorageByMountPath(mountPath string) (driver.Driver, error) { func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + + if storage.MountPath == "/" { + return 0, errors.New("Mount path cannot be '/'") + } + var err error // check driver first driverName := storage.Driver @@ -205,6 +210,9 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { } storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + if storage.MountPath == "/" { + return errors.New("Mount path cannot be '/'") + } err = db.UpdateStorage(&storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") diff --git a/internal/op/user.go b/internal/op/user.go index e775df63e93..30b9d0e6d26 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -78,7 +78,25 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err func CreateUser(u *model.User) error { u.BasePath = utils.FixAndCleanPath(u.BasePath) - return db.CreateUser(u) + + err := db.CreateUser(u) + if err != nil { + return err + } + + roles, err := GetRolesByUserID(u.ID) + if err == nil { + for _, role := range roles { + if len(role.PermissionScopes) > 0 { + u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + break + } + } + _ = db.UpdateUser(u) + userCache.Del(u.Username) + } + + return nil } func DeleteUserById(id uint) error { @@ -106,6 +124,17 @@ func UpdateUser(u *model.User) error { } userCache.Del(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) + if len(u.Role) > 0 { + roles, err := GetRolesByUserID(u.ID) + if err == nil { + for _, role := range roles { + if len(role.PermissionScopes) > 0 { + u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + break + } + } + } + } return db.UpdateUser(u) } diff --git a/server/handles/user.go b/server/handles/user.go index 50eaf969773..b729d117fdc 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -1,6 +1,7 @@ package handles import ( + "github.com/alist-org/alist/v3/pkg/utils" "strconv" "github.com/alist-org/alist/v3/internal/model" @@ -60,10 +61,18 @@ func UpdateUser(c *gin.Context) { common.ErrorResp(c, err, 500) return } - //if !utils.SliceEqual(user.Role, req.Role) { - // common.ErrorStrResp(c, "role can not be changed", 400) - // return - //} + + if user.Username == "admin" { + if !utils.SliceEqual(user.Role, req.Role) { + common.ErrorStrResp(c, "cannot change role of admin user", 403) + return + } + if user.Username != req.Username { + common.ErrorStrResp(c, "cannot change username of admin user", 403) + return + } + } + if req.Password == "" { req.PwdHash = user.PwdHash req.Salt = user.Salt From 5b8c26510b720a2cd308023dc7cd1c8bd1e9d20c Mon Sep 17 00:00:00 2001 From: qianshi Date: Mon, 28 Jul 2025 23:07:07 +0800 Subject: [PATCH 530/659] feat(user-management): Enhance admin management and role handling - Add `CountEnabledAdminsExcluding` function to count enabled admins excluding a specific user. - Implement `CountUsersByRoleAndEnabledExclude` in `internal/db/user.go` to support exclusion logic. - Refactor role handling with switch-case for better readability in `server/handles/role.go`. - Ensure at least one enabled admin remains when disabling an admin in `server/handles/user.go`. - Maintain guest role name consistency when updating roles in `internal/op/role.go`. --- internal/db/user.go | 11 +++++++++++ internal/op/role.go | 6 +++++- internal/op/user.go | 8 ++++++++ server/handles/role.go | 6 +++++- server/handles/user.go | 13 ++++++++++--- 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/internal/db/user.go b/internal/db/user.go index f2b6635a983..9e8ee3fc05e 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -2,6 +2,7 @@ package db import ( "encoding/base64" + "fmt" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-webauthn/webauthn/webauthn" @@ -140,3 +141,13 @@ func UpdateUserBasePathPrefix(oldPath, newPath string) ([]string, error) { return modifiedUsernames, nil } + +func CountUsersByRoleAndEnabledExclude(roleID uint, excludeUserID uint) (int64, error) { + var count int64 + jsonValue := fmt.Sprintf("[%d]", roleID) + err := db.Model(&model.User{}). + Where("disabled = ? AND id != ?", false, excludeUserID). + Where("JSON_CONTAINS(role, ?)", jsonValue). + Count(&count).Error + return count, err +} diff --git a/internal/op/role.go b/internal/op/role.go index 64502f98df4..b7474566438 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -96,8 +96,12 @@ func UpdateRole(r *model.Role) error { if err != nil { return err } - if old.Name == "admin" || old.Name == "guest" { + switch old.Name { + case "admin": return errs.ErrChangeDefaultRole + + case "guest": + r.Name = "guest" } for i := range r.PermissionScopes { r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) diff --git a/internal/op/user.go b/internal/op/user.go index e775df63e93..b9662015902 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -136,3 +136,11 @@ func DelUserCache(username string) error { userCache.Del(username) return nil } + +func CountEnabledAdminsExcluding(userID uint) (int64, error) { + adminRole, err := GetRoleByName("admin") + if err != nil { + return 0, err + } + return db.CountUsersByRoleAndEnabledExclude(adminRole.ID, userID) +} diff --git a/server/handles/role.go b/server/handles/role.go index 1bf7d4996bd..0d071c9f84d 100644 --- a/server/handles/role.go +++ b/server/handles/role.go @@ -66,9 +66,13 @@ func UpdateRole(c *gin.Context) { common.ErrorResp(c, err, 500, true) return } - if role.Name == "admin" || role.Name == "guest" { + switch role.Name { + case "admin": common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) return + + case "guest": + req.Name = "guest" } if err := op.UpdateRole(&req); err != nil { common.ErrorResp(c, err, 500, true) diff --git a/server/handles/user.go b/server/handles/user.go index 50eaf969773..858c2a3bcf0 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -74,9 +74,16 @@ func UpdateUser(c *gin.Context) { if req.OtpSecret == "" { req.OtpSecret = user.OtpSecret } - if req.Disabled && req.IsAdmin() { - common.ErrorStrResp(c, "admin user can not be disabled", 400) - return + if req.Disabled && user.IsAdmin() { + count, err := op.CountEnabledAdminsExcluding(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if count == 0 { + common.ErrorStrResp(c, "at least one enabled admin must be kept", 400) + return + } } if err := op.UpdateUser(&req); err != nil { common.ErrorResp(c, err, 500) From 4d7c2a09ce4716cecc541d9e2518d1fdf1d23172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 29 Jul 2025 09:42:34 +0800 Subject: [PATCH 531/659] docs(README): Add API documentation links across multiple languages (#9225) - Add API documentation section to `README.md` with link to Apifox - Add API documentation section to `README_ja.md` with Japanese translation and link to Apifox - Add API documentation section to `README_cn.md` with Chinese translation and link to Apifox --- README.md | 4 ++++ README_cn.md | 4 ++++ README_ja.md | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/README.md b/README.md index 2bd7e812890..5a93997fe40 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,10 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing +## API Documentation (via Apifox): + + + ## Demo diff --git a/README_cn.md b/README_cn.md index 9052e79b0ea..79ed864bc84 100644 --- a/README_cn.md +++ b/README_cn.md @@ -99,6 +99,10 @@ +## API 文档（通过 Apifox 提供） + + + ## Demo diff --git a/README_ja.md b/README_ja.md index 4dcdfd203bf..9291b2acdc2 100644 --- a/README_ja.md +++ b/README_ja.md @@ -100,6 +100,10 @@ +## APIドキュメント（Apifox 提供） + + + ## デモ From 33530554821085e069cf165083e0d7862879cb65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 29 Jul 2025 18:35:47 +0800 Subject: [PATCH 532/659] Update Dockerfile.ci (#9230) chore(docker): Update base image from alpine:edge to alpine:3.20.7 in Dockerfile.ci --- Dockerfile.ci | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.ci b/Dockerfile.ci index a17aae9fcfd..6075acc639a 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1,4 +1,4 @@ -FROM alpine:edge +FROM alpine:3.20.7 ARG TARGETPLATFORM ARG INSTALL_FFMPEG=false @@ -31,4 +31,4 @@ RUN /entrypoint.sh version ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} VOLUME /opt/alist/data/ EXPOSE 5244 5245 -CMD [ "/entrypoint.sh" ] \ No newline at end of file +CMD [ "/entrypoint.sh" ] From 540d6c7064994b009293d0dac42419280a525166 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Wed, 30 Jul 2025 10:48:33 +0800 Subject: [PATCH 533/659] fix(meta): update OAuth token URL and improve default client credentials (#9231) --- drivers/aliyundrive_open/meta.go | 2 +- drivers/baidu_netdisk/meta.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index 03f97f8b795..bb4354ddc11 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -11,7 +11,7 @@ type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` - OauthTokenURL string `json:"oauth_token_url" default:"https://api.nn.ci/alist/ali_open/token"` + OauthTokenURL string `json:"oauth_token_url" default:"https://api.alistgo.com/alist/ali_open/token"` ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 27571056e11..7577c747fe3 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -11,8 +11,8 @@ type Addition struct { OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` DownloadAPI string `json:"download_api" type:"select" options:"official,crack,crack_video" default:"official"` - ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` - ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` + ClientID string `json:"client_id" required:"true" default:"hq9yQ9w9kR4YHj1kyYafLygVocobh7Sf"` + ClientSecret string `json:"client_secret" required:"true" default:"YH2VpZcFJHYNnV6vLfHQXDBhcE7ZChyE"` CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` AccessToken string UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` From 74332e91fb8f3a920d6c59557dddb66f4a2bb188 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Wed, 30 Jul 2025 12:22:02 +0800 Subject: [PATCH 534/659] feat(ui): add new UI configuration option to settings (#9233) * feat(ui): add new UI configuration option to settings * fix(ui): disable new UI feature by default --------- Co-authored-by: Sky_slience --- internal/bootstrap/data/setting.go | 2 ++ internal/conf/const.go | 1 + 2 files changed, 3 insertions(+) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index fe1d9219a08..e00abf2d379 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -103,6 +103,8 @@ func InitialSettings() []model.SettingItem { {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, + // newui settings + {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, // style settings {Key: conf.Logo, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE}, {Key: conf.Favicon, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 5cb8d850bf0..48ac2037fb2 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -16,6 +16,7 @@ const ( AllowIndexed = "allow_indexed" AllowMounted = "allow_mounted" RobotsTxt = "robots_txt" + UseNewui = "use_newui" Logo = "logo" Favicon = "favicon" From 280960ce3e338c41d3e06181b3bf95d6b0ee422f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Wed, 30 Jul 2025 13:15:35 +0800 Subject: [PATCH 535/659] feat(user-db): enhance user management with role-based queries (allow-edit-role-guest) (#9234) - Add `GetUsersByRole` function to fetch users based on their roles. - Extend `UpdateUserBasePathPrefix` to accept optional user lists. - Ensure path cleaning in `UpdateUserBasePathPrefix` for consistency. - Integrate guest role fetching in `auth.go` middleware. - Utilize `GetUsersByRole` in `role.go` for base path modifications. - Remove redundant line in `role.go` role modification logic. --- internal/db/user.go | 33 ++++++++++++++++++++++++++------- internal/op/role.go | 9 +++++++-- server/middlewares/auth.go | 9 +++++++++ 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/internal/db/user.go b/internal/db/user.go index 9e8ee3fc05e..8f1c28b92f2 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -9,6 +9,7 @@ import ( "github.com/pkg/errors" "gorm.io/gorm" "path" + "slices" "strings" ) @@ -25,6 +26,20 @@ func GetUserByRole(role int) (*model.User, error) { return nil, gorm.ErrRecordNotFound } +func GetUsersByRole(roleID int) ([]model.User, error) { + var users []model.User + if err := db.Find(&users).Error; err != nil { + return nil, err + } + var result []model.User + for _, u := range users { + if slices.Contains(u.Role, roleID) { + result = append(result, u) + } + } + return result, nil +} + func GetUserByName(username string) (*model.User, error) { user := model.User{Username: username} if err := db.Where(user).First(&user).Error; err != nil { @@ -109,25 +124,29 @@ func RemoveAuthn(u *model.User, id string) error { return UpdateAuthn(u.ID, string(res)) } -func UpdateUserBasePathPrefix(oldPath, newPath string) ([]string, error) { +func UpdateUserBasePathPrefix(oldPath, newPath string, usersOpt ...[]model.User) ([]string, error) { var users []model.User var modifiedUsernames []string - if err := db.Find(&users).Error; err != nil { - return nil, errors.WithMessage(err, "failed to load users") - } - oldPathClean := path.Clean(oldPath) + if len(usersOpt) > 0 { + users = usersOpt[0] + } else { + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load users") + } + } + for _, user := range users { basePath := path.Clean(user.BasePath) updated := false if basePath == oldPathClean { - user.BasePath = newPath + user.BasePath = path.Clean(newPath) updated = true } else if strings.HasPrefix(basePath, oldPathClean+"/") { - user.BasePath = newPath + basePath[len(oldPathClean):] + user.BasePath = path.Clean(newPath + basePath[len(oldPathClean):]) updated = true } diff --git a/internal/op/role.go b/internal/op/role.go index b312f8c79e0..c719c6f4cdf 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -100,7 +100,6 @@ func UpdateRole(r *model.Role) error { switch old.Name { case "admin": return errs.ErrChangeDefaultRole - case "guest": r.Name = "guest" } @@ -112,7 +111,13 @@ func UpdateRole(r *model.Role) error { oldPath := old.PermissionScopes[0].Path newPath := r.PermissionScopes[0].Path - modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath) + + users, err := db.GetUsersByRole(int(r.ID)) + if err != nil { + return errors.WithMessage(err, "failed to get users by role") + } + + modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath, users) if err != nil { return errors.WithMessage(err, "failed to update user base path when role updated") } diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 47e7c0566c9..c0743c9ce9d 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -41,6 +41,15 @@ func Auth(c *gin.Context) { c.Abort() return } + if len(guest.Role) > 0 { + roles, err := op.GetRolesByUserID(guest.ID) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("Fail to load guest roles: %v", err), 500) + c.Abort() + return + } + guest.RolesDetail = roles + } c.Set("user", guest) log.Debugf("use empty token: %+v", guest) c.Next() From 394a18cbd96b581cab75a780be04aff84451fd46 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Wed, 30 Jul 2025 16:55:32 +0800 Subject: [PATCH 536/659] Fix 123 download (#9235) * fix(driver): handle additional HTTP status code 210 for URL redirection * fix(driver): 123 download url error --------- Co-authored-by: Sky_slience --- drivers/123/driver.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 32c053e22ab..a8af2b6fd01 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -113,6 +113,8 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) log.Debugln("res code: ", res.StatusCode()) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") + } else if res.StatusCode() == 210 { + link.URL = downloadUrl } else if res.StatusCode() < 300 { link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString() } From ae90fb579bded772f1a7744195ed4e0409a8fadb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sun, 3 Aug 2025 09:26:23 +0800 Subject: [PATCH 537/659] feat(log): enhance log formatter to respect NO_COLOR env variable (#9239) - Adjust log formatter to disable colors when NO_COLOR or ALIST_NO_COLOR environment variables are set. - Reorganize formatter settings for better readability. --- internal/bootstrap/log.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/bootstrap/log.go b/internal/bootstrap/log.go index 00411e5e189..b4f4af08f1b 100644 --- a/internal/bootstrap/log.go +++ b/internal/bootstrap/log.go @@ -14,10 +14,14 @@ import ( func init() { formatter := logrus.TextFormatter{ - ForceColors: true, - EnvironmentOverrideColors: true, - TimestampFormat: "2006-01-02 15:04:05", - FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05", + FullTimestamp: true, + } + if os.Getenv("NO_COLOR") != "" || os.Getenv("ALIST_NO_COLOR") == "1" { + formatter.DisableColors = true + } else { + formatter.ForceColors = true + formatter.EnvironmentOverrideColors = true } logrus.SetFormatter(&formatter) utils.Log.SetFormatter(&formatter) From 46de9e9ebb4bed894eae82270cbf6137cf790dd8 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Sun, 3 Aug 2025 20:00:09 +0800 Subject: [PATCH 538/659] fix(driver): 123 download and modify request headers on the frontend (#9236) Co-authored-by: Sky_slience --- drivers/123/driver.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index a8af2b6fd01..32c053e22ab 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -113,8 +113,6 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) log.Debugln("res code: ", res.StatusCode()) if res.StatusCode() == 302 { link.URL = res.Header().Get("location") - } else if res.StatusCode() == 210 { - link.URL = downloadUrl } else if res.StatusCode() < 300 { link.URL = utils.Json.Get(res.Body(), "data", "redirect_url").ToString() } From 52da07e8a768277e0145457396401ce1b4758e92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 4 Aug 2025 11:56:57 +0800 Subject: [PATCH 539/659] feat(123_open): add new driver support for 123 Open (#9246) - Implement new driver for 123 Open service, enabling file operations such as listing, uploading, moving, and removing files. - Introduce token management for authentication and authorization. - Add API integration for various file operations and actions. - Include utility functions for handling API requests and responses. - Register the new driver in the existing drivers' list. --- drivers/123_open/api.go | 191 +++++++++++++++++++++++++ drivers/123_open/driver.go | 277 ++++++++++++++++++++++++++++++++++++ drivers/123_open/meta.go | 33 +++++ drivers/123_open/token.go | 85 +++++++++++ drivers/123_open/types.go | 70 +++++++++ drivers/123_open/upload.go | 282 +++++++++++++++++++++++++++++++++++++ drivers/123_open/util.go | 20 +++ drivers/all.go | 1 + internal/errs/driver.go | 1 + internal/model/obj.go | 15 ++ 10 files changed, 975 insertions(+) create mode 100644 drivers/123_open/api.go create mode 100644 drivers/123_open/driver.go create mode 100644 drivers/123_open/meta.go create mode 100644 drivers/123_open/token.go create mode 100644 drivers/123_open/types.go create mode 100644 drivers/123_open/upload.go create mode 100644 drivers/123_open/util.go diff --git a/drivers/123_open/api.go b/drivers/123_open/api.go new file mode 100644 index 00000000000..1d2a6f164ff --- /dev/null +++ b/drivers/123_open/api.go @@ -0,0 +1,191 @@ +package _123Open + +import ( + "fmt" + "github.com/go-resty/resty/v2" + "net/http" +) + +const ( + // baseurl + ApiBaseURL = "https://open-api.123pan.com" + + // auth + ApiToken = "/api/v1/access_token" + + // file list + ApiFileList = "/api/v2/file/list" + + // direct link + ApiGetDirectLink = "/api/v1/direct-link/url" + + // mkdir + ApiMakeDir = "/upload/v1/file/mkdir" + + // remove + ApiRemove = "/api/v1/file/trash" + + // upload + ApiUploadDomainURL = "/upload/v2/file/domain" + ApiSingleUploadURL = "/upload/v2/file/single/create" + ApiCreateUploadURL = "/upload/v2/file/create" + ApiUploadSliceURL = "/upload/v2/file/slice" + ApiUploadCompleteURL = "/upload/v2/file/upload_complete" + + // move + ApiMove = "/api/v1/file/move" + + // rename + ApiRename = "/api/v1/file/name" +) + +type Response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type TokenResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data TokenData `json:"data"` +} + +type TokenData struct { + AccessToken string `json:"accessToken"` + ExpiredAt string `json:"expiredAt"` +} + +type FileListResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileListData `json:"data"` +} + +type FileListData struct { + LastFileId int64 `json:"lastFileId"` + FileList []File `json:"fileList"` +} + +type DirectLinkResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data DirectLinkData `json:"data"` +} + +type DirectLinkData struct { + URL string `json:"url"` +} + +type MakeDirRequest struct { + Name string `json:"name"` + ParentID int64 `json:"parentID"` +} + +type MakeDirResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data MakeDirData `json:"data"` +} + +type MakeDirData struct { + DirID int64 `json:"dirID"` +} + +type RemoveRequest struct { + FileIDs []int64 `json:"fileIDs"` +} + +type UploadCreateResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadCreateData `json:"data"` +} + +type UploadCreateData struct { + FileID int64 `json:"fileId"` + Reuse bool `json:"reuse"` + PreuploadID string `json:"preuploadId"` + SliceSize int64 `json:"sliceSize"` + Servers []string `json:"servers"` +} + +type UploadUrlResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadUrlData `json:"data"` +} + +type UploadUrlData struct { + PresignedURL string `json:"presignedUrl"` +} + +type UploadCompleteResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadCompleteData `json:"data"` +} + +type UploadCompleteData struct { + FileID int `json:"fileID"` + Completed bool `json:"completed"` +} + +func (d *Open123) Request(endpoint string, method string, setup func(*resty.Request), result any) (*resty.Response, error) { + client := resty.New() + token, err := d.tm.getToken() + if err != nil { + return nil, err + } + + req := client.R(). + SetHeader("Authorization", "Bearer "+token). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", "application/json"). + SetResult(result) + + if setup != nil { + setup(req) + } + + switch method { + case http.MethodGet: + return req.Get(ApiBaseURL + endpoint) + case http.MethodPost: + return req.Post(ApiBaseURL + endpoint) + case http.MethodPut: + return req.Put(ApiBaseURL + endpoint) + default: + return nil, fmt.Errorf("unsupported method: %s", method) + } +} + +func (d *Open123) RequestTo(fullURL string, method string, setup func(*resty.Request), result any) (*resty.Response, error) { + client := resty.New() + + token, err := d.tm.getToken() + if err != nil { + return nil, err + } + + req := client.R(). + SetHeader("Authorization", "Bearer "+token). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", "application/json"). + SetResult(result) + + if setup != nil { + setup(req) + } + + switch method { + case http.MethodGet: + return req.Get(fullURL) + case http.MethodPost: + return req.Post(fullURL) + case http.MethodPut: + return req.Put(fullURL) + default: + return nil, fmt.Errorf("unsupported method: %s", method) + } +} diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go new file mode 100644 index 00000000000..39ed146eb03 --- /dev/null +++ b/drivers/123_open/driver.go @@ -0,0 +1,277 @@ +package _123Open + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "net/http" + "strconv" +) + +type Open123 struct { + model.Storage + Addition + + UploadThread int + tm *tokenManager +} + +func (d *Open123) Config() driver.Config { + return config +} + +func (d *Open123) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Open123) Init(ctx context.Context) error { + d.tm = newTokenManager(d.ClientID, d.ClientSecret) + + if _, err := d.tm.getToken(); err != nil { + return fmt.Errorf("token 初始化失败: %w", err) + } + + return nil +} + +func (d *Open123) Drop(ctx context.Context) error { + return nil +} + +func (d *Open123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + parentFileId, err := strconv.ParseInt(dir.GetID(), 10, 64) + if err != nil { + return nil, err + } + + fileLastId := int64(0) + var results []File + + for fileLastId != -1 { + files, err := d.getFiles(parentFileId, 100, fileLastId) + if err != nil { + return nil, err + } + for _, f := range files.Data.FileList { + if f.Trashed == 0 { + results = append(results, f) + } + } + fileLastId = files.Data.LastFileId + } + + objs := make([]model.Obj, 0, len(results)) + for _, f := range results { + objs = append(objs, f) + } + return objs, nil +} + +func (d *Open123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.LinkIsDir + } + + fileID := file.GetID() + + var result DirectLinkResp + url := fmt.Sprintf("%s?fileID=%s", ApiGetDirectLink, fileID) + _, err := d.Request(url, http.MethodGet, nil, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("get link failed: %s", result.Message) + } + + return &model.Link{ + URL: result.Data.URL, + }, nil +} + +func (d *Open123) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID, err := strconv.ParseInt(parentDir.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid parent ID: %w", err) + } + + var result MakeDirResp + reqBody := MakeDirRequest{ + Name: dirName, + ParentID: parentID, + } + + _, err = d.Request(ApiMakeDir, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("mkdir failed: %s", result.Message) + } + + newDir := File{ + FileId: result.Data.DirID, + FileName: dirName, + Type: 1, + ParentFileId: int(parentID), + Size: 0, + Trashed: 0, + } + return newDir, nil +} + +func (d *Open123) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcID, err := strconv.ParseInt(srcObj.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid src file ID: %w", err) + } + dstID, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid dest dir ID: %w", err) + } + + var result Response[any] + reqBody := map[string]interface{}{ + "fileIDs": []int64{srcID}, + "toParentFileID": dstID, + } + + _, err = d.Request(ApiMove, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("move failed: %s", result.Message) + } + + files, err := d.getFiles(dstID, 100, 0) + if err != nil { + return nil, fmt.Errorf("move succeed but failed to get target dir: %w", err) + } + for _, f := range files.Data.FileList { + if f.FileId == srcID { + return f, nil + } + } + return nil, fmt.Errorf("move succeed but file not found in target dir") +} + +func (d *Open123) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + srcID, err := strconv.ParseInt(srcObj.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid file ID: %w", err) + } + + var result Response[any] + reqBody := map[string]interface{}{ + "fileId": srcID, + "fileName": newName, + } + + _, err = d.Request(ApiRename, http.MethodPut, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("rename failed: %s", result.Message) + } + + parentID := 0 + if file, ok := srcObj.(File); ok { + parentID = file.ParentFileId + } + files, err := d.getFiles(int64(parentID), 100, 0) + if err != nil { + return nil, fmt.Errorf("rename succeed but failed to get parent dir: %w", err) + } + for _, f := range files.Data.FileList { + if f.FileId == srcID { + return f, nil + } + } + return nil, fmt.Errorf("rename succeed but file not found in parent dir") +} + +func (d *Open123) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { + idStr := obj.GetID() + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + return fmt.Errorf("invalid file ID: %w", err) + } + + var result Response[any] + reqBody := RemoveRequest{ + FileIDs: []int64{id}, + } + + _, err = d.Request(ApiRemove, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return err + } + if result.Code != 0 { + return fmt.Errorf("remove failed: %s", result.Message) + } + + return nil +} + +func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + etag := file.GetHash().GetHash(utils.MD5) + + if len(etag) < utils.MD5.Width { + up = model.UpdateProgressWithRange(up, 50, 100) + _, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return err + } + if createResp.Data.Reuse { + return nil + } + + return d.Upload(ctx, file, parentFileId, createResp, up) +} + +func (d *Open123) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotSupport +} + +func (d *Open123) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Open123) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotSupport +} + +func (d *Open123) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +//func (d *Open123) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Open123)(nil) diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go new file mode 100644 index 00000000000..d99bb75ba2c --- /dev/null +++ b/drivers/123_open/meta.go @@ -0,0 +1,33 @@ +package _123Open + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + + ClientID string `json:"client_id" required:"true" label:"Client ID"` + ClientSecret string `json:"client_secret" required:"true" label:"Client Secret"` +} + +var config = driver.Config{ + Name: "123 Open", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Open123{} + }) +} diff --git a/drivers/123_open/token.go b/drivers/123_open/token.go new file mode 100644 index 00000000000..435c0b0de67 --- /dev/null +++ b/drivers/123_open/token.go @@ -0,0 +1,85 @@ +package _123Open + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" +) + +const tokenURL = ApiBaseURL + ApiToken + +type tokenManager struct { + clientID string + clientSecret string + + mu sync.Mutex + accessToken string + expireTime time.Time +} + +func newTokenManager(clientID, clientSecret string) *tokenManager { + return &tokenManager{ + clientID: clientID, + clientSecret: clientSecret, + } +} + +func (tm *tokenManager) getToken() (string, error) { + tm.mu.Lock() + defer tm.mu.Unlock() + + if tm.accessToken != "" && time.Now().Before(tm.expireTime.Add(-5*time.Minute)) { + return tm.accessToken, nil + } + + reqBody := map[string]string{ + "clientID": tm.clientID, + "clientSecret": tm.clientSecret, + } + body, _ := json.Marshal(reqBody) + req, err := http.NewRequest("POST", tokenURL, bytes.NewBuffer(body)) + if err != nil { + return "", err + } + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var result TokenResp + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err + } + + if result.Code != 0 { + return "", fmt.Errorf("get token failed: %s", result.Message) + } + + tm.accessToken = result.Data.AccessToken + expireAt, err := time.Parse(time.RFC3339, result.Data.ExpiredAt) + if err != nil { + return "", fmt.Errorf("parse expire time failed: %w", err) + } + tm.expireTime = expireAt + + return tm.accessToken, nil +} + +func (tm *tokenManager) buildHeaders() (http.Header, error) { + token, err := tm.getToken() + if err != nil { + return nil, err + } + header := http.Header{} + header.Set("Authorization", "Bearer "+token) + header.Set("Platform", "open_platform") + header.Set("Content-Type", "application/json") + return header, nil +} diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go new file mode 100644 index 00000000000..afece279e60 --- /dev/null +++ b/drivers/123_open/types.go @@ -0,0 +1,70 @@ +package _123Open + +import ( + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" + "time" +) + +type File struct { + FileName string `json:"filename"` + Size int64 `json:"size"` + CreateAt string `json:"createAt"` + UpdateAt string `json:"updateAt"` + FileId int64 `json:"fileId"` + Type int `json:"type"` + Etag string `json:"etag"` + S3KeyFlag string `json:"s3KeyFlag"` + ParentFileId int `json:"parentFileId"` + Category int `json:"category"` + Status int `json:"status"` + Trashed int `json:"trashed"` +} + +func (f File) GetID() string { + return fmt.Sprint(f.FileId) +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) IsDir() bool { + return f.Type == 1 +} + +func (f File) GetModified() string { + return f.UpdateAt +} + +func (f File) GetThumb() string { + return "" +} + +func (f File) ModTime() time.Time { + t, err := time.Parse("2006-01-02 15:04:05", f.UpdateAt) + if err != nil { + return time.Time{} + } + return t +} + +func (f File) CreateTime() time.Time { + t, err := time.Parse("2006-01-02 15:04:05", f.CreateAt) + if err != nil { + return time.Time{} + } + return t +} + +func (f File) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.MD5, f.Etag) +} + +func (f File) GetPath() string { + return "" +} diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go new file mode 100644 index 00000000000..76e8ead46f8 --- /dev/null +++ b/drivers/123_open/upload.go @@ -0,0 +1,282 @@ +package _123Open + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "golang.org/x/sync/errgroup" + "io" + "mime/multipart" + "net/http" + "runtime" + "strconv" + "time" +) + +func (d *Open123) create(parentFileID int64, filename, etag string, size int64, duplicate int, containDir bool) (*UploadCreateResp, error) { + var resp UploadCreateResp + + _, err := d.Request(ApiCreateUploadURL, http.MethodPost, func(req *resty.Request) { + body := base.Json{ + "parentFileID": parentFileID, + "filename": filename, + "etag": etag, + "size": size, + } + if duplicate > 0 { + body["duplicate"] = duplicate + } + if containDir { + body["containDir"] = true + } + req.SetBody(body) + }, &resp) + + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Open123) GetUploadDomains() ([]string, error) { + var resp struct { + Code int `json:"code"` + Message string `json:"message"` + Data []string `json:"data"` + } + + _, err := d.Request(ApiUploadDomainURL, http.MethodGet, nil, &resp) + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("get upload domain failed: %s", resp.Message) + } + return resp.Data, nil +} + +func (d *Open123) UploadSingle(ctx context.Context, createResp *UploadCreateResp, file model.FileStreamer, parentID int64) error { + domain := createResp.Data.Servers[0] + + etag := file.GetHash().GetHash(utils.MD5) + if len(etag) < utils.MD5.Width { + _, _, err := stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + reader = driver.NewLimitedUploadStream(ctx, reader) + + var b bytes.Buffer + mw := multipart.NewWriter(&b) + mw.WriteField("parentFileID", fmt.Sprint(parentID)) + mw.WriteField("filename", file.GetName()) + mw.WriteField("etag", etag) + mw.WriteField("size", fmt.Sprint(file.GetSize())) + fw, _ := mw.CreateFormFile("file", file.GetName()) + _, err = io.Copy(fw, reader) + mw.Close() + + req, err := http.NewRequestWithContext(ctx, "POST", domain+ApiSingleUploadURL, &b) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.tm.accessToken) + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", mw.FormDataContentType()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + var result struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + FileID int64 `json:"fileID"` + Completed bool `json:"completed"` + } `json:"data"` + } + body, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(body, &result); err != nil { + return fmt.Errorf("unmarshal response error: %v, body: %s", err, string(body)) + } + if result.Code != 0 { + return fmt.Errorf("upload failed: %s", result.Message) + } + if !result.Data.Completed || result.Data.FileID == 0 { + return fmt.Errorf("upload incomplete or missing fileID") + } + return nil +} + +func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, parentID int64, createResp *UploadCreateResp, up driver.UpdateProgress) error { + if cacher, ok := file.(interface{ CacheFullInTempFile() (model.File, error) }); ok { + if _, err := cacher.CacheFullInTempFile(); err != nil { + return err + } + } + + size := file.GetSize() + chunkSize := createResp.Data.SliceSize + uploadNums := (size + chunkSize - 1) / chunkSize + uploadDomain := createResp.Data.Servers[0] + + if d.UploadThread <= 0 { + cpuCores := runtime.NumCPU() + threads := cpuCores * 2 + if threads < 4 { + threads = 4 + } + if threads > 16 { + threads = 16 + } + d.UploadThread = threads + fmt.Printf("[Upload] Auto set upload concurrency: %d (CPU cores=%d)\n", d.UploadThread, cpuCores) + } + + fmt.Printf("[Upload] File size: %d bytes, chunk size: %d bytes, total slices: %d, concurrency: %d\n", + size, chunkSize, uploadNums, d.UploadThread) + + if size <= 1<<30 { + return d.UploadSingle(ctx, createResp, file, parentID) + } + + if createResp.Data.Reuse { + up(100) + return nil + } + + client := resty.New() + semaphore := make(chan struct{}, d.UploadThread) + threadG, _ := errgroup.WithContext(ctx) + + var progressArr = make([]int64, uploadNums) + + for partIndex := int64(0); partIndex < uploadNums; partIndex++ { + partIndex := partIndex + semaphore <- struct{}{} + + threadG.Go(func() error { + defer func() { <-semaphore }() + offset := partIndex * chunkSize + length := min(chunkSize, size-offset) + partNumber := partIndex + 1 + + fmt.Printf("[Slice %d] Starting read from offset %d, length %d\n", partNumber, offset, length) + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: length}) + if err != nil { + return fmt.Errorf("[Slice %d] RangeRead error: %v", partNumber, err) + } + + buf := make([]byte, length) + n, err := io.ReadFull(reader, buf) + if err != nil && err != io.EOF { + return fmt.Errorf("[Slice %d] Read error: %v", partNumber, err) + } + buf = buf[:n] + hash := md5.Sum(buf) + sliceMD5Str := hex.EncodeToString(hash[:]) + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + writer.WriteField("preuploadID", createResp.Data.PreuploadID) + writer.WriteField("sliceNo", strconv.FormatInt(partNumber, 10)) + writer.WriteField("sliceMD5", sliceMD5Str) + partName := fmt.Sprintf("%s.part%d", file.GetName(), partNumber) + fw, _ := writer.CreateFormFile("slice", partName) + fw.Write(buf) + writer.Close() + + resp, err := client.R(). + SetHeader("Authorization", "Bearer "+d.tm.accessToken). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", writer.FormDataContentType()). + SetBody(body.Bytes()). + Post(uploadDomain + ApiUploadSliceURL) + + if err != nil { + return fmt.Errorf("[Slice %d] Upload HTTP error: %v", partNumber, err) + } + if resp.StatusCode() != 200 { + return fmt.Errorf("[Slice %d] Upload failed with status: %s, resp: %s", partNumber, resp.Status(), resp.String()) + } + + progressArr[partIndex] = length + var totalUploaded int64 = 0 + for _, v := range progressArr { + totalUploaded += v + } + if up != nil { + percent := float64(totalUploaded) / float64(size) * 100 + up(percent) + } + + fmt.Printf("[Slice %d] MD5: %s\n", partNumber, sliceMD5Str) + fmt.Printf("[Slice %d] Upload finished\n", partNumber) + return nil + }) + } + + if err := threadG.Wait(); err != nil { + return err + } + + var completeResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Completed bool `json:"completed"` + FileID int64 `json:"fileID"` + } `json:"data"` + } + + for { + reqBody := fmt.Sprintf(`{"preuploadID":"%s"}`, createResp.Data.PreuploadID) + req, err := http.NewRequestWithContext(ctx, "POST", uploadDomain+ApiUploadCompleteURL, bytes.NewBufferString(reqBody)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.tm.accessToken) + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + if err := json.Unmarshal(body, &completeResp); err != nil { + return fmt.Errorf("completion response unmarshal error: %v, body: %s", err, string(body)) + } + if completeResp.Code != 0 { + return fmt.Errorf("completion API returned error code %d: %s", completeResp.Code, completeResp.Message) + } + if completeResp.Data.Completed && completeResp.Data.FileID != 0 { + fmt.Printf("[Upload] Upload completed successfully. FileID: %d\n", completeResp.Data.FileID) + break + } + time.Sleep(time.Second) + } + up(100) + return nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go new file mode 100644 index 00000000000..429a5e5dda8 --- /dev/null +++ b/drivers/123_open/util.go @@ -0,0 +1,20 @@ +package _123Open + +import ( + "fmt" + "net/http" +) + +func (d *Open123) getFiles(parentFileId int64, limit int, lastFileId int64) (*FileListResp, error) { + var result FileListResp + url := fmt.Sprintf("%s?parentFileId=%d&limit=%d&lastFileId=%d", ApiFileList, parentFileId, limit, lastFileId) + + _, err := d.Request(url, http.MethodGet, nil, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("list error: %s", result.Message) + } + return &result, nil +} diff --git a/drivers/all.go b/drivers/all.go index 224fb8ddb4b..a8c8620989e 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -6,6 +6,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/115_share" _ "github.com/alist-org/alist/v3/drivers/123" _ "github.com/alist-org/alist/v3/drivers/123_link" + _ "github.com/alist-org/alist/v3/drivers/123_open" _ "github.com/alist-org/alist/v3/drivers/123_share" _ "github.com/alist-org/alist/v3/drivers/139" _ "github.com/alist-org/alist/v3/drivers/189" diff --git a/internal/errs/driver.go b/internal/errs/driver.go index 4b6b5cac48e..7f67c0e2c2d 100644 --- a/internal/errs/driver.go +++ b/internal/errs/driver.go @@ -4,4 +4,5 @@ import "errors" var ( EmptyToken = errors.New("empty token") + LinkIsDir = errors.New("link is dir") ) diff --git a/internal/model/obj.go b/internal/model/obj.go index f0fce7a133a..93fa7a96475 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -55,6 +55,21 @@ type FileStreamer interface { type UpdateProgress func(percentage float64) +// Reference implementation from OpenListTeam: +// https://github.com/OpenListTeam/OpenList/blob/a703b736c9346c483bae56905a39bc07bf781cff/internal/model/obj.go#L58 +func UpdateProgressWithRange(inner UpdateProgress, start, end float64) UpdateProgress { + return func(p float64) { + if p < 0 { + p = 0 + } + if p > 100 { + p = 100 + } + scaled := start + (end-start)*(p/100.0) + inner(scaled) + } +} + type URL interface { URL() string } From 85fe4e5bb3455e70ec3b9114d3370c35019d702c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 4 Aug 2025 12:02:45 +0800 Subject: [PATCH 540/659] feat(alist_v3): add IntSlice type for JSON unmarshalling (#9247) - Add `IntSlice` type to handle both single int and array in JSON. - Modify `MeResp` struct to use `IntSlice` for `Role` field. - Import `encoding/json` for JSON operations. --- drivers/alist_v3/types.go | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go index 83ecde8be17..3e8e2f71eac 100644 --- a/drivers/alist_v3/types.go +++ b/drivers/alist_v3/types.go @@ -1,6 +1,7 @@ package alist_v3 import ( + "encoding/json" "time" "github.com/alist-org/alist/v3/internal/model" @@ -72,15 +73,15 @@ type LoginResp struct { } type MeResp struct { - Id int `json:"id"` - Username string `json:"username"` - Password string `json:"password"` - BasePath string `json:"base_path"` - Role []int `json:"role"` - Disabled bool `json:"disabled"` - Permission int `json:"permission"` - SsoId string `json:"sso_id"` - Otp bool `json:"otp"` + Id int `json:"id"` + Username string `json:"username"` + Password string `json:"password"` + BasePath string `json:"base_path"` + Role IntSlice `json:"role"` + Disabled bool `json:"disabled"` + Permission int `json:"permission"` + SsoId string `json:"sso_id"` + Otp bool `json:"otp"` } type ArchiveMetaReq struct { @@ -168,3 +169,17 @@ type DecompressReq struct { PutIntoNewDir bool `json:"put_into_new_dir"` SrcDir string `json:"src_dir"` } + +type IntSlice []int + +func (s *IntSlice) UnmarshalJSON(data []byte) error { + if len(data) > 0 && data[0] == '[' { + return json.Unmarshal(data, (*[]int)(s)) + } + var single int + if err := json.Unmarshal(data, &single); err != nil { + return err + } + *s = []int{single} + return nil +} From 6b2d81eede823fd7e5e56d4da78da815a4fe1434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Wed, 6 Aug 2025 16:31:36 +0800 Subject: [PATCH 541/659] feat(user): enhance path management and role handling (#9249) - Add `GetUsersByRole` function for fetching users by role. - Introduce `GetAllBasePathsFromRoles` to aggregate paths from roles. - Refine path handling in `pkg/utils/path.go` for normalization. - Comment out base path prefix updates to simplify role operations. --- internal/model/user.go | 35 +++++++++++++++++++++++++++++++++-- internal/op/role.go | 41 ++++++++++++++++++++--------------------- internal/op/storage.go | 20 ++++++++++++++------ internal/op/user.go | 26 +++++++++++++++----------- pkg/utils/path.go | 7 +++++++ 5 files changed, 89 insertions(+), 40 deletions(-) diff --git a/internal/model/user.go b/internal/model/user.go index 0b9e576a484..221747a407e 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -149,9 +149,21 @@ func (u *User) JoinPath(reqPath string) (string, error) { if err != nil { return "", err } - if u.CheckPathLimit() && !utils.IsSubPath(u.BasePath, path) { - return "", errs.PermissionDenied + + if u.CheckPathLimit() { + basePaths := GetAllBasePathsFromRoles(u) + match := false + for _, base := range basePaths { + if utils.IsSubPath(base, path) { + match = true + break + } + } + if !match { + return "", errs.PermissionDenied + } } + return path, nil } @@ -193,3 +205,22 @@ func (u *User) WebAuthnCredentials() []webauthn.Credential { func (u *User) WebAuthnIcon() string { return "https://alistgo.com/logo.svg" } + +// GetAllBasePathsFromRoles returns all permission paths from user's roles +func GetAllBasePathsFromRoles(u *User) []string { + basePaths := make([]string, 0) + seen := make(map[string]struct{}) + + for _, role := range u.RolesDetail { + for _, entry := range role.PermissionScopes { + if entry.Path == "" { + continue + } + if _, ok := seen[entry.Path]; !ok { + basePaths = append(basePaths, entry.Path) + seen[entry.Path] = struct{}{} + } + } + } + return basePaths +} diff --git a/internal/op/role.go b/internal/op/role.go index c719c6f4cdf..e24ba5ab1a8 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -2,7 +2,6 @@ package op import ( "fmt" - "github.com/pkg/errors" "time" "github.com/Xhofe/go-cache" @@ -106,26 +105,26 @@ func UpdateRole(r *model.Role) error { for i := range r.PermissionScopes { r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) } - if len(old.PermissionScopes) > 0 && len(r.PermissionScopes) > 0 && - old.PermissionScopes[0].Path != r.PermissionScopes[0].Path { - - oldPath := old.PermissionScopes[0].Path - newPath := r.PermissionScopes[0].Path - - users, err := db.GetUsersByRole(int(r.ID)) - if err != nil { - return errors.WithMessage(err, "failed to get users by role") - } - - modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath, users) - if err != nil { - return errors.WithMessage(err, "failed to update user base path when role updated") - } - - for _, name := range modifiedUsernames { - userCache.Del(name) - } - } + //if len(old.PermissionScopes) > 0 && len(r.PermissionScopes) > 0 && + // old.PermissionScopes[0].Path != r.PermissionScopes[0].Path { + // + // oldPath := old.PermissionScopes[0].Path + // newPath := r.PermissionScopes[0].Path + // + // users, err := db.GetUsersByRole(int(r.ID)) + // if err != nil { + // return errors.WithMessage(err, "failed to get users by role") + // } + // + // modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath, users) + // if err != nil { + // return errors.WithMessage(err, "failed to update user base path when role updated") + // } + // + // for _, name := range modifiedUsernames { + // userCache.Del(name) + // } + //} roleCache.Del(fmt.Sprint(r.ID)) roleCache.Del(r.Name) return db.UpdateRole(r) diff --git a/internal/op/storage.go b/internal/op/storage.go index 2833afa84ca..5ab1da1840a 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -232,12 +232,20 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { roleCache.Del(fmt.Sprint(id)) } - modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldStorage.MountPath, storage.MountPath) - if err != nil { - return errors.WithMessage(err, "failed to update user base path") - } - for _, name := range modifiedUsernames { - userCache.Del(name) + //modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldStorage.MountPath, storage.MountPath) + //if err != nil { + // return errors.WithMessage(err, "failed to update user base path") + //} + for _, id := range modifiedRoleIDs { + roleCache.Del(fmt.Sprint(id)) + + users, err := db.GetUsersByRole(int(id)) + if err != nil { + return errors.WithMessage(err, "failed to get users by role") + } + for _, user := range users { + userCache.Del(user.Username) + } } } if err != nil { diff --git a/internal/op/user.go b/internal/op/user.go index 942daae65d1..44b19db3508 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -50,6 +50,10 @@ func GetUserByRole(role int) (*model.User, error) { return db.GetUserByRole(role) } +func GetUsersByRole(role int) ([]model.User, error) { + return db.GetUsersByRole(role) +} + func GetUserByName(username string) (*model.User, error) { if username == "" { return nil, errs.EmptyUsername @@ -124,17 +128,17 @@ func UpdateUser(u *model.User) error { } userCache.Del(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) - if len(u.Role) > 0 { - roles, err := GetRolesByUserID(u.ID) - if err == nil { - for _, role := range roles { - if len(role.PermissionScopes) > 0 { - u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) - break - } - } - } - } + //if len(u.Role) > 0 { + // roles, err := GetRolesByUserID(u.ID) + // if err == nil { + // for _, role := range roles { + // if len(role.PermissionScopes) > 0 { + // u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + // break + // } + // } + // } + //} return db.UpdateUser(u) } diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 135f8e4ebca..6f3a55fc3d3 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -88,6 +88,13 @@ func JoinBasePath(basePath, reqPath string) (string, error) { strings.Contains(reqPath, "/../") { return "", errs.RelativePath } + + reqPath = FixAndCleanPath(reqPath) + + if strings.HasPrefix(reqPath, "/") { + return reqPath, nil + } + return stdpath.Join(FixAndCleanPath(basePath), FixAndCleanPath(reqPath)), nil } From aea3ba1499d56cb3de39da7f57c973197801af8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 Aug 2025 08:09:00 -0700 Subject: [PATCH 542/659] feat: add tag backup and fix bugs (#9265) * feat(label): enhance label file binding and router setup (feat/add-tag-backup) - Add `GetLabelsByFileNamesPublic` to retrieve labels using file names. - Refactor router setup for label and file binding routes. - Improve `toObjsResp` for efficient label retrieval by file names. - Comment out unnecessary user ID parameter in `toObjsResp`. * feat(label): enhance label file binding and router setup - Add `GetLabelsByFileNamesPublic` for label retrieval by file names. - Refactor router setup for label and file binding routes. - Improve `toObjsResp` for efficient label retrieval by file names. - Comment out unnecessary user ID parameter in `toObjsResp`. * refactor(db): comment out debug print in GetLabelIds (#feat/add-tag-backup) - Comment out debug print statement in GetLabelIds to clean up logs. - Enhance code readability by removing unnecessary debug output. * feat(label-file-binding): add batch creation and improve label ID handling - Introduced `CreateLabelFileBinDingBatch` API for batch label binding. - Added `collectLabelIDs` helper function to handle label ID parsing. - Enhanced label ID handling to support varied delimiters and input formats. - Refactored `CreateLabelFileBinDing` logic for improved code readability. - Updated router to include `POST /label_file_binding/create_batch`. --- internal/db/db.go | 2 +- internal/db/label_file_binding.go | 148 ++++++++++++++++++++++++-- internal/errs/role.go | 2 +- internal/model/label_file_binding.go | 2 +- internal/op/label_file_binding.go | 58 +++++++++-- server/handles/fsread.go | 20 +++- server/handles/label_file_binding.go | 149 ++++++++++++++++++++++++++- server/handles/user.go | 8 +- server/router.go | 20 +++- 9 files changed, 375 insertions(+), 34 deletions(-) diff --git a/internal/db/db.go b/internal/db/db.go index c6491dc984f..0d8ab42130a 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinDing), new(model.ObjFile)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/label_file_binding.go b/internal/db/label_file_binding.go index ec722efb979..4dda80f2c42 100644 --- a/internal/db/label_file_binding.go +++ b/internal/db/label_file_binding.go @@ -1,15 +1,18 @@ package db import ( + "fmt" "github.com/alist-org/alist/v3/internal/model" "github.com/pkg/errors" "gorm.io/gorm" + "gorm.io/gorm/clause" "time" ) // GetLabelIds Get all label_ids from database order by file_name func GetLabelIds(userId uint, fileName string) ([]uint, error) { - labelFileBinDingDB := db.Model(&model.LabelFileBinDing{}) + //fmt.Printf(">>> [GetLabelIds] userId: %d, fileName: %s\n", userId, fileName) + labelFileBinDingDB := db.Model(&model.LabelFileBinding{}) var labelIds []uint if err := labelFileBinDingDB.Where("file_name = ?", fileName).Where("user_id = ?", userId).Pluck("label_id", &labelIds).Error; err != nil { return nil, errors.WithStack(err) @@ -18,7 +21,7 @@ func GetLabelIds(userId uint, fileName string) ([]uint, error) { } func CreateLabelFileBinDing(fileName string, labelId, userId uint) error { - var labelFileBinDing model.LabelFileBinDing + var labelFileBinDing model.LabelFileBinding labelFileBinDing.UserId = userId labelFileBinDing.LabelId = labelId labelFileBinDing.FileName = fileName @@ -32,7 +35,7 @@ func CreateLabelFileBinDing(fileName string, labelId, userId uint) error { // GetLabelFileBinDingByLabelIdExists Get Label by label_id, used to del label usually func GetLabelFileBinDingByLabelIdExists(labelId, userId uint) bool { - var labelFileBinDing model.LabelFileBinDing + var labelFileBinDing model.LabelFileBinding result := db.Where("label_id = ?", labelId).Where("user_id = ?", userId).First(&labelFileBinDing) exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) return exists @@ -40,17 +43,150 @@ func GetLabelFileBinDingByLabelIdExists(labelId, userId uint) bool { // DelLabelFileBinDingByFileName used to del usually func DelLabelFileBinDingByFileName(userId uint, fileName string) error { - return errors.WithStack(db.Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinDing{}).Error) + return errors.WithStack(db.Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinding{}).Error) } // DelLabelFileBinDingById used to del usually func DelLabelFileBinDingById(labelId, userId uint, fileName string) error { - return errors.WithStack(db.Where("label_id = ?", labelId).Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinDing{}).Error) + return errors.WithStack(db.Where("label_id = ?", labelId).Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinding{}).Error) } -func GetLabelFileBinDingByLabelId(labelIds []uint, userId uint) (result []model.LabelFileBinDing, err error) { +func GetLabelFileBinDingByLabelId(labelIds []uint, userId uint) (result []model.LabelFileBinding, err error) { if err := db.Where("label_id in (?)", labelIds).Where("user_id = ?", userId).Find(&result).Error; err != nil { return nil, errors.WithStack(err) } return result, nil } + +func GetLabelBindingsByFileNamesPublic(fileNames []string) (map[string][]uint, error) { + var binds []model.LabelFileBinding + if err := db.Where("file_name IN ?", fileNames).Find(&binds).Error; err != nil { + return nil, errors.WithStack(err) + } + out := make(map[string][]uint, len(fileNames)) + seen := make(map[string]struct{}, len(binds)) + for _, b := range binds { + key := fmt.Sprintf("%s-%d", b.FileName, b.LabelId) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out[b.FileName] = append(out[b.FileName], b.LabelId) + } + return out, nil +} + +func GetLabelsByFileNamesPublic(fileNames []string) (map[string][]model.Label, error) { + bindMap, err := GetLabelBindingsByFileNamesPublic(fileNames) + if err != nil { + return nil, err + } + + idSet := make(map[uint]struct{}) + for _, ids := range bindMap { + for _, id := range ids { + idSet[id] = struct{}{} + } + } + if len(idSet) == 0 { + return make(map[string][]model.Label, 0), nil + } + allIDs := make([]uint, 0, len(idSet)) + for id := range idSet { + allIDs = append(allIDs, id) + } + labels, err := GetLabelByIds(allIDs) // 你已有的函数 + if err != nil { + return nil, err + } + + labelByID := make(map[uint]model.Label, len(labels)) + for _, l := range labels { + labelByID[l.ID] = l + } + + out := make(map[string][]model.Label, len(bindMap)) + for fname, ids := range bindMap { + for _, id := range ids { + if lab, ok := labelByID[id]; ok { + out[fname] = append(out[fname], lab) + } + } + } + return out, nil +} + +func ListLabelFileBinDing(userId uint, labelIDs []uint, fileName string, page, pageSize int) ([]model.LabelFileBinding, int64, error) { + q := db.Model(&model.LabelFileBinding{}).Where("user_id = ?", userId) + + if len(labelIDs) > 0 { + q = q.Where("label_id IN ?", labelIDs) + } + if fileName != "" { + q = q.Where("file_name LIKE ?", "%"+fileName+"%") + } + + var total int64 + if err := q.Count(&total).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + + var rows []model.LabelFileBinding + if err := q. + Order("id DESC"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&rows).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return rows, total, nil +} + +func RestoreLabelFileBindings(bindings []model.LabelFileBinding, keepIDs bool, override bool) error { + if len(bindings) == 0 { + return nil + } + tx := db.Begin() + + if override { + type key struct { + uid uint + name string + } + toDel := make(map[key]struct{}, len(bindings)) + for i := range bindings { + k := key{uid: bindings[i].UserId, name: bindings[i].FileName} + toDel[k] = struct{}{} + } + for k := range toDel { + if err := tx.Where("user_id = ? AND file_name = ?", k.uid, k.name). + Delete(&model.LabelFileBinding{}).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } + } + + for i := range bindings { + b := bindings[i] + if !keepIDs { + b.ID = 0 + } + if b.CreateTime.IsZero() { + b.CreateTime = time.Now() + } + if override { + if err := tx.Create(&b).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } else { + if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&b).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } + } + + return errors.WithStack(tx.Commit().Error) +} diff --git a/internal/errs/role.go b/internal/errs/role.go index fbd67404822..a818ea21264 100644 --- a/internal/errs/role.go +++ b/internal/errs/role.go @@ -3,5 +3,5 @@ package errs import "errors" var ( - ErrChangeDefaultRole = errors.New("cannot modify admin or guest role") + ErrChangeDefaultRole = errors.New("cannot modify admin role") ) diff --git a/internal/model/label_file_binding.go b/internal/model/label_file_binding.go index 3f9ea3b2271..af57fed4d88 100644 --- a/internal/model/label_file_binding.go +++ b/internal/model/label_file_binding.go @@ -2,7 +2,7 @@ package model import "time" -type LabelFileBinDing struct { +type LabelFileBinding struct { ID uint `json:"id" gorm:"primaryKey"` // unique key UserId uint `json:"user_id"` // use to user_id LabelId uint `json:"label_id"` // use to label_id diff --git a/internal/op/label_file_binding.go b/internal/op/label_file_binding.go index 79137ed38af..2802f0c0b38 100644 --- a/internal/op/label_file_binding.go +++ b/internal/op/label_file_binding.go @@ -23,6 +23,7 @@ type CreateLabelFileBinDingReq struct { Type int `json:"type"` HashInfoStr string `json:"hashinfo"` LabelIds string `json:"label_ids"` + LabelIDs []uint64 `json:"labelIdList"` } type ObjLabelResp struct { @@ -54,23 +55,29 @@ func GetLabelByFileName(userId uint, fileName string) ([]model.Label, error) { return labels, nil } +func GetLabelsByFileNamesPublic(fileNames []string) (map[string][]model.Label, error) { + return db.GetLabelsByFileNamesPublic(fileNames) +} + func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { if err := db.DelLabelFileBinDingByFileName(userId, req.Name); err != nil { return errors.WithMessage(err, "failed del label_file_bin_ding in database") } - if req.LabelIds == "" { + + ids, err := collectLabelIDs(req) + if err != nil { + return err + } + if len(ids) == 0 { return nil } - labelMap := strings.Split(req.LabelIds, ",") - for _, value := range labelMap { - labelId, err := strconv.ParseUint(value, 10, 64) - if err != nil { - return fmt.Errorf("invalid label ID '%s': %v", value, err) - } - if err = db.CreateLabelFileBinDing(req.Name, uint(labelId), userId); err != nil { + + for _, id := range ids { + if err = db.CreateLabelFileBinDing(req.Name, uint(id), userId); err != nil { return errors.WithMessage(err, "failed labels in database") } } + if !db.GetFileByNameExists(req.Name) { objFile := model.ObjFile{ Id: req.Id, @@ -86,8 +93,7 @@ func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { Type: req.Type, HashInfoStr: req.HashInfoStr, } - err := db.CreateObjFile(objFile) - if err != nil { + if err := db.CreateObjFile(objFile); err != nil { return errors.WithMessage(err, "failed file in database") } } @@ -97,7 +103,7 @@ func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { func GetFileByLabel(userId uint, labelId string) (result []ObjLabelResp, err error) { labelMap := strings.Split(labelId, ",") var labelIds []uint - var labelsFile []model.LabelFileBinDing + var labelsFile []model.LabelFileBinding var labels []model.Label var labelsFileMap = make(map[string][]model.Label) var labelsMap = make(map[uint]model.Label) @@ -157,3 +163,33 @@ func StringSliceToUintSlice(strSlice []string) ([]uint, error) { } return uintSlice, nil } + +func RestoreLabelFileBindings(bindings []model.LabelFileBinding, keepIDs bool, override bool) error { + return db.RestoreLabelFileBindings(bindings, keepIDs, override) +} + +func collectLabelIDs(req CreateLabelFileBinDingReq) ([]uint64, error) { + if len(req.LabelIDs) > 0 { + return req.LabelIDs, nil + } + s := strings.TrimSpace(req.LabelIds) + if s == "" { + return nil, nil + } + replacer := strings.NewReplacer("，", ",", "、", ",", "；", ",", ";", ",") + s = replacer.Replace(s) + parts := strings.Split(s, ",") + ids := make([]uint64, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + id, err := strconv.ParseUint(p, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid label ID '%s': %v", p, err) + } + ids = append(ids, id) + } + return ids, nil +} diff --git a/server/handles/fsread.go b/server/handles/fsread.go index b49f0b646b9..cc403c4aeb1 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -114,7 +114,7 @@ func FsList(c *gin.Context) { provider = storage.GetStorage().Driver } common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath), user.ID), + Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), Total: int64(total), Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), @@ -224,12 +224,22 @@ func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { return total, objs[start:end] } -func toObjsResp(objs []model.Obj, parent string, encrypt bool, userId uint) []ObjLabelResp { +func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjLabelResp { var resp []ObjLabelResp + + names := make([]string, 0, len(objs)) + for _, obj := range objs { + if !obj.IsDir() { + names = append(names, obj.GetName()) + } + } + + labelsByName, _ := op.GetLabelsByFileNamesPublic(names) + for _, obj := range objs { var labels []model.Label - if obj.IsDir() == false { - labels, _ = op.GetLabelByFileName(userId, obj.GetName()) + if !obj.IsDir() { + labels = labelsByName[obj.GetName()] } thumb, _ := model.GetThumb(obj) resp = append(resp, ObjLabelResp{ @@ -369,7 +379,7 @@ func FsGet(c *gin.Context) { Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, - Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath), user.ID), + Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), }) } diff --git a/server/handles/label_file_binding.go b/server/handles/label_file_binding.go index 78af929b34e..04f0c105fc2 100644 --- a/server/handles/label_file_binding.go +++ b/server/handles/label_file_binding.go @@ -8,7 +8,9 @@ import ( "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "net/url" "strconv" + "strings" ) type DelLabelFileBinDingReq struct { @@ -16,18 +18,36 @@ type DelLabelFileBinDingReq struct { LabelId string `json:"label_id"` } +type pageResp[T any] struct { + Content []T `json:"content"` + Total int64 `json:"total"` +} + +type restoreLabelBindingsReq struct { + KeepIDs bool `json:"keep_ids"` + Override bool `json:"override"` + Bindings []model.LabelFileBinding `json:"bindings"` +} + func GetLabelByFileName(c *gin.Context) { fileName := c.Query("file_name") if fileName == "" { common.ErrorResp(c, errors.New("file_name must not empty"), 400) return } + decodedFileName, err := url.QueryUnescape(fileName) + if err != nil { + common.ErrorResp(c, errors.New("invalid file_name"), 400) + return + } + fmt.Println(">>> 原始 fileName:", fileName) + fmt.Println(">>> 解码后 fileName:", decodedFileName) userObj, ok := c.Value("user").(*model.User) if !ok { common.ErrorStrResp(c, "user invalid", 401) return } - labels, err := op.GetLabelByFileName(userObj.ID, fileName) + labels, err := op.GetLabelByFileName(userObj.ID, decodedFileName) if err != nil { common.ErrorResp(c, err, 500, true) return @@ -101,3 +121,130 @@ func GetFileByLabel(c *gin.Context) { } common.SuccessResp(c, fileList) } + +func ListLabelFileBinding(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + + pageStr := c.DefaultQuery("page", "1") + sizeStr := c.DefaultQuery("page_size", "50") + page, err := strconv.Atoi(pageStr) + if err != nil || page <= 0 { + page = 1 + } + pageSize, err := strconv.Atoi(sizeStr) + if err != nil || pageSize <= 0 || pageSize > 200 { + pageSize = 50 + } + + fileName := c.Query("file_name") + labelIDStr := c.Query("label_id") + var labelIDs []uint + if labelIDStr != "" { + parts := strings.Split(labelIDStr, ",") + for _, p := range parts { + if p == "" { + continue + } + id64, err := strconv.ParseUint(strings.TrimSpace(p), 10, 64) + if err != nil { + common.ErrorResp(c, fmt.Errorf("invalid label_id '%s': %v", p, err), 400) + return + } + labelIDs = append(labelIDs, uint(id64)) + } + } + + list, total, err := db.ListLabelFileBinDing(userObj.ID, labelIDs, fileName, page, pageSize) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, pageResp[model.LabelFileBinding]{ + Content: list, + Total: total, + }) +} + +func RestoreLabelFileBinding(c *gin.Context) { + var req restoreLabelBindingsReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if len(req.Bindings) == 0 { + common.ErrorStrResp(c, "empty bindings", 400) + return + } + + if u, ok := c.Value("user").(*model.User); ok { + for i := range req.Bindings { + if req.Bindings[i].UserId == 0 { + req.Bindings[i].UserId = u.ID + } + } + } + + for i := range req.Bindings { + b := req.Bindings[i] + if b.UserId == 0 || b.LabelId == 0 || strings.TrimSpace(b.FileName) == "" { + common.ErrorStrResp(c, "invalid binding: user_id/label_id/file_name required", 400) + return + } + } + + if err := op.RestoreLabelFileBindings(req.Bindings, req.KeepIDs, req.Override); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, gin.H{ + "msg": fmt.Sprintf("restored %d rows", len(req.Bindings)), + }) +} + +func CreateLabelFileBinDingBatch(c *gin.Context) { + var req struct { + Items []op.CreateLabelFileBinDingReq `json:"items" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil || len(req.Items) == 0 { + common.ErrorResp(c, err, 400) + return + } + + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + + type perResult struct { + Name string `json:"name"` + Ok bool `json:"ok"` + ErrMsg string `json:"errMsg,omitempty"` + } + results := make([]perResult, 0, len(req.Items)) + succeed := 0 + + for _, item := range req.Items { + if item.IsDir { + results = append(results, perResult{Name: item.Name, Ok: false, ErrMsg: "Unable to bind folder"}) + continue + } + if err := op.CreateLabelFileBinDing(item, userObj.ID); err != nil { + results = append(results, perResult{Name: item.Name, Ok: false, ErrMsg: err.Error()}) + continue + } + succeed++ + results = append(results, perResult{Name: item.Name, Ok: true}) + } + + common.SuccessResp(c, gin.H{ + "total": len(req.Items), + "succeed": succeed, + "failed": len(req.Items) - succeed, + "results": results, + }) +} diff --git a/server/handles/user.go b/server/handles/user.go index d5eebba4780..b4c152c5a86 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -67,10 +67,10 @@ func UpdateUser(c *gin.Context) { common.ErrorStrResp(c, "cannot change role of admin user", 403) return } - if user.Username != req.Username { - common.ErrorStrResp(c, "cannot change username of admin user", 403) - return - } + //if user.Username != req.Username { + // common.ErrorStrResp(c, "cannot change username of admin user", 403) + // return + //} } if req.Password == "" { diff --git a/server/router.go b/server/router.go index bf43a6258f4..72546f4eb6b 100644 --- a/server/router.go +++ b/server/router.go @@ -92,6 +92,8 @@ func Init(e *gin.Engine) { _fs(auth.Group("/fs")) _task(auth.Group("/task", middlewares.AuthNotGuest)) + _label(auth.Group("/label")) + _labelFileBinding(auth.Group("/label_file_binding")) admin(auth.Group("/admin", middlewares.AuthAdmin)) if flags.Debug || flags.Dev { debug(g.Group("/debug")) @@ -170,17 +172,17 @@ func admin(g *gin.RouterGroup) { index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) label := g.Group("/label") - label.GET("/list", handles.ListLabel) - label.GET("/get", handles.GetLabel) label.POST("/create", handles.CreateLabel) label.POST("/update", handles.UpdateLabel) label.POST("/delete", handles.DeleteLabel) labelFileBinding := g.Group("/label_file_binding") - labelFileBinding.GET("/get", handles.GetLabelByFileName) - labelFileBinding.GET("/get_file_by_label", handles.GetFileByLabel) + labelFileBinding.GET("/list", handles.ListLabelFileBinding) labelFileBinding.POST("/create", handles.CreateLabelFileBinDing) + labelFileBinding.POST("/create_batch", handles.CreateLabelFileBinDingBatch) labelFileBinding.POST("/delete", handles.DelLabelByFileName) + labelFileBinding.POST("/restore", handles.RestoreLabelFileBinding) + } func _fs(g *gin.RouterGroup) { @@ -216,6 +218,16 @@ func _task(g *gin.RouterGroup) { handles.SetupTaskRoute(g) } +func _label(g *gin.RouterGroup) { + g.GET("/list", handles.ListLabel) + g.GET("/get", handles.GetLabel) +} + +func _labelFileBinding(g *gin.RouterGroup) { + g.GET("/get", handles.GetLabelByFileName) + g.GET("/get_file_by_label", handles.GetFileByLabel) +} + func Cors(r *gin.Engine) { config := cors.DefaultConfig() // config.AllowAllOrigins = true From fcfb3369d17150e6b09ed13f1da1b70caa0b3e0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 Aug 2025 08:10:55 -0700 Subject: [PATCH 543/659] fix: webdav error location (#9266) * feat: improve WebDAV permission handling and user role fetching - Added logic to handle root permissions in WebDAV requests. - Improved the user role fetching mechanism. - Enhanced path checks and permission scopes in role_perm.go. - Set FetchRole function to avoid import cycles between modules. * fix(webdav): resolve connection reset issue by encoding paths - Adjust path encoding in webdav.go to prevent connection reset. - Utilize utils.EncodePath for correct path formatting. - Ensure proper handling of directory paths with trailing slash. * fix(webdav): resolve connection reset issue by encoding paths - Adjust path encoding in webdav.go to prevent connection reset. - Utilize utils.FixAndCleanPath for correct path formatting. - Ensure proper handling of directory paths with trailing slash. * fix: resolve webdav handshake error in permission checks - Updated role permission logic to handle bidirectional subpaths. - This adjustment fixes the issue where remote host terminates the handshake due to improper path matching. * fix: resolve webdav handshake error in permission checks (fix/fix-webdav-error) - Updated role permission logic to handle bidirectional subpaths, fixing handshake termination by remote host due to path mismatch. - Refactored function naming for consistency and clarity. - Enhanced filtering of objects based on user permissions. * fix: resolve webdav handshake error in permission checks - Updated role permission logic to handle bidirectional subpaths, fixing handshake termination by remote host due to path mismatch. - Refactored function naming for consistency and clarity. - Enhanced filtering of objects based on user permissions. --- internal/model/user.go | 18 ++++++- internal/op/role.go | 4 ++ server/common/role_perm.go | 41 ++++++++++++--- server/handles/fsread.go | 18 ++++++- server/webdav.go | 6 ++- server/webdav/file.go | 4 ++ server/webdav/webdav.go | 101 ++++++++++++++++++++++++++++++++++++- 7 files changed, 180 insertions(+), 12 deletions(-) diff --git a/internal/model/user.go b/internal/model/user.go index 221747a407e..8ea1ef1aaff 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -145,12 +145,15 @@ func (u *User) CheckPathLimit() bool { } func (u *User) JoinPath(reqPath string) (string, error) { + if reqPath == "/" { + return utils.FixAndCleanPath(u.BasePath), nil + } path, err := utils.JoinBasePath(u.BasePath, reqPath) if err != nil { return "", err } - if u.CheckPathLimit() { + if path != "/" && u.CheckPathLimit() { basePaths := GetAllBasePathsFromRoles(u) match := false for _, base := range basePaths { @@ -206,12 +209,23 @@ func (u *User) WebAuthnIcon() string { return "https://alistgo.com/logo.svg" } +// FetchRole is used to load role details by id. It should be set by the op package +// to avoid an import cycle between model and op. +var FetchRole func(uint) (*Role, error) + // GetAllBasePathsFromRoles returns all permission paths from user's roles func GetAllBasePathsFromRoles(u *User) []string { basePaths := make([]string, 0) seen := make(map[string]struct{}) - for _, role := range u.RolesDetail { + for _, rid := range u.Role { + if FetchRole == nil { + continue + } + role, err := FetchRole(uint(rid)) + if err != nil || role == nil { + continue + } for _, entry := range role.PermissionScopes { if entry.Path == "" { continue diff --git a/internal/op/role.go b/internal/op/role.go index e24ba5ab1a8..4d187506417 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -15,6 +15,10 @@ import ( var roleCache = cache.NewMemCache[*model.Role](cache.WithShards[*model.Role](2)) var roleG singleflight.Group[*model.Role] +func init() { + model.FetchRole = GetRole +} + func GetRole(id uint) (*model.Role, error) { key := fmt.Sprint(id) if r, ok := roleCache.Get(key); ok { diff --git a/server/common/role_perm.go b/server/common/role_perm.go index 1e539d966c5..36dedf98c5e 100644 --- a/server/common/role_perm.go +++ b/server/common/role_perm.go @@ -43,17 +43,23 @@ func MergeRolePermissions(u *model.User, reqPath string) int32 { if err != nil { continue } - for _, entry := range role.PermissionScopes { - if utils.IsSubPath(entry.Path, reqPath) { + if reqPath == "/" || utils.PathEqual(reqPath, u.BasePath) { + for _, entry := range role.PermissionScopes { perm |= entry.Permission } + } else { + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(entry.Path, reqPath) { + perm |= entry.Permission + } + } } } return perm } func CanAccessWithRoles(u *model.User, meta *model.Meta, reqPath, password string) bool { - if !canReadPathByRole(u, reqPath) { + if !CanReadPathByRole(u, reqPath) { return false } perm := MergeRolePermissions(u, reqPath) @@ -78,7 +84,30 @@ func CanAccessWithRoles(u *model.User, meta *model.Meta, reqPath, password strin return meta.Password == password } -func canReadPathByRole(u *model.User, reqPath string) bool { +func CanReadPathByRole(u *model.User, reqPath string) bool { + if u == nil { + return false + } + if reqPath == "/" || utils.PathEqual(reqPath, u.BasePath) { + return len(u.Role) > 0 + } + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.PathEqual(entry.Path, reqPath) || utils.IsSubPath(entry.Path, reqPath) || utils.IsSubPath(reqPath, entry.Path) { + return true + } + } + } + return false +} + +// HasChildPermission checks whether any child path under reqPath grants the +// specified permission bit. +func HasChildPermission(u *model.User, reqPath string, bit uint) bool { if u == nil { return false } @@ -88,7 +117,7 @@ func canReadPathByRole(u *model.User, reqPath string) bool { continue } for _, entry := range role.PermissionScopes { - if utils.IsSubPath(entry.Path, reqPath) { + if utils.IsSubPath(reqPath, entry.Path) && HasPermission(entry.Permission, bit) { return true } } @@ -102,7 +131,7 @@ func canReadPathByRole(u *model.User, reqPath string) bool { func CheckPathLimitWithRoles(u *model.User, reqPath string) bool { perm := MergeRolePermissions(u, reqPath) if HasPermission(perm, PermPathLimit) { - return canReadPathByRole(u, reqPath) + return CanReadPathByRole(u, reqPath) } return true } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index cc403c4aeb1..8cf3c9b0290 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -107,7 +107,14 @@ func FsList(c *gin.Context) { common.ErrorResp(c, err, 500) return } - total, objs := pagination(objs, &req.PageReq) + filtered := make([]model.Obj, 0, len(objs)) + for _, obj := range objs { + childPath := stdpath.Join(reqPath, obj.GetName()) + if common.CanReadPathByRole(user, childPath) { + filtered = append(filtered, obj) + } + } + total, objs := pagination(filtered, &req.PageReq) provider := "unknown" storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) if err == nil { @@ -161,7 +168,14 @@ func FsDirs(c *gin.Context) { common.ErrorResp(c, err, 500) return } - dirs := filterDirs(objs) + visible := make([]model.Obj, 0, len(objs)) + for _, obj := range objs { + childPath := stdpath.Join(reqPath, obj.GetName()) + if common.CanReadPathByRole(user, childPath) { + visible = append(visible, obj) + } + } + dirs := filterDirs(visible) common.SuccessResp(c, dirs) } diff --git a/server/webdav.go b/server/webdav.go index a65896dfc79..582c469d73a 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -95,6 +95,9 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + if roles, err := op.GetRolesByUserID(user.ID); err == nil { + user.RolesDetail = roles + } reqPath := c.Param("path") if reqPath == "" { reqPath = "/" @@ -107,7 +110,8 @@ func WebDAVAuth(c *gin.Context) { return } perm := common.MergeRolePermissions(user, reqPath) - if user.Disabled || !common.HasPermission(perm, common.PermWebdavRead) { + webdavRead := common.HasPermission(perm, common.PermWebdavRead) + if user.Disabled || (!webdavRead && (c.Request.Method != "PROPFIND" || !common.HasChildPermission(user, reqPath, common.PermWebdavRead))) { if c.Request.Method == "OPTIONS" { c.Set("user", guest) c.Next() diff --git a/server/webdav/file.go b/server/webdav/file.go index ab78d26105d..419c7b07207 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -94,6 +94,7 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn depth = 0 } meta, _ := op.GetNearestMeta(name) + user := ctx.Value("user").(*model.User) // Read directory names. objs, err := fs.List(context.WithValue(ctx, "meta", meta), name, &fs.ListArgs{}) //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) @@ -108,6 +109,9 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn for _, fileInfo := range objs { filename := path.Join(name, fileInfo.GetName()) + if !common.CanReadPathByRole(user, filename) { + continue + } if err != nil { if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { return err diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index f22e15aadb9..dde73559f8a 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -648,6 +648,98 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status mw := multistatusWriter{w: w} + if utils.PathEqual(reqPath, user.BasePath) { + hasRootPerm := false + for _, role := range user.RolesDetail { + for _, entry := range role.PermissionScopes { + if utils.PathEqual(entry.Path, user.BasePath) { + hasRootPerm = true + break + } + } + if hasRootPerm { + break + } + } + if !hasRootPerm { + basePaths := model.GetAllBasePathsFromRoles(user) + type infoItem struct { + path string + info model.Obj + } + infos := []infoItem{{reqPath, fi}} + seen := make(map[string]struct{}) + for _, p := range basePaths { + if !utils.IsSubPath(user.BasePath, p) { + continue + } + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(p), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + dir := strings.Split(rel, "/")[0] + if dir == "" { + continue + } + if _, ok := seen[dir]; ok { + continue + } + seen[dir] = struct{}{} + sp := utils.FixAndCleanPath(path.Join(user.BasePath, dir)) + info, err := fs.Get(ctx, sp, &fs.GetArgs{}) + if err != nil { + continue + } + infos = append(infos, infoItem{sp, info}) + } + for _, item := range infos { + var pstats []Propstat + if pf.Propname != nil { + pnames, err := propnames(ctx, h.LockSystem, item.info) + if err != nil { + return http.StatusInternalServerError, err + } + pstat := Propstat{Status: http.StatusOK} + for _, xmlname := range pnames { + pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + } + pstats = append(pstats, pstat) + } else if pf.Allprop != nil { + pstats, err = allprop(ctx, h.LockSystem, item.info, pf.Prop) + if err != nil { + return http.StatusInternalServerError, err + } + } else { + pstats, err = props(ctx, h.LockSystem, item.info, pf.Prop) + if err != nil { + return http.StatusInternalServerError, err + } + } + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(item.path), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + href := utils.EncodePath(path.Join("/", h.Prefix, rel), true) + if href != "/" && item.info.IsDir() { + href += "/" + } + if err := mw.write(makePropstatResponse(href, pstats)); err != nil { + return http.StatusInternalServerError, err + } + } + if err := mw.close(); err != nil { + return http.StatusInternalServerError, err + } + return 0, nil + } + } + walkFn := func(reqPath string, info model.Obj, err error) error { if err != nil { return err @@ -671,7 +763,14 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status if err != nil { return err } - href := path.Join(h.Prefix, strings.TrimPrefix(reqPath, user.BasePath)) + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(reqPath), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + href := utils.EncodePath(path.Join("/", h.Prefix, rel), true) if href != "/" && info.IsDir() { href += "/" } From 97d4f79b96330ad782c9aa8f122b43d9a37fd6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 16 Aug 2025 05:55:17 -0700 Subject: [PATCH 544/659] fix: resolve webdav decode issue (#9268) * fix: resolve webdav handshake error in permission checks - Updated role permission logic to handle bidirectional subpaths, fixing handshake termination by remote host due to path mismatch. - Refactored function naming for consistency and clarity. - Enhanced filtering of objects based on user permissions. - Modified `makePropstatResponse` to preserve encoded href paths. - Added test for `makePropstatResponse` to ensure href encoding. * Delete server/webdav/makepropstatresponse_test.go * ci(workflow): set GOPROXY for Go builds on GitHub Actions - Use `GOPROXY=https://proxy.golang.org,direct` to speed up module downloads - Mitigates network flakiness (e.g., checksum DB timeouts/rate limits) - `,direct` provides fallback for private/unproxyable modules - No build logic changes; only affects dependency resolution across all matrix targets --------- Co-authored-by: AlistGo --- .github/workflows/build.yml | 2 ++ server/webdav/webdav.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2c934e7a5e..cf6eff39e48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,6 +25,8 @@ jobs: - android-arm64 name: Build runs-on: ${{ matrix.platform }} + env: + GOPROXY: https://proxy.golang.org,direct steps: - name: Checkout diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index dde73559f8a..93211e8a77e 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -833,7 +833,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu func makePropstatResponse(href string, pstats []Propstat) *response { resp := response{ - Href: []string{(&url.URL{Path: href}).EscapedPath()}, + Href: []string{href}, Propstat: make([]propstat, 0, len(pstats)), } for _, p := range pstats { From eca500861a8b74a2780789f595bc5f5a492b64ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 18 Aug 2025 16:38:21 +0800 Subject: [PATCH 545/659] feat: add user registration endpoint and role-based default settings (#9277) * feat(setting): add role-based default and registration settings (closed #feat/register-and-statistics) - Added `AllowRegister` and `DefaultRole` settings to site configuration. - Integrated dynamic role options for `DefaultRole` using `op.GetRoles`. - Updated `setting.go` handlers to manage `DefaultRole` options dynamically. - Modified `const.go` to include new site settings constants. - Updated dependencies in `go.mod` and `go.sum` to support new functionality. * feat(register-and-statistics): add user registration endpoint - Added `POST /auth/register` endpoint to support user registration. - Implemented registration logic in `auth.go` with dynamic role assignment. - Integrated settings `AllowRegister` and `DefaultRole` for registration flow. - Updated imports to include new modules: `conf`, `setting`. - Adjusted user creation logic to use `DefaultRole` setting dynamically. * feat(register-and-statistics): add user registration endpoint (#register-and-statistics) - Added `POST /auth/register` endpoint to support user registration. - Implemented registration logic in `auth.go` with dynamic role assignment. - Integrated `AllowRegister` and `DefaultRole` settings for registration flow. - Updated imports to include new modules: `conf`, `setting`. - Adjusted user creation logic to use `DefaultRole` dynamically. * feat(register-and-statistics): enhance role management logic (#register-and-statistics) - Refactored CreateRole and UpdateRole functions to handle default role. - Added dynamic role assignment logic in 'role.go' using conf settings. - Improved request handling in 'handles/role.go' with structured data. - Implemented default role logic in 'db/role.go' to update non-default roles. - Modified 'model/role.go' to include a 'Default' field for role management. * feat(register-and-statistics): enhance role management logic - Refactor CreateRole and UpdateRole to handle default roles. - Add dynamic role assignment using conf settings in 'role.go'. - Improve request handling with structured data in 'handles/role.go'. - Implement default role logic in 'db/role.go' for non-default roles. - Modify 'model/role.go' to include 'Default' field for role management. * feat(register-and-statistics): improve role handling logic - Switch from role names to role IDs for better consistency. - Update logic to prioritize "guest" for default role ID. - Adjust `DefaultRole` setting to use role IDs. - Refactor `getRoleOptions` to return role IDs as a comma-separated string. * feat(register-and-statistics): improve role handling logic --- go.mod | 9 ++---- go.sum | 13 ++++++-- internal/bootstrap/data/setting.go | 18 +++++++++++ internal/conf/const.go | 16 ++++++---- internal/db/role.go | 20 ++++++++++-- internal/model/role.go | 1 + internal/op/hook.go | 13 ++++++++ internal/op/role.go | 51 ++++++++++++++++++++++++++++-- server/handles/auth.go | 31 ++++++++++++++++++ server/handles/role.go | 20 +++++++++--- server/handles/setting.go | 30 ++++++++++++++++++ server/handles/user.go | 3 ++ server/router.go | 1 + 13 files changed, 202 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index e8afe0e7a62..5bc953fb4de 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/alist-org/alist/v3 go 1.23.4 require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 @@ -79,11 +81,7 @@ require ( gorm.io/gorm v1.25.11 ) -require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 // indirect -) +require github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect require ( github.com/STARRY-S/zip v0.2.1 // indirect @@ -109,7 +107,6 @@ require ( github.com/ipfs/boxo v0.12.0 // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/klauspost/pgzip v1.2.6 // indirect - github.com/kr/text v0.2.0 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 diff --git a/go.sum b/go.sum index 6fbaeb2b3ef..a9faa92fa69 100644 --- a/go.sum +++ b/go.sum @@ -21,10 +21,16 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -172,7 +178,6 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -398,6 +403,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/larksuite/oapi-sdk-go/v3 v3.3.1 h1:DLQQEgHUAGZB6RVlceB1f6A94O206exxW2RIMH+gMUc= github.com/larksuite/oapi-sdk-go/v3 v3.3.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= @@ -492,6 +499,8 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -739,8 +748,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index e00abf2d379..39c3b1be571 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -2,6 +2,7 @@ package data import ( "strconv" + "strings" "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" @@ -91,6 +92,21 @@ func InitialSettings() []model.SettingItem { } else { token = random.Token() } + roles, _, err := op.GetRoles(1, model.MaxInt) + if err != nil { + utils.Log.Fatalf("failed get roles: %+v", err) + } + roleNames := make([]string, len(roles)) + defaultRoleID := "" + for i, role := range roles { + roleNames[i] = role.Name + if role.Name == "guest" { + defaultRoleID = strconv.Itoa(int(role.ID)) + } + } + if defaultRoleID == "" && len(roles) > 0 { + defaultRoleID = strconv.Itoa(int(roles[0].ID)) + } initialSettingItems = []model.SettingItem{ // site settings {Key: conf.VERSION, Value: conf.Version, Type: conf.TypeString, Group: model.SITE, Flag: model.READONLY}, @@ -103,6 +119,8 @@ func InitialSettings() []model.SettingItem { {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, + {Key: conf.AllowRegister, Value: "false", Type: conf.TypeBool, Group: model.SITE}, + {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Options: strings.Join(roleNames, ","), Group: model.SITE}, // newui settings {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, // style settings diff --git a/internal/conf/const.go b/internal/conf/const.go index 48ac2037fb2..0bf0cd67f7e 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -10,13 +10,15 @@ const ( const ( // site - VERSION = "version" - SiteTitle = "site_title" - Announcement = "announcement" - AllowIndexed = "allow_indexed" - AllowMounted = "allow_mounted" - RobotsTxt = "robots_txt" - UseNewui = "use_newui" + VERSION = "version" + SiteTitle = "site_title" + Announcement = "announcement" + AllowIndexed = "allow_indexed" + AllowMounted = "allow_mounted" + RobotsTxt = "robots_txt" + AllowRegister = "allow_register" + DefaultRole = "default_role" + UseNewui = "use_newui" Logo = "logo" Favicon = "favicon" diff --git a/internal/db/role.go b/internal/db/role.go index e6d0d9568b4..ae62a8ed898 100644 --- a/internal/db/role.go +++ b/internal/db/role.go @@ -35,11 +35,27 @@ func GetRoles(pageIndex, pageSize int) (roles []model.Role, count int64, err err } func CreateRole(r *model.Role) error { - return errors.WithStack(db.Create(r).Error) + if err := db.Create(r).Error; err != nil { + return errors.WithStack(err) + } + if r.Default { + if err := db.Model(&model.Role{}).Where("id <> ?", r.ID).Update("default", false).Error; err != nil { + return errors.WithStack(err) + } + } + return nil } func UpdateRole(r *model.Role) error { - return errors.WithStack(db.Save(r).Error) + if err := db.Save(r).Error; err != nil { + return errors.WithStack(err) + } + if r.Default { + if err := db.Model(&model.Role{}).Where("id <> ?", r.ID).Update("default", false).Error; err != nil { + return errors.WithStack(err) + } + } + return nil } func DeleteRole(id uint) error { diff --git a/internal/model/role.go b/internal/model/role.go index ecc9aee29d1..87855551ddb 100644 --- a/internal/model/role.go +++ b/internal/model/role.go @@ -17,6 +17,7 @@ type Role struct { ID uint `json:"id" gorm:"primaryKey"` Name string `json:"name" gorm:"unique" binding:"required"` Description string `json:"description"` + Default bool `json:"default" gorm:"default:false"` // PermissionScopes stores structured permission list and is ignored by gorm. PermissionScopes []PermissionEntry `json:"permission_scopes" gorm:"-"` // RawPermission is the JSON representation of PermissionScopes stored in DB. diff --git a/internal/op/hook.go b/internal/op/hook.go index 23b8e59af2c..08ea4603e5c 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -2,6 +2,7 @@ package op import ( "regexp" + "strconv" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -82,6 +83,18 @@ var settingItemHooks = map[string]SettingItemHook{ conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") return nil }, + conf.DefaultRole: func(item *model.SettingItem) error { + v := strings.TrimSpace(item.Value) + if v == "" { + return nil + } + r, err := GetRoleByName(v) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + return nil + }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { diff --git a/internal/op/role.go b/internal/op/role.go index 4d187506417..5c9aad06f41 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -2,9 +2,11 @@ package op import ( "fmt" + "strconv" "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -50,6 +52,23 @@ func GetRoleByName(name string) (*model.Role, error) { return r, err } +func GetDefaultRoleID() int { + item, err := GetSettingItemByKey(conf.DefaultRole) + if err == nil && item != nil && item.Value != "" { + if id, err := strconv.Atoi(item.Value); err == nil && id != 0 { + return id + } + if r, err := db.GetRoleByName(item.Value); err == nil { + return int(r.ID) + } + } + var r model.Role + if err := db.GetDb().Where("`default` = ?", true).First(&r).Error; err == nil { + return int(r.ID) + } + return int(model.GUEST) +} + func GetRolesByUserID(userID uint) ([]model.Role, error) { user, err := GetUserById(userID) if err != nil { @@ -92,7 +111,21 @@ func CreateRole(r *model.Role) error { } roleCache.Del(fmt.Sprint(r.ID)) roleCache.Del(r.Name) - return db.CreateRole(r) + if err := db.CreateRole(r); err != nil { + return err + } + if r.Default { + roleCache.Clear() + item, err := GetSettingItemByKey(conf.DefaultRole) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + if err := SaveSettingItem(item); err != nil { + return err + } + } + return nil } func UpdateRole(r *model.Role) error { @@ -131,7 +164,21 @@ func UpdateRole(r *model.Role) error { //} roleCache.Del(fmt.Sprint(r.ID)) roleCache.Del(r.Name) - return db.UpdateRole(r) + if err := db.UpdateRole(r); err != nil { + return err + } + if r.Default { + roleCache.Clear() + item, err := GetSettingItemByKey(conf.DefaultRole) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + if err := SaveSettingItem(item); err != nil { + return err + } + } + return nil } func DeleteRole(id uint) error { diff --git a/server/handles/auth.go b/server/handles/auth.go index 96a9ba9e20d..26447ddd594 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -9,8 +9,10 @@ import ( "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" @@ -89,6 +91,35 @@ func loginHash(c *gin.Context, req *LoginReq) { loginCache.Del(ip) } +type RegisterReq struct { + Username string `json:"username" binding:"required"` + Password string `json:"password" binding:"required"` +} + +// Register a new user +func Register(c *gin.Context) { + if !setting.GetBool(conf.AllowRegister) { + common.ErrorStrResp(c, "registration is disabled", 403) + return + } + var req RegisterReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := &model.User{ + Username: req.Username, + Role: model.Roles{op.GetDefaultRoleID()}, + Authn: "[]", + } + user.SetPassword(req.Password) + if err := op.CreateUser(user); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + type UserResp struct { model.User Otp bool `json:"otp"` diff --git a/server/handles/role.go b/server/handles/role.go index 0d071c9f84d..17271a530de 100644 --- a/server/handles/role.go +++ b/server/handles/role.go @@ -44,7 +44,7 @@ func GetRole(c *gin.Context) { func CreateRole(c *gin.Context) { var req model.Role - if err := c.ShouldBind(&req); err != nil { + if err := c.ShouldBindJSON(&req); err != nil { common.ErrorResp(c, err, 400) return } @@ -56,8 +56,14 @@ func CreateRole(c *gin.Context) { } func UpdateRole(c *gin.Context) { - var req model.Role - if err := c.ShouldBind(&req); err != nil { + var req struct { + ID uint `json:"id"` + Name string `json:"name" binding:"required"` + Description string `json:"description"` + PermissionScopes []model.PermissionEntry `json:"permission_scopes"` + Default *bool `json:"default"` + } + if err := c.ShouldBindJSON(&req); err != nil { common.ErrorResp(c, err, 400) return } @@ -74,7 +80,13 @@ func UpdateRole(c *gin.Context) { case "guest": req.Name = "guest" } - if err := op.UpdateRole(&req); err != nil { + role.Name = req.Name + role.Description = req.Description + role.PermissionScopes = req.PermissionScopes + if req.Default != nil { + role.Default = *req.Default + } + if err := op.UpdateRole(role); err != nil { common.ErrorResp(c, err, 500, true) } else { common.SuccessResp(c) diff --git a/server/handles/setting.go b/server/handles/setting.go index f778b1803c5..3ce5fcbf94f 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -14,6 +14,18 @@ import ( "github.com/gin-gonic/gin" ) +func getRoleOptions() string { + roles, _, err := op.GetRoles(1, model.MaxInt) + if err != nil { + return "" + } + names := make([]string, len(roles)) + for i, r := range roles { + names[i] = r.Name + } + return strings.Join(names, ",") +} + func ResetToken(c *gin.Context) { token := random.Token() item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} @@ -34,6 +46,12 @@ func GetSetting(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if item.Key == conf.DefaultRole { + copy := *item + copy.Options = getRoleOptions() + common.SuccessResp(c, copy) + return + } common.SuccessResp(c, item) } else { items, err := op.GetSettingItemInKeys(strings.Split(keys, ",")) @@ -41,6 +59,12 @@ func GetSetting(c *gin.Context) { common.ErrorResp(c, err, 400) return } + for i := range items { + if items[i].Key == conf.DefaultRole { + items[i].Options = getRoleOptions() + break + } + } common.SuccessResp(c, items) } } @@ -88,6 +112,12 @@ func ListSettings(c *gin.Context) { common.ErrorResp(c, err, 400) return } + for i := range settings { + if settings[i].Key == conf.DefaultRole { + settings[i].Options = getRoleOptions() + break + } + } common.SuccessResp(c, settings) } diff --git a/server/handles/user.go b/server/handles/user.go index b4c152c5a86..01368beec6e 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -36,6 +36,9 @@ func CreateUser(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if len(req.Role) == 0 { + req.Role = model.Roles{op.GetDefaultRoleID()} + } if req.IsAdmin() || req.IsGuest() { common.ErrorStrResp(c, "admin or guest user can not be created", 400, true) return diff --git a/server/router.go b/server/router.go index 72546f4eb6b..e8902f7a648 100644 --- a/server/router.go +++ b/server/router.go @@ -61,6 +61,7 @@ func Init(e *gin.Engine) { api.POST("/auth/login", handles.Login) api.POST("/auth/login/hash", handles.LoginHash) api.POST("/auth/login/ldap", handles.LoginLdap) + api.POST("/auth/register", handles.Register) auth.GET("/me", handles.CurrentUser) auth.POST("/me/update", handles.UpdateCurrent) auth.GET("/me/sshkey/list", handles.ListMyPublicKey) From 74e384175b25a0a1968c1aa2e75b720b23104727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 19 Aug 2025 00:53:52 +0800 Subject: [PATCH 546/659] fix(lanzou): correct comment parsing logic in lanzou driver (#9278) - Adjusted logic to skip incrementing index when exiting comments. - Added checks to continue loop if inside a single-line or block comment. - Prevents erroneous parsing and retains intended comment exclusion. --- drivers/lanzou/help.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index c3f5c6bb5bc..b3d69006791 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -94,6 +94,7 @@ func RemoveJSComment(data string) string { } if inComment && v == '*' && i+1 < len(data) && data[i+1] == '/' { inComment = false + i++ continue } if v == '/' && i+1 < len(data) { @@ -108,6 +109,9 @@ func RemoveJSComment(data string) string { continue } } + if inComment || inSingleLineComment { + continue + } result.WriteByte(v) } From a9fcd51bc4f0e95127bf486690a10023da05241d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 19 Aug 2025 15:01:32 +0800 Subject: [PATCH 547/659] fix: ensure DefaultRole stores role ID while exposing role name in APIs (#9279) * fix(setting): ensure DefaultRole stores role ID while exposing role name in APIs - Simplified initial settings to use `model.GUEST` as the default role ID instead of querying roles at startup. - Updated `GetSetting`, `ListSettings` handlers to: - Convert stored role ID into the corresponding role name when returning data. - Preserve dynamic role options for selection. - Removed unused `strings` import and role preloading logic from `InitialSettings`. - This change avoids DB dependency during initialization while keeping consistent role display for frontend clients. * fix(setting): ensure DefaultRole stores role ID while exposing role name in APIs (fix/settings-get-role) - Simplify initial settings to use `model.GUEST` as the default role ID instead of querying roles at startup. - Update `GetSetting`, `ListSettings` handlers to: - Convert stored role ID into the corresponding role name when returning data. - Preserve dynamic role options for selection. - Remove unused `strings` import and role preloading logic from `InitialSettings`. - Avoid DB dependency during initialization while keeping consistent role display for frontend clients. --- internal/bootstrap/data/setting.go | 19 ++------------ internal/op/hook.go | 8 +++--- server/handles/setting.go | 40 +++++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 39c3b1be571..17a63af21c2 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -2,7 +2,6 @@ package data import ( "strconv" - "strings" "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" @@ -92,21 +91,7 @@ func InitialSettings() []model.SettingItem { } else { token = random.Token() } - roles, _, err := op.GetRoles(1, model.MaxInt) - if err != nil { - utils.Log.Fatalf("failed get roles: %+v", err) - } - roleNames := make([]string, len(roles)) - defaultRoleID := "" - for i, role := range roles { - roleNames[i] = role.Name - if role.Name == "guest" { - defaultRoleID = strconv.Itoa(int(role.ID)) - } - } - if defaultRoleID == "" && len(roles) > 0 { - defaultRoleID = strconv.Itoa(int(roles[0].ID)) - } + defaultRoleID := strconv.Itoa(model.GUEST) initialSettingItems = []model.SettingItem{ // site settings {Key: conf.VERSION, Value: conf.Version, Type: conf.TypeString, Group: model.SITE, Flag: model.READONLY}, @@ -120,7 +105,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, {Key: conf.AllowRegister, Value: "false", Type: conf.TypeBool, Group: model.SITE}, - {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Options: strings.Join(roleNames, ","), Group: model.SITE}, + {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Group: model.SITE}, // newui settings {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, // style settings diff --git a/internal/op/hook.go b/internal/op/hook.go index 08ea4603e5c..f08966c4ade 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -88,12 +88,12 @@ var settingItemHooks = map[string]SettingItemHook{ if v == "" { return nil } - r, err := GetRoleByName(v) + id, err := strconv.Atoi(v) if err != nil { - return err + return errors.WithStack(err) } - item.Value = strconv.Itoa(int(r.ID)) - return nil + _, err = GetRole(uint(id)) + return err }, } diff --git a/server/handles/setting.go b/server/handles/setting.go index 3ce5fcbf94f..e0dbb490033 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -19,9 +19,12 @@ func getRoleOptions() string { if err != nil { return "" } - names := make([]string, len(roles)) - for i, r := range roles { - names[i] = r.Name + names := make([]string, 0, len(roles)) + for _, r := range roles { + if r.Name == "admin" || r.Name == "guest" { + continue + } + names = append(names, r.Name) } return strings.Join(names, ",") } @@ -49,6 +52,11 @@ func GetSetting(c *gin.Context) { if item.Key == conf.DefaultRole { copy := *item copy.Options = getRoleOptions() + if id, err := strconv.Atoi(copy.Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + copy.Value = r.Name + } + } common.SuccessResp(c, copy) return } @@ -61,6 +69,11 @@ func GetSetting(c *gin.Context) { } for i := range items { if items[i].Key == conf.DefaultRole { + if id, err := strconv.Atoi(items[i].Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + items[i].Value = r.Name + } + } items[i].Options = getRoleOptions() break } @@ -75,6 +88,22 @@ func SaveSettings(c *gin.Context) { common.ErrorResp(c, err, 400) return } + + for i := range req { + if req[i].Key == conf.DefaultRole { + role, err := op.GetRoleByName(req[i].Value) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorStrResp(c, "cannot set admin or guest as default role", 400) + return + } + req[i].Value = strconv.Itoa(int(role.ID)) + } + } + if err := op.SaveSettingItems(req); err != nil { common.ErrorResp(c, err, 500) } else { @@ -114,6 +143,11 @@ func ListSettings(c *gin.Context) { } for i := range settings { if settings[i].Key == conf.DefaultRole { + if id, err := strconv.Atoi(settings[i].Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + settings[i].Value = r.Name + } + } settings[i].Options = getRoleOptions() break } From d7723c378f67f13e38d9b5f5acd75ea07741c964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 25 Aug 2025 19:46:10 +0800 Subject: [PATCH 548/659] chore(deps): Upgrade 115driver to v1.1.1 (#9283) - Upgraded `github.com/SheltonZhu/115driver` from v1.0.34 to v1.1.1 - Updated the corresponding version verification information in `go.sum` --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5bc953fb4de..9923c489eef 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 - github.com/SheltonZhu/115driver v1.0.34 + github.com/SheltonZhu/115driver v1.1.1 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 diff --git a/go.sum b/go.sum index a9faa92fa69..e47d96234ae 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,8 @@ github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4 github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.0.34 h1:zhMLp4vgq7GksqvSxQQDOVfK6EOHldQl4b2n8tnZ+EE= -github.com/SheltonZhu/115driver v1.0.34/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= +github.com/SheltonZhu/115driver v1.1.1 h1:9EMhe2ZJflGiAaZbYInw2jqxTcqZNF+DtVDsEy70aFU= +github.com/SheltonZhu/115driver v1.1.1/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= From 3319f6ea6ad4e2a388e27f4d231a4aa5459aad4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 25 Aug 2025 19:46:24 +0800 Subject: [PATCH 549/659] feat(search): Optimized search result filtering and paging logic (#9287) - Introduced the `filteredNodes` list to optimize the node filtering process - Filtered results based on the page limit during paging - Modified search logic to ensure nodes are within the user's base path - Added access permission checks for node metadata - Adjusted paging logic to avoid redundant node retrieval --- server/handles/search.go | 43 +++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/server/handles/search.go b/server/handles/search.go index 7d421a21e59..832fc94fb07 100644 --- a/server/handles/search.go +++ b/server/handles/search.go @@ -43,28 +43,39 @@ func Search(c *gin.Context) { common.ErrorResp(c, err, 400) return } - nodes, total, err := search.Search(c, req.SearchReq) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - var filteredNodes []model.SearchNode - for _, node := range nodes { - if !strings.HasPrefix(node.Parent, user.BasePath) { - continue + var ( + filteredNodes []model.SearchNode + ) + for len(filteredNodes) < req.PerPage { + nodes, _, err := search.Search(c, req.SearchReq) + if err != nil { + common.ErrorResp(c, err, 500) + return } - meta, err := op.GetNearestMeta(node.Parent) - if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { - continue + if len(nodes) == 0 { + break } - if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), req.Password) { - continue + for _, node := range nodes { + if !strings.HasPrefix(node.Parent, user.BasePath) { + continue + } + meta, err := op.GetNearestMeta(node.Parent) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + continue + } + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), req.Password) { + continue + } + filteredNodes = append(filteredNodes, node) + if len(filteredNodes) >= req.PerPage { + break + } } - filteredNodes = append(filteredNodes, node) + req.Page++ } common.SuccessResp(c, common.PageResp{ Content: utils.MustSliceConvert(filteredNodes, nodeToSearchResp), - Total: total, + Total: int64(len(filteredNodes)), }) } From c64f899a636b66d12056c6ae3fcab6c21b0cb146 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 25 Aug 2025 19:46:38 +0800 Subject: [PATCH 550/659] feat: implement session management (#9286) * feat(auth): Added device session management - Added the `handleSession` function to manage user device sessions and verify client identity - Updated `auth.go` to call `handleSession` for device handling when a user logs in - Added the `Session` model to database migrations - Added `device.go` and `session.go` files to handle device session logic - Updated `settings.go` to add device-related configuration items, such as the maximum number of devices, device eviction policy, and session TTL * feat(session): Adds session management features - Added `SessionInactive` error type in `device.go` - Added session-related APIs in `router.go` to support listing and evicting sessions - Added `ListSessionsByUser`, `ListSessions`, and `MarkInactive` methods in `session.go` - Returns an appropriate error when the session state is `SessionInactive` * feat(auth): Marks the device session as invalid. - Import the `session` package into the `auth` module to handle device session status. - Add a check in the login logic. If `device_key` is obtained, call `session.MarkInactive` to mark the device session as invalid. - Store the invalid status in the context variable `session_inactive` for subsequent middleware checks. - Add a check in the session refresh logic to abort the process if the current session has been marked invalid. * feat(auth, session): Added device information processing and session management changes - Updated device handling logic in `auth.go` to pass user agent and IP information - Adjusted database queries in `session.go` to optimize session query fields and add `user_agent` and `ip` fields - Modified the `Handle` method to add `ua` and `ip` parameters to store the user agent and IP address - Added the `SessionResp` structure to return a session response containing `user_agent` and `ip` - Updated the `/admin/user/create` and `/webdav` endpoints to pass the user agent and IP address to the device handler --- internal/bootstrap/data/setting.go | 3 + internal/conf/const.go | 3 + internal/db/db.go | 2 +- internal/db/session.go | 65 +++++++++++++++++++ internal/device/session.go | 67 +++++++++++++++++++ internal/errs/device.go | 8 +++ internal/model/session.go | 16 +++++ internal/session/session.go | 8 +++ pkg/utils/mask.go | 30 +++++++++ server/handles/auth.go | 8 +++ server/handles/session.go | 92 +++++++++++++++++++++++++++ server/middlewares/auth.go | 30 ++++++++- server/middlewares/session_refresh.go | 26 ++++++++ server/router.go | 7 ++ server/webdav.go | 17 +++++ 15 files changed, 378 insertions(+), 4 deletions(-) create mode 100644 internal/db/session.go create mode 100644 internal/device/session.go create mode 100644 internal/errs/device.go create mode 100644 internal/model/session.go create mode 100644 internal/session/session.go create mode 100644 pkg/utils/mask.go create mode 100644 server/handles/session.go create mode 100644 server/middlewares/session_refresh.go diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 17a63af21c2..bbb633e33a6 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -165,6 +165,9 @@ func InitialSettings() []model.SettingItem { {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, + {Key: conf.MaxDevices, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL}, + {Key: conf.DeviceEvictPolicy, Value: "deny", Type: conf.TypeSelect, Options: "deny,evict_oldest", Group: model.GLOBAL}, + {Key: conf.DeviceSessionTTL, Value: "86400", Type: conf.TypeNumber, Group: model.GLOBAL}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 0bf0cd67f7e..1a5581633a0 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -48,6 +48,9 @@ const ( ForwardDirectLinkParams = "forward_direct_link_params" IgnoreDirectLinkParams = "ignore_direct_link_params" WebauthnLoginEnabled = "webauthn_login_enabled" + MaxDevices = "max_devices" + DeviceEvictPolicy = "device_evict_policy" + DeviceSessionTTL = "device_session_ttl" // index SearchIndex = "search_index" diff --git a/internal/db/db.go b/internal/db/db.go index 0d8ab42130a..4577059d4f8 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile), new(model.Session)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/session.go b/internal/db/session.go new file mode 100644 index 00000000000..8db9fa69f8b --- /dev/null +++ b/internal/db/session.go @@ -0,0 +1,65 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm/clause" +) + +func GetSession(userID uint, deviceKey string) (*model.Session, error) { + s := model.Session{UserID: userID, DeviceKey: deviceKey} + if err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where(&s).First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed find session") + } + return &s, nil +} + +func CreateSession(s *model.Session) error { + return errors.WithStack(db.Create(s).Error) +} + +func UpsertSession(s *model.Session) error { + return errors.WithStack(db.Clauses(clause.OnConflict{UpdateAll: true}).Create(s).Error) +} + +func DeleteSession(userID uint, deviceKey string) error { + return errors.WithStack(db.Where("user_id = ? AND device_key = ?", userID, deviceKey).Delete(&model.Session{}).Error) +} + +func CountSessionsByUser(userID uint) (int64, error) { + var count int64 + err := db.Model(&model.Session{}).Where("user_id = ?", userID).Count(&count).Error + return count, errors.WithStack(err) +} + +func DeleteSessionsBefore(ts int64) error { + return errors.WithStack(db.Where("last_active < ?", ts).Delete(&model.Session{}).Error) +} + +func GetOldestSession(userID uint) (*model.Session, error) { + var s model.Session + if err := db.Where("user_id = ?", userID).Order("last_active ASC").First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed get oldest session") + } + return &s, nil +} + +func UpdateSessionLastActive(userID uint, deviceKey string, lastActive int64) error { + return errors.WithStack(db.Model(&model.Session{}).Where("user_id = ? AND device_key = ?", userID, deviceKey).Update("last_active", lastActive).Error) +} + +func ListSessionsByUser(userID uint) ([]model.Session, error) { + var sessions []model.Session + err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where("user_id = ? AND status = ?", userID, model.SessionActive).Find(&sessions).Error + return sessions, errors.WithStack(err) +} + +func ListSessions() ([]model.Session, error) { + var sessions []model.Session + err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where("status = ?", model.SessionActive).Find(&sessions).Error + return sessions, errors.WithStack(err) +} + +func MarkInactive(sessionID string) error { + return errors.WithStack(db.Model(&model.Session{}).Where("device_key = ?", sessionID).Update("status", model.SessionInactive).Error) +} diff --git a/internal/device/session.go b/internal/device/session.go new file mode 100644 index 00000000000..d407c858dfa --- /dev/null +++ b/internal/device/session.go @@ -0,0 +1,67 @@ +package device + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// Handle verifies device sessions for a user and upserts current session. +func Handle(userID uint, deviceKey, ua, ip string) error { + ttl := setting.GetInt(conf.DeviceSessionTTL, 86400) + if ttl > 0 { + _ = db.DeleteSessionsBefore(time.Now().Unix() - int64(ttl)) + } + + ip = utils.MaskIP(ip) + + now := time.Now().Unix() + sess, err := db.GetSession(userID, deviceKey) + if err == nil { + if sess.Status == model.SessionInactive { + return errors.WithStack(errs.SessionInactive) + } + sess.LastActive = now + sess.Status = model.SessionActive + sess.UserAgent = ua + sess.IP = ip + return db.UpsertSession(sess) + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + oldest, err := db.GetOldestSession(userID) + if err == nil { + _ = db.DeleteSession(userID, oldest.DeviceKey) + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + + s := &model.Session{UserID: userID, DeviceKey: deviceKey, UserAgent: ua, IP: ip, LastActive: now, Status: model.SessionActive} + return db.CreateSession(s) +} + +// Refresh updates last_active for the session. +func Refresh(userID uint, deviceKey string) { + _ = db.UpdateSessionLastActive(userID, deviceKey, time.Now().Unix()) +} diff --git a/internal/errs/device.go b/internal/errs/device.go new file mode 100644 index 00000000000..3b79298a672 --- /dev/null +++ b/internal/errs/device.go @@ -0,0 +1,8 @@ +package errs + +import "errors" + +var ( + TooManyDevices = errors.New("too many active devices") + SessionInactive = errors.New("session inactive") +) diff --git a/internal/model/session.go b/internal/model/session.go new file mode 100644 index 00000000000..3cb6d0dab90 --- /dev/null +++ b/internal/model/session.go @@ -0,0 +1,16 @@ +package model + +// Session represents a device session of a user. +type Session struct { + UserID uint `json:"user_id" gorm:"index"` + DeviceKey string `json:"device_key" gorm:"primaryKey;size:64"` + UserAgent string `json:"user_agent" gorm:"size:255"` + IP string `json:"ip" gorm:"size:64"` + LastActive int64 `json:"last_active"` + Status int `json:"status"` +} + +const ( + SessionActive = iota + SessionInactive +) diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 00000000000..47d1b70125c --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,8 @@ +package session + +import "github.com/alist-org/alist/v3/internal/db" + +// MarkInactive marks the session with the given ID as inactive. +func MarkInactive(sessionID string) error { + return db.MarkInactive(sessionID) +} diff --git a/pkg/utils/mask.go b/pkg/utils/mask.go new file mode 100644 index 00000000000..1513ad40368 --- /dev/null +++ b/pkg/utils/mask.go @@ -0,0 +1,30 @@ +package utils + +import "strings" + +// MaskIP anonymizes middle segments of an IP address. +func MaskIP(ip string) string { + if ip == "" { + return "" + } + if strings.Contains(ip, ":") { + parts := strings.Split(ip, ":") + if len(parts) > 2 { + for i := 1; i < len(parts)-1; i++ { + if parts[i] != "" { + parts[i] = "*" + } + } + return strings.Join(parts, ":") + } + return ip + } + parts := strings.Split(ip, ".") + if len(parts) == 4 { + for i := 1; i < len(parts)-1; i++ { + parts[i] = "*" + } + return strings.Join(parts, ".") + } + return ip +} diff --git a/server/handles/auth.go b/server/handles/auth.go index 26447ddd594..30714f6544c 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -12,6 +12,7 @@ import ( "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/session" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" @@ -247,6 +248,13 @@ func Verify2FA(c *gin.Context) { } func LogOut(c *gin.Context) { + if keyVal, ok := c.Get("device_key"); ok { + if err := session.MarkInactive(keyVal.(string)); err != nil { + common.ErrorResp(c, err, 500) + return + } + c.Set("session_inactive", true) + } err := common.InvalidateToken(c.GetHeader("Authorization")) if err != nil { common.ErrorResp(c, err, 500) diff --git a/server/handles/session.go b/server/handles/session.go new file mode 100644 index 00000000000..886be66ab01 --- /dev/null +++ b/server/handles/session.go @@ -0,0 +1,92 @@ +package handles + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +type SessionResp struct { + SessionID string `json:"session_id"` + UserID uint `json:"user_id,omitempty"` + LastActive int64 `json:"last_active"` + Status int `json:"status"` + UA string `json:"ua"` + IP string `json:"ip"` +} + +func ListMySessions(c *gin.Context) { + user := c.MustGet("user").(*model.User) + sessions, err := db.ListSessionsByUser(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + resp := make([]SessionResp, len(sessions)) + for i, s := range sessions { + resp[i] = SessionResp{ + SessionID: s.DeviceKey, + LastActive: s.LastActive, + Status: s.Status, + UA: s.UserAgent, + IP: s.IP, + } + } + common.SuccessResp(c, resp) +} + +type EvictSessionReq struct { + SessionID string `json:"session_id"` +} + +func EvictMySession(c *gin.Context) { + var req EvictSessionReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if _, err := db.GetSession(user.ID, req.SessionID); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := db.MarkInactive(req.SessionID); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +func ListSessions(c *gin.Context) { + sessions, err := db.ListSessions() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + resp := make([]SessionResp, len(sessions)) + for i, s := range sessions { + resp[i] = SessionResp{ + SessionID: s.DeviceKey, + UserID: s.UserID, + LastActive: s.LastActive, + Status: s.Status, + UA: s.UserAgent, + IP: s.IP, + } + } + common.SuccessResp(c, resp) +} + +func EvictSession(c *gin.Context) { + var req EvictSessionReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := db.MarkInactive(req.SessionID); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index c0743c9ce9d..72eaefe6096 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -5,9 +5,11 @@ import ( "fmt" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -24,7 +26,9 @@ func Auth(c *gin.Context) { c.Abort() return } - c.Set("user", admin) + if !handleSession(c, admin) { + return + } log.Debugf("use admin token: %+v", admin) c.Next() return @@ -50,7 +54,9 @@ func Auth(c *gin.Context) { } guest.RolesDetail = roles } - c.Set("user", guest) + if !handleSession(c, guest) { + return + } log.Debugf("use empty token: %+v", guest) c.Next() return @@ -87,11 +93,29 @@ func Auth(c *gin.Context) { } user.RolesDetail = roles } - c.Set("user", user) + if !handleSession(c, user) { + return + } log.Debugf("use login token: %+v", user) c.Next() } +func handleSession(c *gin.Context, user *model.User) bool { + clientID := c.GetHeader("Client-Id") + if clientID == "" { + clientID = c.Query("client_id") + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s-%s-%s", user.ID, c.Request.UserAgent(), c.ClientIP(), clientID)) + if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + common.ErrorResp(c, err, 403) + c.Abort() + return false + } + c.Set("device_key", key) + c.Set("user", user) + return true +} + func Authn(c *gin.Context) { token := c.GetHeader("Authorization") if subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { diff --git a/server/middlewares/session_refresh.go b/server/middlewares/session_refresh.go new file mode 100644 index 00000000000..2073020ce79 --- /dev/null +++ b/server/middlewares/session_refresh.go @@ -0,0 +1,26 @@ +package middlewares + +import ( + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/model" + "github.com/gin-gonic/gin" +) + +// SessionRefresh updates session's last_active after successful requests. +func SessionRefresh(c *gin.Context) { + c.Next() + if c.Writer.Status() >= 400 { + return + } + if inactive, ok := c.Get("session_inactive"); ok { + if b, ok := inactive.(bool); ok && b { + return + } + } + userVal, uok := c.Get("user") + keyVal, kok := c.Get("device_key") + if uok && kok { + user := userVal.(*model.User) + device.Refresh(user.ID, keyVal.(string)) + } +} diff --git a/server/router.go b/server/router.go index e8902f7a648..4d79c1fde52 100644 --- a/server/router.go +++ b/server/router.go @@ -22,6 +22,7 @@ func Init(e *gin.Engine) { }) } Cors(e) + e.Use(middlewares.SessionRefresh) g := e.Group(conf.URL.Path) if conf.Conf.Scheme.HttpPort != -1 && conf.Conf.Scheme.HttpsPort != -1 && conf.Conf.Scheme.ForceHttps { e.Use(middlewares.ForceHttps) @@ -70,6 +71,8 @@ func Init(e *gin.Engine) { auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) auth.GET("/auth/logout", handles.LogOut) + auth.GET("/me/sessions", handles.ListMySessions) + auth.POST("/me/sessions/evict", handles.EvictMySession) // auth api.GET("/auth/sso", handles.SSOLoginRedirect) @@ -184,6 +187,10 @@ func admin(g *gin.RouterGroup) { labelFileBinding.POST("/delete", handles.DelLabelByFileName) labelFileBinding.POST("/restore", handles.RestoreLabelFileBinding) + session := g.Group("/session") + session.GET("/list", handles.ListSessions) + session.POST("/evict", handles.EvictSession) + } func _fs(g *gin.RouterGroup) { diff --git a/server/webdav.go b/server/webdav.go index 582c469d73a..e0980139e4f 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -3,6 +3,7 @@ package server import ( "context" "crypto/subtle" + "fmt" "net/http" "net/url" "path" @@ -12,9 +13,11 @@ import ( "github.com/alist-org/alist/v3/server/middlewares" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/webdav" "github.com/gin-gonic/gin" @@ -69,6 +72,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", admin.ID, c.ClientIP())) + if err := device.Handle(admin.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + c.Set("device_key", key) c.Set("user", admin) c.Next() return @@ -146,6 +156,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", user.ID, c.ClientIP())) + if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + c.Set("device_key", key) c.Set("user", user) c.Next() } From de09ba08b6429edd58f3ed0f602e644966fc6ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Wed, 27 Aug 2025 17:46:34 +0800 Subject: [PATCH 551/659] chore(deps): Update 115driver dependency to v1.1.2 (#9294) - Upgrade `github.com/SheltonZhu/115driver` to v1.1.2 in `go.mod` - Modify `replace` to point to `github.com/okatu-loli/115driver v1.1.2` - Remove old version checksum from `go.sum` and add new version checksum --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 9923c489eef..51f9beec443 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 - github.com/SheltonZhu/115driver v1.1.1 + github.com/SheltonZhu/115driver v1.1.2 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 @@ -265,4 +265,4 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect ) -// replace github.com/xhofe/115-sdk-go => ../../xhofe/115-sdk-go +replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.1.2 diff --git a/go.sum b/go.sum index e47d96234ae..1b088a7186e 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,6 @@ github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4 github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= -github.com/SheltonZhu/115driver v1.1.1 h1:9EMhe2ZJflGiAaZbYInw2jqxTcqZNF+DtVDsEy70aFU= -github.com/SheltonZhu/115driver v1.1.1/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= @@ -490,6 +488,8 @@ github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/okatu-loli/115driver v1.1.2 h1:XZT3r/51SZRQGzre2IeA+0/k4T1FneqArdhE4Wd600Q= +github.com/okatu-loli/115driver v1.1.2/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= From 3bf0af1e6826ba810df24f05db73cede74f121d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 28 Aug 2025 09:57:13 +0800 Subject: [PATCH 552/659] fix(session): Fixed the session status update logic. (#9296) - Removed the error returned when the session status is `SessionInactive`. - Updated the `LastActive` field of the session to always record the current time. --- internal/device/session.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/device/session.go b/internal/device/session.go index d407c858dfa..49bf74b6e9a 100644 --- a/internal/device/session.go +++ b/internal/device/session.go @@ -25,11 +25,9 @@ func Handle(userID uint, deviceKey, ua, ip string) error { now := time.Now().Unix() sess, err := db.GetSession(userID, deviceKey) if err == nil { - if sess.Status == model.SessionInactive { - return errors.WithStack(errs.SessionInactive) - } - sess.LastActive = now + // reactivate existing session if it was inactive sess.Status = model.SessionActive + sess.LastActive = now sess.UserAgent = ua sess.IP = ip return db.UpsertSession(sess) From 84adba3acc2141dbe61663e4ec199ffc4333f76b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 28 Aug 2025 09:57:34 +0800 Subject: [PATCH 553/659] feat(user): Enhanced role assignment logic (#9297) - Imported the `utils` package - Modified the role assignment logic to prevent assigning administrator or guest roles to users --- server/handles/user.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/handles/user.go b/server/handles/user.go index 01368beec6e..ac3a06e8180 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -1,9 +1,10 @@ package handles import ( - "github.com/alist-org/alist/v3/pkg/utils" "strconv" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" @@ -97,6 +98,14 @@ func UpdateUser(c *gin.Context) { return } } + + if !utils.SliceEqual(user.Role, req.Role) { + if req.IsAdmin() || req.IsGuest() { + common.ErrorStrResp(c, "cannot assign admin or guest role to user", 400, true) + return + } + } + if err := op.UpdateUser(&req); err != nil { common.ErrorResp(c, err, 500) } else { From 8623da5361e487e3310374a88339d5d98ff8ac77 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 29 Aug 2025 11:53:55 +0800 Subject: [PATCH 554/659] feat(session): Added user session limit and device eviction logic - Renamed `CountSessionsByUser` to `CountActiveSessionsByUser` and added session status filtering - Added user and device session limit, with policy handling when exceeding the limit - Introduced device eviction policy: If the maximum number of devices is exceeded, the oldest session will be evicted using the "evict_oldest" policy - Modified `LastActive` update logic to ensure accurate session activity time --- internal/db/session.go | 6 ++++-- internal/device/session.go | 22 ++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/internal/db/session.go b/internal/db/session.go index 8db9fa69f8b..e8dce441175 100644 --- a/internal/db/session.go +++ b/internal/db/session.go @@ -26,9 +26,11 @@ func DeleteSession(userID uint, deviceKey string) error { return errors.WithStack(db.Where("user_id = ? AND device_key = ?", userID, deviceKey).Delete(&model.Session{}).Error) } -func CountSessionsByUser(userID uint) (int64, error) { +func CountActiveSessionsByUser(userID uint) (int64, error) { var count int64 - err := db.Model(&model.Session{}).Where("user_id = ?", userID).Count(&count).Error + err := db.Model(&model.Session{}). + Where("user_id = ? AND status = ?", userID, model.SessionActive). + Count(&count).Error return count, errors.WithStack(err) } diff --git a/internal/device/session.go b/internal/device/session.go index 49bf74b6e9a..5d5a39969c1 100644 --- a/internal/device/session.go +++ b/internal/device/session.go @@ -25,7 +25,25 @@ func Handle(userID uint, deviceKey, ua, ip string) error { now := time.Now().Unix() sess, err := db.GetSession(userID, deviceKey) if err == nil { - // reactivate existing session if it was inactive + if sess.Status == model.SessionInactive { + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, cerr := db.CountActiveSessionsByUser(userID) + if cerr != nil { + return cerr + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, gerr := db.GetOldestSession(userID); gerr == nil { + _ = db.DeleteSession(userID, oldest.DeviceKey) + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + } sess.Status = model.SessionActive sess.LastActive = now sess.UserAgent = ua @@ -38,7 +56,7 @@ func Handle(userID uint, deviceKey, ua, ip string) error { max := setting.GetInt(conf.MaxDevices, 0) if max > 0 { - count, err := db.CountSessionsByUser(userID) + count, err := db.CountActiveSessionsByUser(userID) if err != nil { return err } From 9a7c82a71e3ca00f8cdd61f9b854e87464982ba1 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 29 Aug 2025 13:31:44 +0800 Subject: [PATCH 555/659] feat(auth): Optimized device session handling logic - Introduced middleware to handle device sessions - Changed `handleSession` to `HandleSession` in multiple places in `auth.go` to maintain consistent naming - Updated response structure to return `device_key` and `token` --- server/handles/auth.go | 8 +++++++- server/middlewares/auth.go | 9 +++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/server/handles/auth.go b/server/handles/auth.go index 30714f6544c..8c7d7d9f56f 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -15,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/internal/session" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" + "github.com/alist-org/alist/v3/server/middlewares" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" ) @@ -82,13 +83,18 @@ func loginHash(c *gin.Context, req *LoginReq) { return } } + // generate device session + if !middlewares.HandleSession(c, user) { + return + } // generate token token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return } - common.SuccessResp(c, gin.H{"token": token}) + key := c.GetString("device_key") + common.SuccessResp(c, gin.H{"token": token, "device_key": key}) loginCache.Del(ip) } diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 72eaefe6096..714c11548c5 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -26,7 +26,7 @@ func Auth(c *gin.Context) { c.Abort() return } - if !handleSession(c, admin) { + if !HandleSession(c, admin) { return } log.Debugf("use admin token: %+v", admin) @@ -54,7 +54,7 @@ func Auth(c *gin.Context) { } guest.RolesDetail = roles } - if !handleSession(c, guest) { + if !HandleSession(c, guest) { return } log.Debugf("use empty token: %+v", guest) @@ -93,14 +93,15 @@ func Auth(c *gin.Context) { } user.RolesDetail = roles } - if !handleSession(c, user) { + if !HandleSession(c, user) { return } log.Debugf("use login token: %+v", user) c.Next() } -func handleSession(c *gin.Context, user *model.User) bool { +// HandleSession verifies device sessions and stores context values. +func HandleSession(c *gin.Context, user *model.User) bool { clientID := c.GetHeader("Client-Id") if clientID == "" { clientID = c.Query("client_id") From 63391a20914b33604ced8f23412feb50e6a51796 Mon Sep 17 00:00:00 2001 From: Sky_slience <95515853+skysliences@users.noreply.github.com> Date: Fri, 29 Aug 2025 14:56:54 +0800 Subject: [PATCH 556/659] fix(readme): remove outdated sponsor links from README files (#9300) Co-authored-by: Sky_slience --- README.md | 2 -- README_cn.md | 2 -- README_ja.md | 2 -- 3 files changed, 6 deletions(-) diff --git a/README.md b/README.md index 5a93997fe40..d352b5b3f50 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,6 @@ https://alistgo.com/guide/sponsor.html ### Special sponsors - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## Contributors diff --git a/README_cn.md b/README_cn.md index 79ed864bc84..27f417d1698 100644 --- a/README_cn.md +++ b/README_cn.md @@ -118,8 +118,6 @@ AList 是一个开源软件，如果你碰巧喜欢这个项目，并希望我 ### 特别赞助 - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - 苹果生态下优雅的网盘视频播放器，iPhone，iPad，Mac，Apple TV全平台支持。 -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (国内API服务器赞助) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## 贡献者 diff --git a/README_ja.md b/README_ja.md index 9291b2acdc2..cd59086078c 100644 --- a/README_ja.md +++ b/README_ja.md @@ -120,8 +120,6 @@ https://alistgo.com/guide/sponsor.html ### スペシャルスポンサー - [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. -- [亚洲云](https://www.asiayun.com/aff/QQCOOQKZ) - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商 (sponsored Chinese API server) -- [找资源](http://zhaoziyuan2.cc/) - 阿里云盘资源搜索引擎 ## コントリビューター From 4b288a08ef264d4be4b36c638992b1d9f5484526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 29 Aug 2025 21:20:29 +0800 Subject: [PATCH 557/659] fix: session invalid issue (#9301) * feat(auth): Enhanced device login session management - Upon login, obtain and verify `Client-Id` to ensure unique device sessions. - If there are too many device sessions, clean up old ones according to the configured policy or return an error. - If a device session is invalid, deregister the old token and return a 401 error. - Added `EnsureActiveOnLogin` function to handle the creation and refresh of device sessions during login. * feat(session): Modified session deletion logic to mark sessions as inactive. - Changed session deletion logic to mark sessions as inactive using the `MarkInactive` method. - Adjusted error handling to ensure an error is returned if marking fails. * feat(session): Added device limits and eviction policies - Added a device limit, controlling the maximum number of devices using the `MaxDevices` configuration option. - If the number of devices exceeds the limit, the configured eviction policy is used. - If the policy is `evict_oldest`, the oldest device is evicted. - Otherwise, an error message indicating too many devices is returned. * refactor(session): Filter for the user's oldest active session - Renamed `GetOldestSession` to `GetOldestActiveSession` to more accurately reflect its functionality - Updated the SQL query to add the `status = SessionActive` condition to retrieve only active sessions - Replaced all callpoints and unified the new function name to ensure logical consistency --- internal/db/session.go | 8 ++-- internal/device/session.go | 75 +++++++++++++++++++++++++++++++++----- server/handles/auth.go | 24 ++++++++++-- server/middlewares/auth.go | 12 +++++- 4 files changed, 100 insertions(+), 19 deletions(-) diff --git a/internal/db/session.go b/internal/db/session.go index e8dce441175..35c778c3ac8 100644 --- a/internal/db/session.go +++ b/internal/db/session.go @@ -38,10 +38,12 @@ func DeleteSessionsBefore(ts int64) error { return errors.WithStack(db.Where("last_active < ?", ts).Delete(&model.Session{}).Error) } -func GetOldestSession(userID uint) (*model.Session, error) { +// GetOldestActiveSession returns the oldest active session for the specified user. +func GetOldestActiveSession(userID uint) (*model.Session, error) { var s model.Session - if err := db.Where("user_id = ?", userID).Order("last_active ASC").First(&s).Error; err != nil { - return nil, errors.Wrap(err, "failed get oldest session") + if err := db.Where("user_id = ? AND status = ?", userID, model.SessionActive). + Order("last_active ASC").First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed get oldest active session") } return &s, nil } diff --git a/internal/device/session.go b/internal/device/session.go index 5d5a39969c1..1d9e7ea53cd 100644 --- a/internal/device/session.go +++ b/internal/device/session.go @@ -23,20 +23,68 @@ func Handle(userID uint, deviceKey, ua, ip string) error { ip = utils.MaskIP(ip) now := time.Now().Unix() + sess, err := db.GetSession(userID, deviceKey) + if err == nil { + if sess.Status == model.SessionInactive { + return errors.WithStack(errs.SessionInactive) + } + sess.Status = model.SessionActive + sess.LastActive = now + sess.UserAgent = ua + sess.IP = ip + return db.UpsertSession(sess) + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, err := db.GetOldestActiveSession(userID); err == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + + s := &model.Session{UserID: userID, DeviceKey: deviceKey, UserAgent: ua, IP: ip, LastActive: now, Status: model.SessionActive} + return db.CreateSession(s) +} + +// EnsureActiveOnLogin is used only in login flow: +// - If session exists (even Inactive): reactivate and refresh fields. +// - If not exists: apply max-devices policy, then create Active session. +func EnsureActiveOnLogin(userID uint, deviceKey, ua, ip string) error { + ip = utils.MaskIP(ip) + now := time.Now().Unix() + sess, err := db.GetSession(userID, deviceKey) if err == nil { if sess.Status == model.SessionInactive { max := setting.GetInt(conf.MaxDevices, 0) if max > 0 { - count, cerr := db.CountActiveSessionsByUser(userID) - if cerr != nil { - return cerr + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err } if count >= int64(max) { policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") if policy == "evict_oldest" { - if oldest, gerr := db.GetOldestSession(userID); gerr == nil { - _ = db.DeleteSession(userID, oldest.DeviceKey) + if oldest, gerr := db.GetOldestActiveSession(userID); gerr == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } } } else { return errors.WithStack(errs.TooManyDevices) @@ -63,9 +111,10 @@ func Handle(userID uint, deviceKey, ua, ip string) error { if count >= int64(max) { policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") if policy == "evict_oldest" { - oldest, err := db.GetOldestSession(userID) - if err == nil { - _ = db.DeleteSession(userID, oldest.DeviceKey) + if oldest, gerr := db.GetOldestActiveSession(userID); gerr == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } } } else { return errors.WithStack(errs.TooManyDevices) @@ -73,8 +122,14 @@ func Handle(userID uint, deviceKey, ua, ip string) error { } } - s := &model.Session{UserID: userID, DeviceKey: deviceKey, UserAgent: ua, IP: ip, LastActive: now, Status: model.SessionActive} - return db.CreateSession(s) + return db.CreateSession(&model.Session{ + UserID: userID, + DeviceKey: deviceKey, + UserAgent: ua, + IP: ip, + LastActive: now, + Status: model.SessionActive, + }) } // Refresh updates last_active for the session. diff --git a/server/handles/auth.go b/server/handles/auth.go index 8c7d7d9f56f..dd7d202b907 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -3,6 +3,8 @@ package handles import ( "bytes" "encoding/base64" + "errors" + "fmt" "image/png" "path" "strings" @@ -10,12 +12,14 @@ import ( "github.com/Xhofe/go-cache" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/session" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" - "github.com/alist-org/alist/v3/server/middlewares" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" ) @@ -83,17 +87,29 @@ func loginHash(c *gin.Context, req *LoginReq) { return } } - // generate device session - if !middlewares.HandleSession(c, user) { + + clientID := c.GetHeader("Client-Id") + if clientID == "" { + clientID = c.Query("client_id") + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", + user.ID, clientID)) + + if err := device.EnsureActiveOnLogin(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + if errors.Is(err, errs.TooManyDevices) { + common.ErrorResp(c, err, 403) + } else { + common.ErrorResp(c, err, 400, true) + } return } + // generate token token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return } - key := c.GetString("device_key") common.SuccessResp(c, gin.H{"token": token, "device_key": key}) loginCache.Del(ip) } diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 714c11548c5..204b4b7205e 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -2,10 +2,12 @@ package middlewares import ( "crypto/subtle" + "errors" "fmt" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" @@ -106,9 +108,15 @@ func HandleSession(c *gin.Context, user *model.User) bool { if clientID == "" { clientID = c.Query("client_id") } - key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s-%s-%s", user.ID, c.Request.UserAgent(), c.ClientIP(), clientID)) + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", user.ID, clientID)) if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { - common.ErrorResp(c, err, 403) + token := c.GetHeader("Authorization") + if errors.Is(err, errs.SessionInactive) { + _ = common.InvalidateToken(token) + common.ErrorResp(c, err, 401) + } else { + common.ErrorResp(c, err, 403) + } c.Abort() return false } From 23107483a126a53419cbf18f8fbdecf937f0d0a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 4 Sep 2025 22:14:33 +0800 Subject: [PATCH 558/659] Refactor (storage): Comment out the path validation logic (#9308) - Comment out the error return logic for paths with "/" - Remove storage path restrictions to allow for flexible handling of root paths --- internal/op/storage.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/op/storage.go b/internal/op/storage.go index 5ab1da1840a..27221e70e8c 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -47,9 +47,9 @@ func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) - if storage.MountPath == "/" { - return 0, errors.New("Mount path cannot be '/'") - } + //if storage.MountPath == "/" { + // return 0, errors.New("Mount path cannot be '/'") + //} var err error // check driver first @@ -210,9 +210,9 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { } storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) - if storage.MountPath == "/" { - return errors.New("Mount path cannot be '/'") - } + //if storage.MountPath == "/" { + // return errors.New("Mount path cannot be '/'") + //} err = db.UpdateStorage(&storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") From 930f9f6096a17a940573b0378add12f3b08e6aa1 Mon Sep 17 00:00:00 2001 From: Sakkyoi Cheng <22865542+sakkyoi@users.noreply.github.com> Date: Thu, 4 Sep 2025 22:15:39 +0800 Subject: [PATCH 559/659] fix(ssologin): missing role in SSO auto-registration and minor callback issue (#9305) * fix(ssologin): return after error response * fix(ssologin): set default role for SSO user creation --- server/handles/ssologin.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index eb6599e7a4d..779cc13239b 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -4,6 +4,7 @@ import ( "encoding/base64" "errors" "fmt" + "github.com/alist-org/alist/v3/internal/op" "net/http" "net/url" "path" @@ -154,7 +155,7 @@ func autoRegister(username, userID string, err error) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.SSODefaultPermission, 0)), BasePath: setting.GetStr(conf.SSODefaultDir), - Role: nil, + Role: model.Roles{op.GetDefaultRoleID()}, Disabled: false, SsoID: userID, } @@ -256,6 +257,7 @@ func OIDCLoginCallback(c *gin.Context) { user, err = autoRegister(userID, userID, err) if err != nil { common.ErrorResp(c, err, 400) + return } } token, err := common.GenerateToken(user) From fcbc79cb24971d727c8a0677c7af38c545fc7145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 5 Sep 2025 19:58:27 +0800 Subject: [PATCH 560/659] feat: Support 123pan safebox (#9311) * feat(meta): Added a SafePassword field - Added the SafePassword field to meta.go - Revised the field format to align with the code style - The SafePassword field is used to supplement the extended functionality * feat(driver): Added support for safe unlocking logic - Added safe file unlocking logic in `driver.go`, returning an error if unlocking fails. - Introduced the `safeBoxUnlocked` variable of type `sync.Map` to record the IDs of unlocked files. - Enhanced error handling logic to automatically attempt to unlock safe files and re-retrieve the file list. - Added the `IsLock` field to file types in `types.go` to identify whether they are safe files. - Added a constant definition for the `SafeBoxUnlock` interface address in `util.go`. - Added the `unlockSafeBox` method to unlock a safe with a specified file ID via the API. - Optimized the file retrieval logic to automatically call the unlock method when the safe is locked. * Refactor (driver): Optimize lock field type - Changed the `IsLock` field type from `int` to `bool` for better semantics. - Updated the check logic to use direct Boolean comparisons to improve code readability and accuracy. --- drivers/123/driver.go | 24 ++++++++++++++++++++++-- drivers/123/meta.go | 5 +++-- drivers/123/types.go | 1 + drivers/123/util.go | 26 ++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 32c053e22ab..cf221fee6d8 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -6,6 +6,8 @@ import ( "fmt" "net/http" "net/url" + "strconv" + "strings" "sync" "time" @@ -28,7 +30,8 @@ import ( type Pan123 struct { model.Storage Addition - apiRateLimit sync.Map + apiRateLimit sync.Map + safeBoxUnlocked sync.Map } func (d *Pan123) Config() driver.Config { @@ -52,9 +55,26 @@ func (d *Pan123) Drop(ctx context.Context) error { } func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if f, ok := dir.(File); ok && f.IsLock { + if err := d.unlockSafeBox(f.FileId); err != nil { + return nil, err + } + } files, err := d.getFiles(ctx, dir.GetID(), dir.GetName()) if err != nil { - return nil, err + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "safe box") || strings.Contains(err.Error(), "保险箱") { + if id, e := strconv.ParseInt(dir.GetID(), 10, 64); e == nil { + if e = d.unlockSafeBox(id); e == nil { + files, err = d.getFiles(ctx, dir.GetID(), dir.GetName()) + } else { + return nil, e + } + } + } + if err != nil { + return nil, err + } } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return src, nil diff --git a/drivers/123/meta.go b/drivers/123/meta.go index cb2cbc15ba0..6c5f013ad4a 100644 --- a/drivers/123/meta.go +++ b/drivers/123/meta.go @@ -6,8 +6,9 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + SafePassword string `json:"safe_password"` driver.RootID //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` diff --git a/drivers/123/types.go b/drivers/123/types.go index a8682c52fc9..962e5fbdac0 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -20,6 +20,7 @@ type File struct { Etag string `json:"Etag"` S3KeyFlag string `json:"S3KeyFlag"` DownloadUrl string `json:"DownloadUrl"` + IsLock bool `json:"IsLock"` } func (f File) CreateTime() time.Time { diff --git a/drivers/123/util.go b/drivers/123/util.go index b85c9afbac8..bca54b599f4 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -43,6 +43,7 @@ const ( S3Auth = MainApi + "/file/s3_upload_object/auth" UploadCompleteV2 = MainApi + "/file/upload_complete/v2" S3Complete = MainApi + "/file/s3_complete_multipart_upload" + SafeBoxUnlock = MainApi + "/restful/goapi/v1/file/safe_box/auth/unlockbox" //AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) @@ -238,6 +239,22 @@ do: return body, nil } +func (d *Pan123) unlockSafeBox(fileId int64) error { + if _, ok := d.safeBoxUnlocked.Load(fileId); ok { + return nil + } + data := base.Json{"password": d.SafePassword} + url := fmt.Sprintf("%s?fileId=%d", SafeBoxUnlock, fileId) + _, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + if err != nil { + return err + } + d.safeBoxUnlocked.Store(fileId, true) + return nil +} + func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { page := 1 total := 0 @@ -267,6 +284,15 @@ func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([] req.SetQueryParams(query) }, &resp) if err != nil { + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "safe box") || strings.Contains(err.Error(), "保险箱") { + if fid, e := strconv.ParseInt(parentId, 10, 64); e == nil { + if e = d.unlockSafeBox(fid); e == nil { + return d.getFiles(ctx, parentId, name) + } + return nil, e + } + } return nil, err } log.Debug(string(_res)) From d0026030cb36782b92feed55903e85ca14ee6ad0 Mon Sep 17 00:00:00 2001 From: "D@' 3z K!7" <99719341+Da3zKi7@users.noreply.github.com> Date: Wed, 10 Sep 2025 21:46:09 -0600 Subject: [PATCH 561/659] feat(drivers): add MediaFire driver support (#9319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Implement complete MediaFire storage driver - Add authentication via session_token and cookie - Support all core operations: List, Get, Link, Put, Copy, Move, Remove, Rename, MakeDir - Include thumbnail generation for media files - Handle MediaFire's resumable upload API with multi-unit transfers - Add proper error handling and progress reporting Closes 请求支持Mediafire #7869 Co-authored-by: Da3zKi7 --- README.md | 1 + README_cn.md | 1 + README_ja.md | 1 + drivers/all.go | 1 + drivers/mediafire/driver.go | 427 +++++++++++++++++++++++++ drivers/mediafire/meta.go | 54 ++++ drivers/mediafire/types.go | 232 ++++++++++++++ drivers/mediafire/util.go | 620 ++++++++++++++++++++++++++++++++++++ 8 files changed, 1337 insertions(+) create mode 100644 drivers/mediafire/driver.go create mode 100644 drivers/mediafire/meta.go create mode 100644 drivers/mediafire/types.go create mode 100644 drivers/mediafire/util.go diff --git a/README.md b/README.md index d352b5b3f50..b77ed804751 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,7 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) diff --git a/README_cn.md b/README_cn.md index 27f417d1698..757f5f8fb7a 100644 --- a/README_cn.md +++ b/README_cn.md @@ -57,6 +57,7 @@ - [x] [又拍云对象存储](https://www.upyun.com/products/file-storage) - [x] WebDav(支持无API的OneDrive/SharePoint) - [x] Teambition（[中国](https://www.teambition.com/ )，[国际](https://us.teambition.com/ )） + - [x] [MediaFire](https://www.mediafire.com) - [x] [分秒帧](https://www.mediatrack.cn/) - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云，共享群组) - [x] [Yandex.Disk](https://disk.yandex.com/) diff --git a/README_ja.md b/README_ja.md index cd59086078c..e6a624b0929 100644 --- a/README_ja.md +++ b/README_ja.md @@ -57,6 +57,7 @@ - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) diff --git a/drivers/all.go b/drivers/all.go index a8c8620989e..5c3cc570805 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -41,6 +41,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/lanzou" _ "github.com/alist-org/alist/v3/drivers/lenovonas_share" _ "github.com/alist-org/alist/v3/drivers/local" + _ "github.com/alist-org/alist/v3/drivers/mediafire" _ "github.com/alist-org/alist/v3/drivers/mediatrack" _ "github.com/alist-org/alist/v3/drivers/mega" _ "github.com/alist-org/alist/v3/drivers/misskey" diff --git a/drivers/mediafire/driver.go b/drivers/mediafire/driver.go new file mode 100644 index 00000000000..94d056a74aa --- /dev/null +++ b/drivers/mediafire/driver.go @@ -0,0 +1,427 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "context" + "fmt" + "net/http" + "os" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Mediafire struct { + model.Storage + Addition + + actionToken string + + appBase string + apiBase string + hostBase string + maxRetries int + + secChUa string + secChUaPlatform string + userAgent string +} + +func (d *Mediafire) Config() driver.Config { + return config +} + +func (d *Mediafire) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Mediafire) Init(ctx context.Context) error { + if d.SessionToken == "" { + return fmt.Errorf("Init :: [MediaFire] {critical} missing sessionToken") + } + + if d.Cookie == "" { + return fmt.Errorf("Init :: [MediaFire] {critical} missing Cookie") + } + + if _, err := d.getSessionToken(ctx); err != nil { + + //fmt.Printf("Init :: Obtain Session Token \n\n") + + if err := d.renewToken(ctx); err != nil { + + //fmt.Printf("Init :: Renew Session Token \n\n") + } + } + + return nil +} + +func (d *Mediafire) Drop(ctx context.Context) error { + return nil +} + +func (d *Mediafire) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return d.fileToObj(src), nil + }) +} + +func (d *Mediafire) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + + downloadUrl, err := d.getDirectDownloadLink(ctx, file.GetID()) + if err != nil { + return nil, err + } + + res, err := base.NoRedirectClient.R().SetDoNotParseResponse(true).SetContext(ctx).Get(downloadUrl) + if err != nil { + return nil, err + } + defer func() { + _ = res.RawBody().Close() + }() + + if res.StatusCode() == 302 { + downloadUrl = res.Header().Get("location") + } + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "Origin": []string{d.appBase}, + "Referer": []string{d.appBase + "/"}, + "sec-ch-ua": []string{d.secChUa}, + "sec-ch-ua-platform": []string{d.secChUaPlatform}, + "User-Agent": []string{d.userAgent}, + //"User-Agent": []string{base.UserAgent}, + }, + }, nil +} + +func (d *Mediafire) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "parent_key": parentDir.GetID(), + "foldername": dirName, + } + + var resp MediafireFolderCreateResponse + _, err := d.postForm("/folder/create.php", data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + created, _ := time.Parse("2006-01-02T15:04:05Z", resp.Response.CreatedUTC) + + return &model.ObjThumb{ + Object: model.Object{ + ID: resp.Response.FolderKey, + Name: resp.Response.Name, + Size: 0, + Modified: created, + Ctime: created, + IsFolder: true, + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/move.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key_src": srcObj.GetID(), + "folder_key_dst": dstDir.GetID(), + } + } else { + + endpoint = "/file/move.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "folder_key": dstDir.GetID(), + } + } + + var resp MediafireMoveResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return srcObj, nil +} + +func (d *Mediafire) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/update.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": srcObj.GetID(), + "foldername": newName, + } + } else { + + endpoint = "/file/update.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "filename": newName, + } + } + + var resp MediafireRenameResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/copy.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key_src": srcObj.GetID(), + "folder_key_dst": dstDir.GetID(), + } + } else { + + endpoint = "/file/copy.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "folder_key": dstDir.GetID(), + } + } + + var resp MediafireCopyResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + var newID string + if srcObj.IsDir() { + if len(resp.Response.NewFolderKeys) > 0 { + newID = resp.Response.NewFolderKeys[0] + } + } else { + if len(resp.Response.NewQuickKeys) > 0 { + newID = resp.Response.NewQuickKeys[0] + } + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: newID, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Remove(ctx context.Context, obj model.Obj) error { + var data map[string]string + var endpoint string + + if obj.IsDir() { + + endpoint = "/folder/delete.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": obj.GetID(), + } + } else { + + endpoint = "/file/delete.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": obj.GetID(), + } + } + + var resp MediafireRemoveResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return err + } + + if resp.Response.Result != "Success" { + return fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return nil +} + +func (d *Mediafire) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + _, err := d.PutResult(ctx, dstDir, file, up) + return err +} + +func (d *Mediafire) PutResult(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer tempFile.Close() + + osFile, ok := tempFile.(*os.File) + if !ok { + return nil, fmt.Errorf("expected *os.File, got %T", tempFile) + } + + fileHash, err := d.calculateSHA256(osFile) + if err != nil { + return nil, err + } + + checkResp, err := d.uploadCheck(ctx, file.GetName(), file.GetSize(), fileHash, dstDir.GetID()) + if err != nil { + return nil, err + } + + if checkResp.Response.ResumableUpload.AllUnitsReady == "yes" { + up(100.0) + } + + if checkResp.Response.HashExists == "yes" && checkResp.Response.InAccount == "yes" { + up(100.0) + existingFile, err := d.getExistingFileInfo(ctx, fileHash, file.GetName(), dstDir.GetID()) + if err == nil { + return existingFile, nil + } + } + + var pollKey string + + if checkResp.Response.ResumableUpload.AllUnitsReady != "yes" { + + var err error + + pollKey, err = d.uploadUnits(ctx, osFile, checkResp, file.GetName(), fileHash, dstDir.GetID(), up) + if err != nil { + return nil, err + } + } else { + + pollKey = checkResp.Response.ResumableUpload.UploadKey + } + + //fmt.Printf("pollKey: %+v\n", pollKey) + + pollResp, err := d.pollUpload(ctx, pollKey) + if err != nil { + return nil, err + } + + quickKey := pollResp.Response.Doupload.QuickKey + + return &model.ObjThumb{ + Object: model.Object{ + ID: quickKey, + Name: file.GetName(), + Size: file.GetSize(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *Mediafire) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Mediafire)(nil) diff --git a/drivers/mediafire/meta.go b/drivers/mediafire/meta.go new file mode 100644 index 00000000000..243d55570af --- /dev/null +++ b/drivers/mediafire/meta.go @@ -0,0 +1,54 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + //driver.RootID + + SessionToken string `json:"session_token" required:"true" type:"string" help:"Required for MediaFire API"` + Cookie string `json:"cookie" required:"true" type:"string" help:"Required for navigation"` + + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"100"` +} + +var config = driver.Config{ + Name: "MediaFire", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Mediafire{ + appBase: "https://app.mediafire.com", + apiBase: "https://www.mediafire.com/api/1.5", + hostBase: "https://www.mediafire.com", + maxRetries: 3, + secChUa: "\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"139\", \"Google Chrome\";v=\"139\"", + secChUaPlatform: "Windows", + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36", + } + }) +} diff --git a/drivers/mediafire/types.go b/drivers/mediafire/types.go new file mode 100644 index 00000000000..0073b58179c --- /dev/null +++ b/drivers/mediafire/types.go @@ -0,0 +1,232 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +type MediafireRenewTokenResponse struct { + Response struct { + Action string `json:"action"` + SessionToken string `json:"session_token"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireResponse struct { + Response struct { + Action string `json:"action"` + FolderContent struct { + ChunkSize string `json:"chunk_size"` + ContentType string `json:"content_type"` + ChunkNumber string `json:"chunk_number"` + FolderKey string `json:"folderkey"` + Folders []MediafireFolder `json:"folders,omitempty"` + Files []MediafireFile `json:"files,omitempty"` + MoreChunks string `json:"more_chunks"` + } `json:"folder_content"` + Result string `json:"result"` + } `json:"response"` +} + +type MediafireFolder struct { + FolderKey string `json:"folderkey"` + Name string `json:"name"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` +} + +type MediafireFile struct { + QuickKey string `json:"quickkey"` + Filename string `json:"filename"` + Size string `json:"size"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + MimeType string `json:"mimetype"` +} + +type File struct { + ID string + Name string + Size int64 + CreatedUTC string + IsFolder bool +} + +type FolderContentResponse struct { + Folders []MediafireFolder + Files []MediafireFile + MoreChunks bool +} + +type MediafireLinksResponse struct { + Response struct { + Action string `json:"action"` + Links []struct { + QuickKey string `json:"quickkey"` + View string `json:"view"` + NormalDownload string `json:"normal_download"` + OneTime struct { + Download string `json:"download"` + View string `json:"view"` + } `json:"one_time"` + } `json:"links"` + OneTimeKeyRequestCount string `json:"one_time_key_request_count"` + OneTimeKeyRequestMaxCount string `json:"one_time_key_request_max_count"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireDirectDownloadResponse struct { + Response struct { + Action string `json:"action"` + Links []struct { + QuickKey string `json:"quickkey"` + DirectDownload string `json:"direct_download"` + } `json:"links"` + DirectDownloadFreeBandwidth string `json:"direct_download_free_bandwidth"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireFolderCreateResponse struct { + Response struct { + Action string `json:"action"` + FolderKey string `json:"folder_key"` + UploadKey string `json:"upload_key"` + ParentFolderKey string `json:"parent_folderkey"` + Name string `json:"name"` + Description string `json:"description"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + Privacy string `json:"privacy"` + FileCount string `json:"file_count"` + FolderCount string `json:"folder_count"` + Revision string `json:"revision"` + DropboxEnabled string `json:"dropbox_enabled"` + Flag string `json:"flag"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireMoveResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + NewNames []string `json:"new_names"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireRenameResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireCopyResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + NewQuickKeys []string `json:"new_quickkeys,omitempty"` + NewFolderKeys []string `json:"new_folderkeys,omitempty"` + SkippedCount string `json:"skipped_count,omitempty"` + OtherCount string `json:"other_count,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireRemoveResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireCheckResponse struct { + Response struct { + Action string `json:"action"` + HashExists string `json:"hash_exists"` + InAccount string `json:"in_account"` + InFolder string `json:"in_folder"` + FileExists string `json:"file_exists"` + ResumableUpload struct { + AllUnitsReady string `json:"all_units_ready"` + NumberOfUnits string `json:"number_of_units"` + UnitSize string `json:"unit_size"` + Bitmap struct { + Count string `json:"count"` + Words []string `json:"words"` + } `json:"bitmap"` + UploadKey string `json:"upload_key"` + } `json:"resumable_upload"` + AvailableSpace string `json:"available_space"` + UsedStorageSize string `json:"used_storage_size"` + StorageLimit string `json:"storage_limit"` + StorageLimitExceeded string `json:"storage_limit_exceeded"` + UploadURL struct { + Simple string `json:"simple"` + SimpleFallback string `json:"simple_fallback"` + Resumable string `json:"resumable"` + ResumableFallback string `json:"resumable_fallback"` + } `json:"upload_url"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} +type MediafireActionTokenResponse struct { + Response struct { + Action string `json:"action"` + ActionToken string `json:"action_token"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafirePollResponse struct { + Response struct { + Action string `json:"action"` + Doupload struct { + Result string `json:"result"` + Status string `json:"status"` + Description string `json:"description"` + QuickKey string `json:"quickkey"` + Hash string `json:"hash"` + Filename string `json:"filename"` + Size string `json:"size"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + Revision string `json:"revision"` + } `json:"doupload"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireFileSearchResponse struct { + Response struct { + Action string `json:"action"` + FileInfo []File `json:"file_info"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} diff --git a/drivers/mediafire/util.go b/drivers/mediafire/util.go new file mode 100644 index 00000000000..42febf0bb7a --- /dev/null +++ b/drivers/mediafire/util.go @@ -0,0 +1,620 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { + tokenURL := d.hostBase + "/application/get_session_token.php" + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, tokenURL, nil) + if err != nil { + return "", err + } + + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Content-Length", "0") + req.Header.Set("Cookie", d.Cookie) + req.Header.Set("DNT", "1") + req.Header.Set("Origin", d.hostBase) + req.Header.Set("Priority", "u=1, i") + req.Header.Set("Referer", (d.hostBase + "/")) + req.Header.Set("Sec-Ch-Ua", d.secChUa) + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", d.secChUaPlatform) + req.Header.Set("Sec-Fetch-Dest", "empty") + req.Header.Set("Sec-Fetch-Mode", "cors") + req.Header.Set("Sec-Fetch-Site", "same-site") + req.Header.Set("User-Agent", d.userAgent) + //req.Header.Set("Connection", "keep-alive") + + resp, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + //fmt.Printf("getSessionToken :: Raw response: %s\n", string(body)) + //fmt.Printf("getSessionToken :: Parsed response: %+v\n", resp) + + var tokenResp struct { + Response struct { + SessionToken string `json:"session_token"` + } `json:"response"` + } + + if resp.StatusCode == 200 { + if err := json.Unmarshal(body, &tokenResp); err != nil { + return "", err + } + + if tokenResp.Response.SessionToken == "" { + return "", fmt.Errorf("empty session token received") + } + + cookieMap := make(map[string]string) + for _, cookie := range resp.Cookies() { + cookieMap[cookie.Name] = cookie.Value + } + + if len(cookieMap) > 0 { + + var cookies []string + for name, value := range cookieMap { + cookies = append(cookies, fmt.Sprintf("%s=%s", name, value)) + } + d.Cookie = strings.Join(cookies, "; ") + op.MustSaveDriverStorage(d) + + //fmt.Printf("getSessionToken :: Captured cookies: %s\n", d.Cookie) + } + + } else { + return "", fmt.Errorf("getSessionToken :: failed to get session token, status code: %d", resp.StatusCode) + } + + d.SessionToken = tokenResp.Response.SessionToken + op.MustSaveDriverStorage(d) + + return d.SessionToken, nil +} + +func (d *Mediafire) renewToken(_ context.Context) error { + query := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + } + + var resp MediafireRenewTokenResponse + _, err := d.postForm("/user/renew_session_token.php", query, &resp) + if err != nil { + return fmt.Errorf("failed to renew token: %w", err) + } + + //fmt.Printf("getInfo :: Raw response: %s\n", string(body)) + //fmt.Printf("getInfo :: Parsed response: %+v\n", resp) + + if resp.Response.Result != "Success" { + return fmt.Errorf("MediaFire token renewal failed: %s", resp.Response.Result) + } + + d.SessionToken = resp.Response.SessionToken + op.MustSaveDriverStorage(d) + + return nil +} + +func (d *Mediafire) getFiles(ctx context.Context, folderKey string) ([]File, error) { + files := make([]File, 0) + hasMore := true + chunkNumber := 1 + + for hasMore { + resp, err := d.getFolderContent(ctx, folderKey, chunkNumber) + if err != nil { + return nil, err + } + + for _, folder := range resp.Folders { + files = append(files, File{ + ID: folder.FolderKey, + Name: folder.Name, + Size: 0, + CreatedUTC: folder.CreatedUTC, + IsFolder: true, + }) + } + + for _, file := range resp.Files { + size, _ := strconv.ParseInt(file.Size, 10, 64) + files = append(files, File{ + ID: file.QuickKey, + Name: file.Filename, + Size: size, + CreatedUTC: file.CreatedUTC, + IsFolder: false, + }) + } + + hasMore = resp.MoreChunks + chunkNumber++ + } + + return files, nil +} + +func (d *Mediafire) getFolderContent(ctx context.Context, folderKey string, chunkNumber int) (*FolderContentResponse, error) { + + foldersResp, err := d.getFolderContentByType(ctx, folderKey, "folders", chunkNumber) + if err != nil { + return nil, err + } + + filesResp, err := d.getFolderContentByType(ctx, folderKey, "files", chunkNumber) + if err != nil { + return nil, err + } + + return &FolderContentResponse{ + Folders: foldersResp.Response.FolderContent.Folders, + Files: filesResp.Response.FolderContent.Files, + MoreChunks: foldersResp.Response.FolderContent.MoreChunks == "yes" || filesResp.Response.FolderContent.MoreChunks == "yes", + }, nil +} + +func (d *Mediafire) getFolderContentByType(_ context.Context, folderKey, contentType string, chunkNumber int) (*MediafireResponse, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": folderKey, + "content_type": contentType, + "chunk": strconv.Itoa(chunkNumber), + "chunk_size": strconv.FormatInt(d.ChunkSize, 10), + "details": "yes", + "order_direction": d.OrderDirection, + "order_by": d.OrderBy, + "filter": "", + } + + var resp MediafireResponse + _, err := d.postForm("/folder/get_content.php", data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) fileToObj(f File) *model.ObjThumb { + created, _ := time.Parse("2006-01-02T15:04:05Z", f.CreatedUTC) + + var thumbnailURL string + if !f.IsFolder && f.ID != "" { + thumbnailURL = d.hostBase + "/convkey/acaa/" + f.ID + "3g.jpg" + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: f.ID, + //Path: "", + Name: f.Name, + Size: f.Size, + Modified: created, + Ctime: created, + IsFolder: f.IsFolder, + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumbnailURL, + }, + } +} + +func (d *Mediafire) getForm(endpoint string, query map[string]string, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + + req.SetQueryParams(query) + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + //"User-Agent": base.UserAgent, + "User-Agent": d.userAgent, + "Origin": d.appBase, + "Referer": d.appBase + "/", + }) + + // If response OK + if resp != nil { + req.SetResult(resp) + } + + // Targets MediaFire API + res, err := req.Get(d.apiBase + endpoint) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Mediafire) postForm(endpoint string, data map[string]string, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + + req.SetFormData(data) + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "Content-Type": "application/x-www-form-urlencoded", + //"User-Agent": base.UserAgent, + "User-Agent": d.userAgent, + "Origin": d.appBase, + "Referer": d.appBase + "/", + }) + + // If response OK + if resp != nil { + req.SetResult(resp) + } + + // Targets MediaFire API + res, err := req.Post(d.apiBase + endpoint) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Mediafire) getDirectDownloadLink(_ context.Context, fileID string) (string, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "quick_key": fileID, + "link_type": "direct_download", + "response_format": "json", + } + + var resp MediafireDirectDownloadResponse + _, err := d.getForm("/file/get_links.php", data, &resp) + if err != nil { + return "", err + } + + if resp.Response.Result != "Success" { + return "", fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + if len(resp.Response.Links) == 0 { + return "", fmt.Errorf("no download links found") + } + + return resp.Response.Links[0].DirectDownload, nil +} + +func (d *Mediafire) calculateSHA256(file *os.File) (string, error) { + hasher := sha256.New() + if _, err := file.Seek(0, 0); err != nil { + return "", err + } + if _, err := io.Copy(hasher, file); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func (d *Mediafire) uploadCheck(ctx context.Context, filename string, filesize int64, filehash, folderKey string) (*MediafireCheckResponse, error) { + + actionToken, err := d.getActionToken(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get action token: %w", err) + } + + query := map[string]string{ + "session_token": actionToken, /* d.SessionToken */ + "filename": filename, + "size": strconv.FormatInt(filesize, 10), + "hash": filehash, + "folder_key": folderKey, + "resumable": "yes", + "response_format": "json", + } + + var resp MediafireCheckResponse + _, err = d.postForm("/upload/check.php", query, &resp) + if err != nil { + return nil, err + } + + //fmt.Printf("uploadCheck :: Raw response: %s\n", string(body)) + //fmt.Printf("uploadCheck :: Parsed response: %+v\n", resp) + + //fmt.Printf("uploadCheck :: ResumableUpload section: %+v\n", resp.Response.ResumableUpload) + //fmt.Printf("uploadCheck :: Upload key specifically: '%s'\n", resp.Response.ResumableUpload.UploadKey) + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire upload check failed: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) resumableUpload(ctx context.Context, folderKey, uploadKey string, unitData []byte, unitID int, fileHash, filename string, totalFileSize int64) (string, error) { + actionToken, err := d.getActionToken(ctx) + if err != nil { + return "", err + } + + url := d.apiBase + "/upload/resumable.php" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(unitData)) + if err != nil { + return "", err + } + + q := req.URL.Query() + q.Add("folder_key", folderKey) + q.Add("response_format", "json") + q.Add("session_token", actionToken) + q.Add("key", uploadKey) + req.URL.RawQuery = q.Encode() + + req.Header.Set("x-filehash", fileHash) + req.Header.Set("x-filesize", strconv.FormatInt(totalFileSize, 10)) + req.Header.Set("x-unit-id", strconv.Itoa(unitID)) + req.Header.Set("x-unit-size", strconv.FormatInt(int64(len(unitData)), 10)) + req.Header.Set("x-unit-hash", d.sha256Hex(bytes.NewReader(unitData))) + req.Header.Set("x-filename", filename) + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = int64(len(unitData)) + + /* fmt.Printf("Debug resumable upload request:\n") + fmt.Printf(" URL: %s\n", req.URL.String()) + fmt.Printf(" Headers: %+v\n", req.Header) + fmt.Printf(" Unit ID: %d\n", unitID) + fmt.Printf(" Unit Size: %d\n", len(unitData)) + fmt.Printf(" Upload Key: %s\n", uploadKey) + fmt.Printf(" Action Token: %s\n", actionToken) */ + + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %v", err) + } + + //fmt.Printf("MediaFire resumable upload response (status %d): %s\n", res.StatusCode, string(body)) + + var uploadResp struct { + Response struct { + Doupload struct { + Key string `json:"key"` + } `json:"doupload"` + Result string `json:"result"` + } `json:"response"` + } + + if err := json.Unmarshal(body, &uploadResp); err != nil { + return "", fmt.Errorf("failed to parse response: %v", err) + } + + if res.StatusCode != 200 { + return "", fmt.Errorf("resumable upload failed with status %d", res.StatusCode) + } + + return uploadResp.Response.Doupload.Key, nil +} + +func (d *Mediafire) uploadUnits(ctx context.Context, file *os.File, checkResp *MediafireCheckResponse, filename, fileHash, folderKey string, up driver.UpdateProgress) (string, error) { + unitSize, _ := strconv.ParseInt(checkResp.Response.ResumableUpload.UnitSize, 10, 64) + numUnits, _ := strconv.Atoi(checkResp.Response.ResumableUpload.NumberOfUnits) + uploadKey := checkResp.Response.ResumableUpload.UploadKey + + stringWords := checkResp.Response.ResumableUpload.Bitmap.Words + intWords := make([]int, len(stringWords)) + for i, word := range stringWords { + intWords[i], _ = strconv.Atoi(word) + } + + var finalUploadKey string + + for unitID := 0; unitID < numUnits; unitID++ { + + if utils.IsCanceled(ctx) { + return "", ctx.Err() + } + + if d.isUnitUploaded(intWords, unitID) { + up(float64(unitID+1) * 100 / float64(numUnits)) + continue + } + + uploadKey, err := d.uploadSingleUnit(ctx, file, unitID, unitSize, fileHash, filename, uploadKey, folderKey) + if err != nil { + return "", err + } + + finalUploadKey = uploadKey + + up(float64(unitID+1) * 100 / float64(numUnits)) + } + + return finalUploadKey, nil +} + +func (d *Mediafire) uploadSingleUnit(ctx context.Context, file *os.File, unitID int, unitSize int64, fileHash, filename, uploadKey, folderKey string) (string, error) { + start := int64(unitID) * unitSize + size := unitSize + + stat, err := file.Stat() + if err != nil { + return "", err + } + fileSize := stat.Size() + + if start+size > fileSize { + size = fileSize - start + } + + unitData := make([]byte, size) + if _, err := file.ReadAt(unitData, start); err != nil { + return "", err + } + + return d.resumableUpload(ctx, folderKey, uploadKey, unitData, unitID, fileHash, filename, fileSize) +} + +func (d *Mediafire) getActionToken(_ context.Context) (string, error) { + + if d.actionToken != "" { + return d.actionToken, nil + } + + data := map[string]string{ + "type": "upload", + "lifespan": "1440", + "response_format": "json", + "session_token": d.SessionToken, + } + + var resp MediafireActionTokenResponse + _, err := d.postForm("/user/get_action_token.php", data, &resp) + if err != nil { + return "", err + } + + if resp.Response.Result != "Success" { + return "", fmt.Errorf("MediaFire action token failed: %s", resp.Response.Result) + } + + return resp.Response.ActionToken, nil +} + +func (d *Mediafire) pollUpload(ctx context.Context, key string) (*MediafirePollResponse, error) { + + actionToken, err := d.getActionToken(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get action token: %w", err) + } + + //fmt.Printf("Debug Key: %+v\n", key) + + query := map[string]string{ + "key": key, + "response_format": "json", + "session_token": actionToken, /* d.SessionToken */ + } + + var resp MediafirePollResponse + _, err = d.postForm("/upload/poll_upload.php", query, &resp) + if err != nil { + return nil, err + } + + //fmt.Printf("pollUpload :: Raw response: %s\n", string(body)) + //fmt.Printf("pollUpload :: Parsed response: %+v\n", resp) + + //fmt.Printf("pollUpload :: Debug Result: %+v\n", resp.Response.Result) + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire poll upload failed: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) sha256Hex(r io.Reader) string { + h := sha256.New() + io.Copy(h, r) + return hex.EncodeToString(h.Sum(nil)) +} + +func (d *Mediafire) isUnitUploaded(words []int, unitID int) bool { + wordIndex := unitID / 16 + bitIndex := unitID % 16 + if wordIndex >= len(words) { + return false + } + return (words[wordIndex]>>bitIndex)&1 == 1 +} + +func (d *Mediafire) getExistingFileInfo(ctx context.Context, fileHash, filename, folderKey string) (*model.ObjThumb, error) { + + if fileInfo, err := d.getFileByHash(ctx, fileHash); err == nil && fileInfo != nil { + return fileInfo, nil + } + + files, err := d.getFiles(ctx, folderKey) + if err != nil { + return nil, err + } + + for _, file := range files { + if file.Name == filename && !file.IsFolder { + return d.fileToObj(file), nil + } + } + + return nil, fmt.Errorf("existing file not found") +} + +func (d *Mediafire) getFileByHash(_ context.Context, hash string) (*model.ObjThumb, error) { + query := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "hash": hash, + } + + var resp MediafireFileSearchResponse + _, err := d.postForm("/file/get_info.php", query, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire file search failed: %s", resp.Response.Result) + } + + if len(resp.Response.FileInfo) == 0 { + return nil, fmt.Errorf("file not found by hash") + } + + file := resp.Response.FileInfo[0] + return d.fileToObj(file), nil +} From 28a8428559fe8a1a4b7dd3bc6f2a7549e1ea52de Mon Sep 17 00:00:00 2001 From: Chesyre <56254560+Chesyre@users.noreply.github.com> Date: Thu, 11 Sep 2025 05:46:31 +0200 Subject: [PATCH 562/659] feat(driver): add Gofile storage driver (#9318) Add support for Gofile.io cloud storage service with full CRUD operations. Features: - File and folder listing - Upload and download functionality - Create, move, rename, copy, and delete operations - Direct link generation for file access - API token authentication The driver implements all required driver interfaces and follows the existing driver patterns in the codebase. --- drivers/all.go | 1 + drivers/gofile/driver.go | 261 +++++++++++++++++++++++++++++++++++++++ drivers/gofile/meta.go | 26 ++++ drivers/gofile/types.go | 124 +++++++++++++++++++ drivers/gofile/util.go | 257 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 669 insertions(+) create mode 100644 drivers/gofile/driver.go create mode 100644 drivers/gofile/meta.go create mode 100644 drivers/gofile/types.go create mode 100644 drivers/gofile/util.go diff --git a/drivers/all.go b/drivers/all.go index 5c3cc570805..140908a8945 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -32,6 +32,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/ftp" _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/github_releases" + _ "github.com/alist-org/alist/v3/drivers/gofile" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" _ "github.com/alist-org/alist/v3/drivers/halalcloud" diff --git a/drivers/gofile/driver.go b/drivers/gofile/driver.go new file mode 100644 index 00000000000..301eaef3405 --- /dev/null +++ b/drivers/gofile/driver.go @@ -0,0 +1,261 @@ +package gofile + +import ( + "context" + "fmt" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" +) + +type Gofile struct { + model.Storage + Addition + + accountId string +} + +func (d *Gofile) Config() driver.Config { + return config +} + +func (d *Gofile) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Gofile) Init(ctx context.Context) error { + if d.APIToken == "" { + return fmt.Errorf("API token is required") + } + + // Get account ID + accountId, err := d.getAccountId(ctx) + if err != nil { + return fmt.Errorf("failed to get account ID: %w", err) + } + d.accountId = accountId + + // Get account info to set root folder if not specified + if d.RootFolderID == "" { + accountInfo, err := d.getAccountInfo(ctx, accountId) + if err != nil { + return fmt.Errorf("failed to get account info: %w", err) + } + d.RootFolderID = accountInfo.Data.RootFolder + } + + // Save driver storage + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Gofile) Drop(ctx context.Context) error { + return nil +} + +func (d *Gofile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var folderId string + if dir.GetID() == "" { + folderId = d.GetRootId() + } else { + folderId = dir.GetID() + } + + endpoint := fmt.Sprintf("/contents/%s", folderId) + + var response ContentsResponse + err := d.getJSON(ctx, endpoint, &response) + if err != nil { + return nil, err + } + + var objects []model.Obj + + // Process children or contents + contents := response.Data.Children + if contents == nil { + contents = response.Data.Contents + } + + for _, content := range contents { + objects = append(objects, d.convertContentToObj(content)) + } + + return objects, nil +} + +func (d *Gofile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + + // Create a direct link for the file + directLink, err := d.createDirectLink(ctx, file.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to create direct link: %w", err) + } + + return &model.Link{ + URL: directLink, + }, nil +} + +func (d *Gofile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var parentId string + if parentDir.GetID() == "" { + parentId = d.GetRootId() + } else { + parentId = parentDir.GetID() + } + + data := map[string]interface{}{ + "parentFolderId": parentId, + "folderName": dirName, + } + + var response CreateFolderResponse + err := d.postJSON(ctx, "/contents/createFolder", data, &response) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: response.Data.ID, + Name: response.Data.Name, + IsFolder: true, + }, nil +} + +func (d *Gofile) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var dstId string + if dstDir.GetID() == "" { + dstId = d.GetRootId() + } else { + dstId = dstDir.GetID() + } + + data := map[string]interface{}{ + "contentsId": srcObj.GetID(), + "folderId": dstId, + } + + err := d.putJSON(ctx, "/contents/move", data, nil) + if err != nil { + return nil, err + } + + // Return updated object + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + data := map[string]interface{}{ + "attribute": "name", + "attributeValue": newName, + } + + var response UpdateResponse + err := d.putJSON(ctx, fmt.Sprintf("/contents/%s/update", srcObj.GetID()), data, &response) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var dstId string + if dstDir.GetID() == "" { + dstId = d.GetRootId() + } else { + dstId = dstDir.GetID() + } + + data := map[string]interface{}{ + "contentsId": srcObj.GetID(), + "folderId": dstId, + } + + var response CopyResponse + err := d.postJSON(ctx, "/contents/copy", data, &response) + if err != nil { + return nil, err + } + + // Get the new ID from the response + newId := srcObj.GetID() + if response.Data.CopiedContents != nil { + if id, ok := response.Data.CopiedContents[srcObj.GetID()]; ok { + newId = id + } + } + + return &model.Object{ + ID: newId, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Remove(ctx context.Context, obj model.Obj) error { + data := map[string]interface{}{ + "contentsId": obj.GetID(), + } + + return d.deleteJSON(ctx, "/contents", data) +} + +func (d *Gofile) Put(ctx context.Context, dstDir model.Obj, fileStreamer model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var folderId string + if dstDir.GetID() == "" { + folderId = d.GetRootId() + } else { + folderId = dstDir.GetID() + } + + response, err := d.uploadFile(ctx, folderId, fileStreamer, up) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: response.Data.FileId, + Name: response.Data.FileName, + Size: fileStreamer.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Gofile) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Gofile)(nil) \ No newline at end of file diff --git a/drivers/gofile/meta.go b/drivers/gofile/meta.go new file mode 100644 index 00000000000..b8126e337b4 --- /dev/null +++ b/drivers/gofile/meta.go @@ -0,0 +1,26 @@ +package gofile + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APIToken string `json:"api_token" required:"true" help:"Get your API token from your Gofile profile page"` +} + +var config = driver.Config{ + Name: "Gofile", + DefaultRoot: "", + LocalSort: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Gofile{} + }) +} \ No newline at end of file diff --git a/drivers/gofile/types.go b/drivers/gofile/types.go new file mode 100644 index 00000000000..93c9f5d2e6a --- /dev/null +++ b/drivers/gofile/types.go @@ -0,0 +1,124 @@ +package gofile + +import "time" + +type APIResponse struct { + Status string `json:"status"` + Data interface{} `json:"data"` +} + +type AccountResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + } `json:"data"` +} + +type AccountInfoResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Email string `json:"email"` + RootFolder string `json:"rootFolder"` + } `json:"data"` +} + +type Content struct { + ID string `json:"id"` + Type string `json:"type"` // "file" or "folder" + Name string `json:"name"` + Size int64 `json:"size,omitempty"` + CreateTime int64 `json:"createTime"` + ModTime int64 `json:"modTime,omitempty"` + DirectLink string `json:"directLink,omitempty"` + Children map[string]Content `json:"children,omitempty"` + ParentFolder string `json:"parentFolder,omitempty"` + MD5 string `json:"md5,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Link string `json:"link,omitempty"` +} + +type ContentsResponse struct { + Status string `json:"status"` + Data struct { + IsOwner bool `json:"isOwner"` + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ParentFolder string `json:"parentFolder"` + CreateTime int64 `json:"createTime"` + ChildrenList []string `json:"childrenList,omitempty"` + Children map[string]Content `json:"children,omitempty"` + Contents map[string]Content `json:"contents,omitempty"` + Public bool `json:"public,omitempty"` + Description string `json:"description,omitempty"` + Tags string `json:"tags,omitempty"` + Expiry int64 `json:"expiry,omitempty"` + } `json:"data"` +} + +type UploadResponse struct { + Status string `json:"status"` + Data struct { + DownloadPage string `json:"downloadPage"` + Code string `json:"code"` + ParentFolder string `json:"parentFolder"` + FileId string `json:"fileId"` + FileName string `json:"fileName"` + GuestToken string `json:"guestToken,omitempty"` + } `json:"data"` +} + +type DirectLinkResponse struct { + Status string `json:"status"` + Data struct { + DirectLink string `json:"directLink"` + ID string `json:"id"` + } `json:"data"` +} + +type CreateFolderResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ParentFolder string `json:"parentFolder"` + CreateTime int64 `json:"createTime"` + } `json:"data"` +} + +type CopyResponse struct { + Status string `json:"status"` + Data struct { + CopiedContents map[string]string `json:"copiedContents"` // oldId -> newId mapping + } `json:"data"` +} + +type UpdateResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"data"` +} + +type ErrorResponse struct { + Status string `json:"status"` + Error struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error"` +} + +func (c *Content) ModifiedTime() time.Time { + if c.ModTime > 0 { + return time.Unix(c.ModTime, 0) + } + return time.Unix(c.CreateTime, 0) +} + +func (c *Content) IsDir() bool { + return c.Type == "folder" +} \ No newline at end of file diff --git a/drivers/gofile/util.go b/drivers/gofile/util.go new file mode 100644 index 00000000000..5f39dae5a81 --- /dev/null +++ b/drivers/gofile/util.go @@ -0,0 +1,257 @@ +package gofile + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) + +const ( + baseAPI = "https://api.gofile.io" + uploadAPI = "https://upload.gofile.io" +) + +func (d *Gofile) request(ctx context.Context, method, endpoint string, body io.Reader, headers map[string]string) (*http.Response, error) { + var url string + if strings.HasPrefix(endpoint, "http") { + url = endpoint + } else { + url = baseAPI + endpoint + } + + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", "Bearer "+d.APIToken) + req.Header.Set("User-Agent", "AList/3.0") + + for k, v := range headers { + req.Header.Set(k, v) + } + + return base.HttpClient.Do(req) +} + +func (d *Gofile) getJSON(ctx context.Context, endpoint string, result interface{}) error { + resp, err := d.request(ctx, "GET", endpoint, nil, nil) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + return json.NewDecoder(resp.Body).Decode(result) +} + +func (d *Gofile) postJSON(ctx context.Context, endpoint string, data interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "POST", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + if result != nil { + return json.NewDecoder(resp.Body).Decode(result) + } + + return nil +} + +func (d *Gofile) putJSON(ctx context.Context, endpoint string, data interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "PUT", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + if result != nil { + return json.NewDecoder(resp.Body).Decode(result) + } + + return nil +} + +func (d *Gofile) deleteJSON(ctx context.Context, endpoint string, data interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "DELETE", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + return nil +} + +func (d *Gofile) handleError(resp *http.Response) error { + body, _ := io.ReadAll(resp.Body) + + var errorResp ErrorResponse + if err := json.Unmarshal(body, &errorResp); err == nil { + return fmt.Errorf("gofile API error: %s (code: %s)", errorResp.Error.Message, errorResp.Error.Code) + } + + return fmt.Errorf("gofile API error: HTTP %d - %s", resp.StatusCode, string(body)) +} + +func (d *Gofile) uploadFile(ctx context.Context, folderId string, file model.FileStreamer, up driver.UpdateProgress) (*UploadResponse, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + if folderId != "" { + writer.WriteField("folderId", folderId) + } + + part, err := writer.CreateFormFile("file", filepath.Base(file.GetName())) + if err != nil { + return nil, err + } + + // Copy with progress tracking if available + if up != nil { + reader := &progressReader{ + reader: file, + total: file.GetSize(), + up: up, + } + _, err = io.Copy(part, reader) + } else { + _, err = io.Copy(part, file) + } + + if err != nil { + return nil, err + } + + writer.Close() + + headers := map[string]string{ + "Content-Type": writer.FormDataContentType(), + } + + resp, err := d.request(ctx, "POST", uploadAPI+"/uploadfile", &body, headers) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, d.handleError(resp) + } + + var result UploadResponse + err = json.NewDecoder(resp.Body).Decode(&result) + return &result, err +} + +func (d *Gofile) createDirectLink(ctx context.Context, contentId string) (string, error) { + data := map[string]interface{}{} + + var result DirectLinkResponse + err := d.postJSON(ctx, fmt.Sprintf("/contents/%s/directlinks", contentId), data, &result) + if err != nil { + return "", err + } + + return result.Data.DirectLink, nil +} + +func (d *Gofile) convertContentToObj(content Content) model.Obj { + return &model.ObjThumb{ + Object: model.Object{ + ID: content.ID, + Name: content.Name, + Size: content.Size, + Modified: content.ModifiedTime(), + IsFolder: content.IsDir(), + }, + } +} + +func (d *Gofile) getAccountId(ctx context.Context) (string, error) { + var result AccountResponse + err := d.getJSON(ctx, "/accounts/getid", &result) + if err != nil { + return "", err + } + return result.Data.ID, nil +} + +func (d *Gofile) getAccountInfo(ctx context.Context, accountId string) (*AccountInfoResponse, error) { + var result AccountInfoResponse + err := d.getJSON(ctx, fmt.Sprintf("/accounts/%s", accountId), &result) + if err != nil { + return nil, err + } + return &result, nil +} + +// progressReader wraps an io.Reader to track upload progress +type progressReader struct { + reader io.Reader + total int64 + read int64 + up driver.UpdateProgress +} + +func (pr *progressReader) Read(p []byte) (n int, err error) { + n, err = pr.reader.Read(p) + pr.read += int64(n) + if pr.up != nil && pr.total > 0 { + progress := float64(pr.read) * 100 / float64(pr.total) + pr.up(progress) + } + return n, err +} From 6e7c7d1dd02305c29a04f0492f6154cb14e77378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 11 Sep 2025 21:16:33 +0800 Subject: [PATCH 563/659] refactor (auth): Optimize permission path processing logic (#9320) - Changed permission path collection from map to slice to improve code readability - Removed redundant path checks to improve path addition efficiency - Restructured the loop logic for path processing to simplify the path permission assignment process --- server/handles/auth.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/server/handles/auth.go b/server/handles/auth.go index dd7d202b907..3520a459c3d 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -165,25 +165,25 @@ func CurrentUser(c *gin.Context) { var roleNames []string permMap := map[string]int32{} - addedPaths := map[string]bool{} + paths := make([]string, 0) for _, role := range user.RolesDetail { roleNames = append(roleNames, role.Name) for _, entry := range role.PermissionScopes { cleanPath := path.Clean("/" + strings.TrimPrefix(entry.Path, "/")) + if _, ok := permMap[cleanPath]; !ok { + paths = append(paths, cleanPath) + } permMap[cleanPath] |= entry.Permission } } userResp.RoleNames = roleNames - for fullPath, perm := range permMap { - if !addedPaths[fullPath] { - userResp.Permissions = append(userResp.Permissions, model.PermissionEntry{ - Path: fullPath, - Permission: perm, - }) - addedPaths[fullPath] = true - } + for _, fullPath := range paths { + userResp.Permissions = append(userResp.Permissions, model.PermissionEntry{ + Path: fullPath, + Permission: permMap[fullPath], + }) } common.SuccessResp(c, userResp) From 16cce37947cab989863a2d444a020ef1c3c46f26 Mon Sep 17 00:00:00 2001 From: "D@' 3z K!7" <99719341+Da3zKi7@users.noreply.github.com> Date: Fri, 12 Sep 2025 03:53:47 -0600 Subject: [PATCH 564/659] fix(drivers): add session renewal cron for MediaFire driver (#9321) - Implement automatic session token renewal every 6-9 minutes - Add validation for required SessionToken and Cookie fields in Init - Handle session expiration by calling renewToken on validation failure - Prevent storage failures due to MediaFire session timeouts Fixes session closure issues that occur after server restarts or extended periods. Co-authored-by: Da3zKi7 --- drivers/mediafire/driver.go | 14 ++++++++++---- drivers/mediafire/util.go | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/drivers/mediafire/driver.go b/drivers/mediafire/driver.go index 94d056a74aa..e77510eabc0 100644 --- a/drivers/mediafire/driver.go +++ b/drivers/mediafire/driver.go @@ -11,6 +11,7 @@ D@' 3z K!7 - The King Of Cracking import ( "context" "fmt" + "math/rand" "net/http" "os" "time" @@ -19,12 +20,14 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/pkg/utils" ) type Mediafire struct { model.Storage Addition + cron *cron.Cron actionToken string @@ -57,12 +60,15 @@ func (d *Mediafire) Init(ctx context.Context) error { if _, err := d.getSessionToken(ctx); err != nil { - //fmt.Printf("Init :: Obtain Session Token \n\n") + d.renewToken(ctx) - if err := d.renewToken(ctx); err != nil { + num := rand.Intn(4) + 6 + + d.cron = cron.NewCron(time.Minute * time.Duration(num)) + d.cron.Do(func() { + d.renewToken(ctx) + }) - //fmt.Printf("Init :: Renew Session Token \n\n") - } } return nil diff --git a/drivers/mediafire/util.go b/drivers/mediafire/util.go index 42febf0bb7a..091abd0cd64 100644 --- a/drivers/mediafire/util.go +++ b/drivers/mediafire/util.go @@ -106,6 +106,9 @@ func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { } d.SessionToken = tokenResp.Response.SessionToken + + //fmt.Printf("Init :: Obtain Session Token %v", d.SessionToken) + op.MustSaveDriverStorage(d) return d.SessionToken, nil @@ -131,6 +134,9 @@ func (d *Mediafire) renewToken(_ context.Context) error { } d.SessionToken = resp.Response.SessionToken + + //fmt.Printf("Init :: Renew Session Token: %s", resp.Response.Result) + op.MustSaveDriverStorage(d) return nil From e1800f18e4e3746d7792a9064fa2fc0a7fef2b8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 12 Sep 2025 17:56:23 +0800 Subject: [PATCH 565/659] feat: Check usage before deleting storage (#9322) * feat(storage): Added role and user path checking functionality - Added `GetAllRoles` function to retrieve all roles - Added `GetAllUsers` function to retrieve all users - Added `firstPathSegment` function to extract the first segment of a path - Checks whether a storage object is used by a role or user, and returns relevant information for unusing it * fix(storage): Fixed a potential null value issue with not checking firstMount. - Added a check to see if `firstMount` is null to prevent logic errors. - Adjusted the loading logic of `GetAllRoles` and `GetAllUsers` to only execute when `firstMount` is non-null. - Fixed the `usedBy` check logic to ensure that an error message is returned under the correct conditions. - Optimized code structure to reduce unnecessary execution paths. --- internal/db/role.go | 8 ++++++++ internal/db/user.go | 8 ++++++++ internal/op/storage.go | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) diff --git a/internal/db/role.go b/internal/db/role.go index ae62a8ed898..808a6f5f06b 100644 --- a/internal/db/role.go +++ b/internal/db/role.go @@ -34,6 +34,14 @@ func GetRoles(pageIndex, pageSize int) (roles []model.Role, count int64, err err return roles, count, nil } +func GetAllRoles() ([]model.Role, error) { + var roles []model.Role + if err := db.Find(&roles).Error; err != nil { + return nil, errors.WithStack(err) + } + return roles, nil +} + func CreateRole(r *model.Role) error { if err := db.Create(r).Error; err != nil { return errors.WithStack(err) diff --git a/internal/db/user.go b/internal/db/user.go index 8f1c28b92f2..4e5d67ad28e 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -83,6 +83,14 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err return users, count, nil } +func GetAllUsers() ([]model.User, error) { + var users []model.User + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithStack(err) + } + return users, nil +} + func DeleteUserById(id uint) error { return errors.WithStack(db.Delete(&model.User{}, id).Error) } diff --git a/internal/op/storage.go b/internal/op/storage.go index 27221e70e8c..dfb305aaa43 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -41,6 +41,18 @@ func GetStorageByMountPath(mountPath string) (driver.Driver, error) { return storageDriver, nil } +func firstPathSegment(p string) string { + p = utils.FixAndCleanPath(p) + p = strings.TrimPrefix(p, "/") + if p == "" { + return "" + } + if i := strings.Index(p, "/"); i >= 0 { + return p[:i] + } + return p +} + // CreateStorage Save the storage to database so storage can get an id // then instantiate corresponding driver and save it in memory func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { @@ -267,6 +279,34 @@ func DeleteStorageById(ctx context.Context, id uint) error { if err != nil { return errors.WithMessage(err, "failed get storage") } + firstMount := firstPathSegment(storage.MountPath) + if firstMount != "" { + roles, err := db.GetAllRoles() + if err != nil { + return errors.WithMessage(err, "failed to load roles") + } + users, err := db.GetAllUsers() + if err != nil { + return errors.WithMessage(err, "failed to load users") + } + var usedBy []string + for _, r := range roles { + for _, entry := range r.PermissionScopes { + if firstPathSegment(entry.Path) == firstMount { + usedBy = append(usedBy, "role:"+r.Name) + break + } + } + } + for _, u := range users { + if firstPathSegment(u.BasePath) == firstMount { + usedBy = append(usedBy, "user:"+u.Username) + } + } + if len(usedBy) > 0 { + return errors.Errorf("storage is used by %s, please cancel usage first", strings.Join(usedBy, ", ")) + } + } if !storage.Disabled { storageDriver, err := GetStorageByMountPath(storage.MountPath) if err != nil { From 4f8bc478d51c6885f168571b718e7341ec8a2653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sun, 14 Sep 2025 21:03:58 +0800 Subject: [PATCH 566/659] refactor(driver): Refactored directory link check logic (#9324) - Use `filePath` variable to simplify path handling - Replace `isSymlinkDir` with `isLinkedDir` in `isFolder` check - Use simplified path variables in `times.Stat` function calls refactor(util): Optimized directory link check functions - Renamed `isSymlinkDir` to `isLinkedDir` to expand Windows platform support - Corrected path resolution logic to ensure link paths are absolute - Added error handling to prevent path resolution failures --- drivers/local/driver.go | 9 +++++---- drivers/local/util.go | 13 +++++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index faa2b3bd157..7ff72d11cdf 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -146,13 +146,14 @@ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) } } - isFolder := f.IsDir() || isSymlinkDir(f, fullPath) + filePath := filepath.Join(fullPath, f.Name()) + isFolder := f.IsDir() || isLinkedDir(f, filePath) var size int64 if !isFolder { size = f.Size() } var ctime time.Time - t, err := times.Stat(stdpath.Join(fullPath, f.Name())) + t, err := times.Stat(filePath) if err == nil { if t.HasBirthTime() { ctime = t.BirthTime() @@ -161,7 +162,7 @@ func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string file := model.ObjThumb{ Object: model.Object{ - Path: filepath.Join(fullPath, f.Name()), + Path: filePath, Name: f.Name(), Modified: f.ModTime(), Size: size, @@ -197,7 +198,7 @@ func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) { } return nil, err } - isFolder := f.IsDir() || isSymlinkDir(f, path) + isFolder := f.IsDir() || isLinkedDir(f, path) size := f.Size() if isFolder { size = 0 diff --git a/drivers/local/util.go b/drivers/local/util.go index 802f60cf627..b9df717fb40 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -7,6 +7,7 @@ import ( "io/fs" "os" "path/filepath" + "runtime" "sort" "strconv" "strings" @@ -18,14 +19,18 @@ import ( ffmpeg "github.com/u2takey/ffmpeg-go" ) -func isSymlinkDir(f fs.FileInfo, path string) bool { - if f.Mode()&os.ModeSymlink == os.ModeSymlink { - dst, err := os.Readlink(filepath.Join(path, f.Name())) +func isLinkedDir(f fs.FileInfo, path string) bool { + if f.Mode()&os.ModeSymlink == os.ModeSymlink || (runtime.GOOS == "windows" && f.Mode()&os.ModeIrregular != 0) { + dst, err := os.Readlink(path) if err != nil { return false } if !filepath.IsAbs(dst) { - dst = filepath.Join(path, dst) + dst = filepath.Join(filepath.Dir(path), dst) + } + dst, err = filepath.Abs(dst) + if err != nil { + return false } stat, err := os.Stat(dst) if err != nil { From d17889bf8e592192eb321aa2bd640c1bf97dfadf Mon Sep 17 00:00:00 2001 From: Chesyre <56254560+Chesyre@users.noreply.github.com> Date: Tue, 30 Sep 2025 08:16:28 +0200 Subject: [PATCH 567/659] feat(gofile): add configurable link expiration handling (#9329) * feat(driver): add Gofile storage driver Add support for Gofile.io cloud storage service with full CRUD operations. Features: - File and folder listing - Upload and download functionality - Create, move, rename, copy, and delete operations - Direct link generation for file access - API token authentication The driver implements all required driver interfaces and follows the existing driver patterns in the codebase. * feat(gofile): add configurable link expiration handling - Adjusts driver addition metadata to accept LinkExpiry and DirectLinkExpiry options for caching and API expiry control (drivers/gofile/meta.go:10). - Applies the new options when building file links, setting optional local cache expiration (drivers/gofile/driver.go:101) and sending an expireTime to the direct-link API (drivers/gofile/util.go:202). - Logs Gofile API error payloads and validates the structured error response before returning it (drivers/gofile/util.go:141). - Adds the required imports and returns the configured model.Link instance (drivers/gofile/driver.go:6). --- drivers/gofile/driver.go | 18 ++++++++++++++---- drivers/gofile/meta.go | 24 +++++++++++++----------- drivers/gofile/types.go | 2 +- drivers/gofile/util.go | 10 +++++++++- 4 files changed, 37 insertions(+), 17 deletions(-) diff --git a/drivers/gofile/driver.go b/drivers/gofile/driver.go index 301eaef3405..8046bd163fb 100644 --- a/drivers/gofile/driver.go +++ b/drivers/gofile/driver.go @@ -3,6 +3,7 @@ package gofile import ( "context" "fmt" + "time" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -72,7 +73,7 @@ func (d *Gofile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( } var objects []model.Obj - + // Process children or contents contents := response.Data.Children if contents == nil { @@ -97,9 +98,18 @@ func (d *Gofile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("failed to create direct link: %w", err) } - return &model.Link{ + // Configure cache expiration based on user setting + link := &model.Link{ URL: directLink, - }, nil + } + + // Only set expiration if LinkExpiry > 0 (0 means no caching) + if d.LinkExpiry > 0 { + expiration := time.Duration(d.LinkExpiry) * 24 * time.Hour + link.Expiration = &expiration + } + + return link, nil } func (d *Gofile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { @@ -258,4 +268,4 @@ func (d *Gofile) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj return nil, errs.NotImplement } -var _ driver.Driver = (*Gofile)(nil) \ No newline at end of file +var _ driver.Driver = (*Gofile)(nil) diff --git a/drivers/gofile/meta.go b/drivers/gofile/meta.go index b8126e337b4..00656025770 100644 --- a/drivers/gofile/meta.go +++ b/drivers/gofile/meta.go @@ -1,26 +1,28 @@ package gofile import ( - "github.com/alist-org/alist/v3/internal/driver" - "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" ) type Addition struct { - driver.RootID - APIToken string `json:"api_token" required:"true" help:"Get your API token from your Gofile profile page"` + driver.RootID + APIToken string `json:"api_token" required:"true" help:"Get your API token from your Gofile profile page"` + LinkExpiry int `json:"link_expiry" type:"number" default:"30" help:"Direct link cache duration in days. Set to 0 to disable caching"` + DirectLinkExpiry int `json:"direct_link_expiry" type:"number" default:"0" help:"Direct link expiration time in hours on Gofile server. Set to 0 for no expiration"` } var config = driver.Config{ - Name: "Gofile", - DefaultRoot: "", - LocalSort: false, - OnlyProxy: false, - NoCache: false, - NoUpload: false, + Name: "Gofile", + DefaultRoot: "", + LocalSort: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, } func init() { op.RegisterDriver(func() driver.Driver { return &Gofile{} }) -} \ No newline at end of file +} diff --git a/drivers/gofile/types.go b/drivers/gofile/types.go index 93c9f5d2e6a..be307347081 100644 --- a/drivers/gofile/types.go +++ b/drivers/gofile/types.go @@ -121,4 +121,4 @@ func (c *Content) ModifiedTime() time.Time { func (c *Content) IsDir() bool { return c.Type == "folder" -} \ No newline at end of file +} diff --git a/drivers/gofile/util.go b/drivers/gofile/util.go index 5f39dae5a81..1dd6229a773 100644 --- a/drivers/gofile/util.go +++ b/drivers/gofile/util.go @@ -10,10 +10,12 @@ import ( "net/http" "path/filepath" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + log "github.com/sirupsen/logrus" ) const ( @@ -137,9 +139,10 @@ func (d *Gofile) deleteJSON(ctx context.Context, endpoint string, data interface func (d *Gofile) handleError(resp *http.Response) error { body, _ := io.ReadAll(resp.Body) + log.Debugf("Gofile API error (HTTP %d): %s", resp.StatusCode, string(body)) var errorResp ErrorResponse - if err := json.Unmarshal(body, &errorResp); err == nil { + if err := json.Unmarshal(body, &errorResp); err == nil && errorResp.Status == "error" { return fmt.Errorf("gofile API error: %s (code: %s)", errorResp.Error.Message, errorResp.Error.Code) } @@ -199,6 +202,11 @@ func (d *Gofile) uploadFile(ctx context.Context, folderId string, file model.Fil func (d *Gofile) createDirectLink(ctx context.Context, contentId string) (string, error) { data := map[string]interface{}{} + if d.DirectLinkExpiry > 0 { + expireTime := time.Now().Add(time.Duration(d.DirectLinkExpiry) * time.Hour).Unix() + data["expireTime"] = expireTime + } + var result DirectLinkResponse err := d.postJSON(ctx, fmt.Sprintf("/contents/%s/directlinks", contentId), data, &result) if err != nil { From fe564c42dac0015e5c98baba67934209b2485be8 Mon Sep 17 00:00:00 2001 From: textrix Date: Tue, 30 Sep 2025 15:17:54 +0900 Subject: [PATCH 568/659] feat: add pCloud driver support (#9339) - Implement OAuth2 authentication with US/EU region support - Add file operations (list, upload, download, delete, rename, move, copy) - Add folder operations (create, rename, move, delete) - Enhance error handling with pCloud-specific retry logic - Use correct API methods: GET for reads, POST for writes - Implement direct upload approach for better performance - Add exponential backoff for failed requests with 4xxx/5xxx classification --- drivers/all.go | 1 + drivers/pcloud/driver.go | 189 +++++++++++++++++++++++++ drivers/pcloud/meta.go | 30 ++++ drivers/pcloud/types.go | 91 ++++++++++++ drivers/pcloud/util.go | 297 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 608 insertions(+) create mode 100644 drivers/pcloud/driver.go create mode 100644 drivers/pcloud/meta.go create mode 100644 drivers/pcloud/types.go create mode 100644 drivers/pcloud/util.go diff --git a/drivers/all.go b/drivers/all.go index 140908a8945..2ce0c2c6846 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -51,6 +51,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/onedrive" _ "github.com/alist-org/alist/v3/drivers/onedrive_app" _ "github.com/alist-org/alist/v3/drivers/onedrive_sharelink" + _ "github.com/alist-org/alist/v3/drivers/pcloud" _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" _ "github.com/alist-org/alist/v3/drivers/quark_uc" diff --git a/drivers/pcloud/driver.go b/drivers/pcloud/driver.go new file mode 100644 index 00000000000..036dcc40e6c --- /dev/null +++ b/drivers/pcloud/driver.go @@ -0,0 +1,189 @@ +package pcloud + +import ( + "context" + "fmt" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type PCloud struct { + model.Storage + Addition + AccessToken string // Actual access token obtained from refresh token +} + +func (d *PCloud) Config() driver.Config { + return config +} + +func (d *PCloud) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *PCloud) Init(ctx context.Context) error { + // Map hostname selection to actual API endpoints + if d.Hostname == "us" { + d.Hostname = "api.pcloud.com" + } else if d.Hostname == "eu" { + d.Hostname = "eapi.pcloud.com" + } + + // Set default root folder ID if not provided + if d.RootFolderID == "" { + d.RootFolderID = "d0" + } + + // Use the access token directly (like rclone) + d.AccessToken = d.RefreshToken // RefreshToken field actually contains the access_token + return nil +} + +func (d *PCloud) Drop(ctx context.Context) error { + return nil +} + +func (d *PCloud) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = dir.GetID() + } + + files, err := d.getFiles(folderID) + if err != nil { + return nil, err + } + + return utils.SliceConvert(files, func(src FileObject) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *PCloud) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + downloadURL, err := d.getDownloadLink(file.GetID()) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: downloadURL, + }, nil +} + +// Mkdir implements driver.Mkdir +func (d *PCloud) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + parentID := d.RootFolderID + if parentDir.GetID() != "" { + parentID = parentDir.GetID() + } + + return d.createFolder(parentID, dirName) +} + +// Move implements driver.Move +func (d *PCloud) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + // pCloud uses renamefile/renamefolder for both rename and move + endpoint := "/renamefile" + paramName := "fileid" + + if srcObj.IsDir() { + endpoint = "/renamefolder" + paramName = "folderid" + } + + var resp ItemResult + _, err := d.requestWithRetry(endpoint, "POST", func(req *resty.Request) { + req.SetFormData(map[string]string{ + paramName: extractID(srcObj.GetID()), + "tofolderid": extractID(dstDir.GetID()), + "toname": srcObj.GetName(), + }) + }, &resp) + + if err != nil { + return err + } + + if resp.Result != 0 { + return fmt.Errorf("pCloud error: result code %d", resp.Result) + } + + return nil +} + +// Rename implements driver.Rename +func (d *PCloud) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + endpoint := "/renamefile" + paramName := "fileid" + + if srcObj.IsDir() { + endpoint = "/renamefolder" + paramName = "folderid" + } + + var resp ItemResult + _, err := d.requestWithRetry(endpoint, "POST", func(req *resty.Request) { + req.SetFormData(map[string]string{ + paramName: extractID(srcObj.GetID()), + "toname": newName, + }) + }, &resp) + + if err != nil { + return err + } + + if resp.Result != 0 { + return fmt.Errorf("pCloud error: result code %d", resp.Result) + } + + return nil +} + +// Copy implements driver.Copy +func (d *PCloud) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + endpoint := "/copyfile" + paramName := "fileid" + + if srcObj.IsDir() { + endpoint = "/copyfolder" + paramName = "folderid" + } + + var resp ItemResult + _, err := d.requestWithRetry(endpoint, "POST", func(req *resty.Request) { + req.SetFormData(map[string]string{ + paramName: extractID(srcObj.GetID()), + "tofolderid": extractID(dstDir.GetID()), + "toname": srcObj.GetName(), + }) + }, &resp) + + if err != nil { + return err + } + + if resp.Result != 0 { + return fmt.Errorf("pCloud error: result code %d", resp.Result) + } + + return nil +} + +// Remove implements driver.Remove +func (d *PCloud) Remove(ctx context.Context, obj model.Obj) error { + return d.delete(obj.GetID(), obj.IsDir()) +} + +// Put implements driver.Put +func (d *PCloud) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + parentID := d.RootFolderID + if dstDir.GetID() != "" { + parentID = dstDir.GetID() + } + + return d.uploadFile(ctx, stream, parentID, stream.GetName(), stream.GetSize()) +} \ No newline at end of file diff --git a/drivers/pcloud/meta.go b/drivers/pcloud/meta.go new file mode 100644 index 00000000000..84e3dfe474f --- /dev/null +++ b/drivers/pcloud/meta.go @@ -0,0 +1,30 @@ +package pcloud + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Using json tag "access_token" for UI display, but internally it's a refresh token + RefreshToken string `json:"access_token" required:"true" help:"OAuth token from pCloud authorization"` + Hostname string `json:"hostname" type:"select" options:"us,eu" default:"us" help:"Select pCloud server region"` + RootFolderID string `json:"root_folder_id" help:"Get folder ID from URL like https://my.pcloud.com/#/filemanager?folder=12345678901 (leave empty for root folder)"` + ClientID string `json:"client_id" help:"Custom OAuth client ID (optional)"` + ClientSecret string `json:"client_secret" help:"Custom OAuth client secret (optional)"` +} + +// Implement IRootId interface +func (a Addition) GetRootId() string { + return a.RootFolderID +} + +var config = driver.Config{ + Name: "pCloud", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &PCloud{} + }) +} \ No newline at end of file diff --git a/drivers/pcloud/types.go b/drivers/pcloud/types.go new file mode 100644 index 00000000000..d0a6943c347 --- /dev/null +++ b/drivers/pcloud/types.go @@ -0,0 +1,91 @@ +package pcloud + +import ( + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +// ErrorResult represents a pCloud API error response +type ErrorResult struct { + Result int `json:"result"` + Error string `json:"error"` +} + +// TokenResponse represents OAuth token response +type TokenResponse struct { + AccessToken string `json:"access_token"` + TokenType string `json:"token_type"` +} + +// ItemResult represents a common pCloud API response +type ItemResult struct { + Result int `json:"result"` + Metadata *FolderMeta `json:"metadata,omitempty"` +} + +// FolderMeta contains folder metadata including contents +type FolderMeta struct { + Contents []FileObject `json:"contents,omitempty"` +} + +// DownloadLinkResult represents download link response +type DownloadLinkResult struct { + Result int `json:"result"` + Hosts []string `json:"hosts"` + Path string `json:"path"` +} + +// FileObject represents a file or folder object in pCloud +type FileObject struct { + Name string `json:"name"` + Created string `json:"created"` // pCloud returns RFC1123 format string + Modified string `json:"modified"` // pCloud returns RFC1123 format string + IsFolder bool `json:"isfolder"` + FolderID uint64 `json:"folderid,omitempty"` + FileID uint64 `json:"fileid,omitempty"` + Size uint64 `json:"size"` + ParentID uint64 `json:"parentfolderid"` + Icon string `json:"icon,omitempty"` + Hash uint64 `json:"hash,omitempty"` + Category int `json:"category,omitempty"` + ID string `json:"id,omitempty"` +} + +// Convert FileObject to model.Obj +func fileToObj(f FileObject) model.Obj { + // Parse RFC1123 format time from pCloud + modTime, _ := time.Parse(time.RFC1123, f.Modified) + + obj := model.Object{ + Name: f.Name, + Size: int64(f.Size), + Modified: modTime, + IsFolder: f.IsFolder, + } + + if f.IsFolder { + obj.ID = "d" + strconv.FormatUint(f.FolderID, 10) + } else { + obj.ID = "f" + strconv.FormatUint(f.FileID, 10) + } + + return &obj +} + +// Extract numeric ID from string ID (remove 'd' or 'f' prefix) +func extractID(id string) string { + if len(id) > 1 && (id[0] == 'd' || id[0] == 'f') { + return id[1:] + } + return id +} + +// Get folder ID from path, return "0" for root +func getFolderID(path string) string { + if path == "/" || path == "" { + return "0" + } + return extractID(path) +} \ No newline at end of file diff --git a/drivers/pcloud/util.go b/drivers/pcloud/util.go new file mode 100644 index 00000000000..f2c1875e133 --- /dev/null +++ b/drivers/pcloud/util.go @@ -0,0 +1,297 @@ +package pcloud + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + defaultClientID = "DnONSzyJXpm" + defaultClientSecret = "VKEnd3ze4jsKFGg8TJiznwFG8" +) + +// Get API base URL +func (d *PCloud) getAPIURL() string { + return "https://" + d.Hostname +} + +// Get OAuth client credentials +func (d *PCloud) getClientCredentials() (string, string) { + clientID := d.ClientID + clientSecret := d.ClientSecret + + if clientID == "" { + clientID = defaultClientID + } + if clientSecret == "" { + clientSecret = defaultClientSecret + } + + return clientID, clientSecret +} + +// Refresh OAuth access token +func (d *PCloud) refreshToken() error { + clientID, clientSecret := d.getClientCredentials() + + var resp TokenResponse + _, err := base.RestyClient.R(). + SetFormData(map[string]string{ + "client_id": clientID, + "client_secret": clientSecret, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&resp). + Post(d.getAPIURL() + "/oauth2_token") + + if err != nil { + return err + } + + d.AccessToken = resp.AccessToken + return nil +} + +// shouldRetry determines if an error should be retried based on pCloud-specific logic +func (d *PCloud) shouldRetry(statusCode int, apiError *ErrorResult) bool { + // HTTP-level retry conditions + if statusCode == 429 || statusCode >= 500 { + return true + } + + // pCloud API-specific retry conditions (like rclone) + if apiError != nil && apiError.Result != 0 { + // 4xxx: rate limiting + if apiError.Result/1000 == 4 { + return true + } + // 5xxx: internal errors + if apiError.Result/1000 == 5 { + return true + } + } + + return false +} + +// requestWithRetry makes authenticated API request with retry logic +func (d *PCloud) requestWithRetry(endpoint string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + maxRetries := 3 + baseDelay := 500 * time.Millisecond + + for attempt := 0; attempt <= maxRetries; attempt++ { + body, err := d.request(endpoint, method, callback, resp) + if err == nil { + return body, nil + } + + // If this is the last attempt, return the error + if attempt == maxRetries { + return nil, err + } + + // Check if we should retry based on error type + if !d.shouldRetryError(err) { + return nil, err + } + + // Exponential backoff + delay := baseDelay * time.Duration(1< Date: Tue, 30 Sep 2025 00:18:58 -0600 Subject: [PATCH 569/659] feat(drivers): add ProtonDrive driver (#9331) - Implement complete ProtonDrive storage driver with end-to-end encryption support - Add authentication via username/password with credential caching and reusable login - Support all core operations: List, Link, Put, Copy, Move, Remove, Rename, MakeDir - Include encrypted file operations with PGP key management and node passphrase handling - Add temporary HTTP server for secure file downloads with range request support - Support media streaming using temp server range requests - Implement progress tracking for uploads and downloads - Support directory operations with circular move detection - Add proper error handling and panic recovery for external library integration Closes #9312 --- README.md | 1 + README_cn.md | 1 + README_ja.md | 1 + drivers/all.go | 1 + drivers/proton_drive/driver.go | 418 +++++++++++++++ drivers/proton_drive/meta.go | 69 +++ drivers/proton_drive/types.go | 124 +++++ drivers/proton_drive/util.go | 918 +++++++++++++++++++++++++++++++++ go.mod | 23 +- go.sum | 45 ++ 10 files changed, 1600 insertions(+), 1 deletion(-) create mode 100644 drivers/proton_drive/driver.go create mode 100644 drivers/proton_drive/meta.go create mode 100644 drivers/proton_drive/types.go create mode 100644 drivers/proton_drive/util.go diff --git a/README.md b/README.md index b77ed804751..032c2d17362 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) diff --git a/README_cn.md b/README_cn.md index 757f5f8fb7a..cf1b1e1c29a 100644 --- a/README_cn.md +++ b/README_cn.md @@ -59,6 +59,7 @@ - [x] Teambition（[中国](https://www.teambition.com/ )，[国际](https://us.teambition.com/ )） - [x] [MediaFire](https://www.mediafire.com) - [x] [分秒帧](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云，共享群组) - [x] [Yandex.Disk](https://disk.yandex.com/) - [x] [百度网盘](http://pan.baidu.com/) diff --git a/README_ja.md b/README_ja.md index e6a624b0929..a1a21253068 100644 --- a/README_ja.md +++ b/README_ja.md @@ -59,6 +59,7 @@ - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) + - [x] [ProtonDrive](https://proton.me/drive) - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) diff --git a/drivers/all.go b/drivers/all.go index 2ce0c2c6846..5c0f1ca04f4 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -54,6 +54,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/pcloud" _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" + _ "github.com/alist-org/alist/v3/drivers/proton_drive" _ "github.com/alist-org/alist/v3/drivers/quark_uc" _ "github.com/alist-org/alist/v3/drivers/quark_uc_tv" _ "github.com/alist-org/alist/v3/drivers/quqi" diff --git a/drivers/proton_drive/driver.go b/drivers/proton_drive/driver.go new file mode 100644 index 00000000000..6ff8b069cb3 --- /dev/null +++ b/drivers/proton_drive/driver.go @@ -0,0 +1,418 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "sync" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + proton_api_bridge "github.com/henrybear327/Proton-API-Bridge" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +type ProtonDrive struct { + model.Storage + Addition + + protonDrive *proton_api_bridge.ProtonDrive + credentials *common.ProtonDriveCredential + + apiBase string + appVersion string + protonJson string + userAgent string + sdkVersion string + webDriveAV string + + tempServer *http.Server + tempServerPort int + downloadTokens map[string]*downloadInfo + tokenMutex sync.RWMutex + + c *proton.Client + //m *proton.Manager + + credentialCacheFile string + + //userKR *crypto.KeyRing + addrKRs map[string]*crypto.KeyRing + addrData map[string]proton.Address + + MainShare *proton.Share + RootLink *proton.Link + + DefaultAddrKR *crypto.KeyRing + MainShareKR *crypto.KeyRing +} + +func (d *ProtonDrive) Config() driver.Config { + return config +} + +func (d *ProtonDrive) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ProtonDrive) Init(ctx context.Context) error { + + defer func() { + if r := recover(); r != nil { + fmt.Printf("ProtonDrive initialization panic: %v", r) + } + }() + + if d.Username == "" { + return fmt.Errorf("username is required") + } + if d.Password == "" { + return fmt.Errorf("password is required") + } + + //fmt.Printf("ProtonDrive Init: Username=%s, TwoFACode=%s", d.Username, d.TwoFACode) + + if ctx == nil { + return fmt.Errorf("context cannot be nil") + } + + cachedCredentials, err := d.loadCachedCredentials() + useReusableLogin := false + var reusableCredential *common.ReusableCredentialData + + if err == nil && cachedCredentials != nil && + cachedCredentials.UID != "" && cachedCredentials.AccessToken != "" && + cachedCredentials.RefreshToken != "" && cachedCredentials.SaltedKeyPass != "" { + useReusableLogin = true + reusableCredential = cachedCredentials + } else { + useReusableLogin = false + reusableCredential = &common.ReusableCredentialData{} + } + + config := &common.Config{ + AppVersion: d.appVersion, + UserAgent: d.userAgent, + FirstLoginCredential: &common.FirstLoginCredentialData{ + Username: d.Username, + Password: d.Password, + TwoFA: d.TwoFACode, + }, + EnableCaching: true, + ConcurrentBlockUploadCount: 5, + ConcurrentFileCryptoCount: 2, + UseReusableLogin: false, + ReplaceExistingDraft: true, + ReusableCredential: reusableCredential, + CredentialCacheFile: d.credentialCacheFile, + } + + if config.FirstLoginCredential == nil { + return fmt.Errorf("failed to create login credentials, FirstLoginCredential cannot be nil") + } + + //fmt.Printf("Calling NewProtonDrive...") + + protonDrive, credentials, err := proton_api_bridge.NewProtonDrive( + ctx, + config, + func(auth proton.Auth) {}, + func() {}, + ) + + if credentials == nil && !useReusableLogin { + return fmt.Errorf("failed to get credentials from NewProtonDrive") + } + + if err != nil { + return fmt.Errorf("failed to initialize ProtonDrive: %w", err) + } + + d.protonDrive = protonDrive + + var finalCredentials *common.ProtonDriveCredential + + if useReusableLogin { + + // For reusable login, create credentials from cached data + finalCredentials = &common.ProtonDriveCredential{ + UID: reusableCredential.UID, + AccessToken: reusableCredential.AccessToken, + RefreshToken: reusableCredential.RefreshToken, + SaltedKeyPass: reusableCredential.SaltedKeyPass, + } + + d.credentials = finalCredentials + } else { + d.credentials = credentials + } + + clientOptions := []proton.Option{ + proton.WithAppVersion(d.appVersion), + proton.WithUserAgent(d.userAgent), + } + manager := proton.New(clientOptions...) + d.c = manager.NewClient(d.credentials.UID, d.credentials.AccessToken, d.credentials.RefreshToken) + + saltedKeyPassBytes, err := base64.StdEncoding.DecodeString(d.credentials.SaltedKeyPass) + if err != nil { + return fmt.Errorf("failed to decode salted key pass: %w", err) + } + + _, addrKRs, addrs, _, err := getAccountKRs(ctx, d.c, nil, saltedKeyPassBytes) + if err != nil { + return fmt.Errorf("failed to get account keyrings: %w", err) + } + + d.MainShare = protonDrive.MainShare + d.RootLink = protonDrive.RootLink + d.MainShareKR = protonDrive.MainShareKR + d.DefaultAddrKR = protonDrive.DefaultAddrKR + d.addrKRs = addrKRs + d.addrData = addrs + + return nil +} + +func (d *ProtonDrive) Drop(ctx context.Context) error { + if d.tempServer != nil { + d.tempServer.Shutdown(ctx) + } + return nil +} + +func (d *ProtonDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var linkID string + + if dir.GetPath() == "/" { + linkID = d.protonDrive.RootLink.LinkID + } else { + + link, err := d.searchByPath(ctx, dir.GetPath(), true) + if err != nil { + return nil, err + } + linkID = link.LinkID + } + + entries, err := d.protonDrive.ListDirectory(ctx, linkID) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + } + + //fmt.Printf("Found %d entries for path %s\n", len(entries), dir.GetPath()) + //fmt.Printf("Found %d entries\n", len(entries)) + + if len(entries) == 0 { + emptySlice := []model.Obj{} + + //fmt.Printf("Returning empty slice (entries): %+v\n", emptySlice) + + return emptySlice, nil + } + + var objects []model.Obj + for _, entry := range entries { + obj := &model.Object{ + Name: entry.Name, + Size: entry.Link.Size, + Modified: time.Unix(entry.Link.ModifyTime, 0), + IsFolder: entry.IsFolder, + } + objects = append(objects, obj) + } + + return objects, nil +} + +func (d *ProtonDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + link, err := d.searchByPath(ctx, file.GetPath(), false) + if err != nil { + return nil, err + } + + if err := d.ensureTempServer(); err != nil { + return nil, fmt.Errorf("failed to start temp server: %w", err) + } + + token := d.generateDownloadToken(link.LinkID, file.GetName()) + + /* return &model.Link{ + URL: fmt.Sprintf("protondrive://download/%s", link.LinkID), + }, nil */ + + return &model.Link{ + URL: fmt.Sprintf("http://localhost:%d/temp/%s", d.tempServerPort, token), + }, nil +} + +func (d *ProtonDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var parentLinkID string + + if parentDir.GetPath() == "/" { + parentLinkID = d.protonDrive.RootLink.LinkID + } else { + link, err := d.searchByPath(ctx, parentDir.GetPath(), true) + if err != nil { + return nil, err + } + parentLinkID = link.LinkID + } + + _, err := d.protonDrive.CreateNewFolderByID(ctx, parentLinkID, dirName) + if err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + + newDir := &model.Object{ + Name: dirName, + IsFolder: true, + Modified: time.Now(), + } + return newDir, nil +} + +func (d *ProtonDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.DirectMove(ctx, srcObj, dstDir) +} + +func (d *ProtonDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + return d.DirectRename(ctx, srcObj, newName) +} + +func (d *ProtonDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, fmt.Errorf("directory copy not supported") + } + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), false) + if err != nil { + return nil, err + } + + reader, linkSize, fileSystemAttrs, err := d.protonDrive.DownloadFile(ctx, srcLink, 0) + if err != nil { + return nil, fmt.Errorf("failed to download source file: %w", err) + } + defer reader.Close() + + actualSize := linkSize + if fileSystemAttrs != nil && fileSystemAttrs.Size > 0 { + actualSize = fileSystemAttrs.Size + } + + tempFile, err := utils.CreateTempFile(reader, actualSize) + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer tempFile.Close() + + updatedObj := &model.Object{ + Name: srcObj.GetName(), + // Use the accurate and real size + Size: actualSize, + Modified: srcObj.ModTime(), + IsFolder: false, + } + + return d.Put(ctx, dstDir, &fileStreamer{ + ReadCloser: tempFile, + obj: updatedObj, + }, nil) +} + +func (d *ProtonDrive) Remove(ctx context.Context, obj model.Obj) error { + link, err := d.searchByPath(ctx, obj.GetPath(), obj.IsDir()) + if err != nil { + return err + } + + if obj.IsDir() { + return d.protonDrive.MoveFolderToTrashByID(ctx, link.LinkID, false) + } else { + return d.protonDrive.MoveFileToTrashByID(ctx, link.LinkID) + } +} + +func (d *ProtonDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var parentLinkID string + + if dstDir.GetPath() == "/" { + parentLinkID = d.protonDrive.RootLink.LinkID + } else { + link, err := d.searchByPath(ctx, dstDir.GetPath(), true) + if err != nil { + return nil, err + } + parentLinkID = link.LinkID + } + + tempFile, err := utils.CreateTempFile(file, file.GetSize()) + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer tempFile.Close() + + err = d.uploadFile(ctx, parentLinkID, file.GetName(), tempFile, file.GetSize(), up) + if err != nil { + return nil, err + } + + uploadedObj := &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + Modified: file.ModTime(), + IsFolder: false, + } + return uploadedObj, nil +} + +func (d *ProtonDrive) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +var _ driver.Driver = (*ProtonDrive)(nil) diff --git a/drivers/proton_drive/meta.go b/drivers/proton_drive/meta.go new file mode 100644 index 00000000000..ed33a41a2f8 --- /dev/null +++ b/drivers/proton_drive/meta.go @@ -0,0 +1,69 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + //driver.RootID + + Username string `json:"username" required:"true" type:"string"` + Password string `json:"password" required:"true" type:"string"` + TwoFACode string `json:"two_fa_code,omitempty" type:"string"` +} + +type Config struct { + Name string `json:"name"` + LocalSort bool `json:"local_sort"` + OnlyLocal bool `json:"only_local"` + OnlyProxy bool `json:"only_proxy"` + NoCache bool `json:"no_cache"` + NoUpload bool `json:"no_upload"` + NeedMs bool `json:"need_ms"` + DefaultRoot string `json:"default_root"` +} + +var config = driver.Config{ + Name: "ProtonDrive", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ProtonDrive{ + apiBase: "https://drive.proton.me/api", + appVersion: "windows-drive@1.11.3+rclone+proton", + credentialCacheFile: ".prtcrd", + protonJson: "application/vnd.protonmail.v1+json", + sdkVersion: "js@0.3.0", + userAgent: "ProtonDrive/v1.70.0 (Windows NT 10.0.22000; Win64; x64)", + webDriveAV: "web-drive@5.2.0+0f69f7a8", + } + }) +} diff --git a/drivers/proton_drive/types.go b/drivers/proton_drive/types.go new file mode 100644 index 00000000000..37a9edc6c46 --- /dev/null +++ b/drivers/proton_drive/types.go @@ -0,0 +1,124 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "errors" + "io" + "os" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/henrybear327/go-proton-api" +) + +type ProtonFile struct { + *proton.Link + Name string + IsFolder bool +} + +func (p *ProtonFile) GetName() string { + return p.Name +} + +func (p *ProtonFile) GetSize() int64 { + return p.Link.Size +} + +func (p *ProtonFile) GetPath() string { + return p.Name +} + +func (p *ProtonFile) IsDir() bool { + return p.IsFolder +} + +func (p *ProtonFile) ModTime() time.Time { + return time.Unix(p.Link.ModifyTime, 0) +} + +func (p *ProtonFile) CreateTime() time.Time { + return time.Unix(p.Link.CreateTime, 0) +} + +type downloadInfo struct { + LinkID string + FileName string +} + +type fileStreamer struct { + io.ReadCloser + obj model.Obj +} + +func (fs *fileStreamer) GetMimetype() string { return "" } +func (fs *fileStreamer) NeedStore() bool { return false } +func (fs *fileStreamer) IsForceStreamUpload() bool { return false } +func (fs *fileStreamer) GetExist() model.Obj { return nil } +func (fs *fileStreamer) SetExist(model.Obj) {} +func (fs *fileStreamer) RangeRead(http_range.Range) (io.Reader, error) { + return nil, errors.New("not supported") +} +func (fs *fileStreamer) CacheFullInTempFile() (model.File, error) { + return nil, errors.New("not supported") +} +func (fs *fileStreamer) SetTmpFile(r *os.File) {} +func (fs *fileStreamer) GetFile() model.File { return nil } +func (fs *fileStreamer) GetName() string { return fs.obj.GetName() } +func (fs *fileStreamer) GetSize() int64 { return fs.obj.GetSize() } +func (fs *fileStreamer) GetPath() string { return fs.obj.GetPath() } +func (fs *fileStreamer) IsDir() bool { return fs.obj.IsDir() } +func (fs *fileStreamer) ModTime() time.Time { return fs.obj.ModTime() } +func (fs *fileStreamer) CreateTime() time.Time { return fs.obj.ModTime() } +func (fs *fileStreamer) GetHash() utils.HashInfo { return fs.obj.GetHash() } +func (fs *fileStreamer) GetID() string { return fs.obj.GetID() } + +type httpRange struct { + start, end int64 +} + +type MoveRequest struct { + ParentLinkID string `json:"ParentLinkID"` + NodePassphrase string `json:"NodePassphrase"` + NodePassphraseSignature *string `json:"NodePassphraseSignature"` + Name string `json:"Name"` + NameSignatureEmail string `json:"NameSignatureEmail"` + Hash string `json:"Hash"` + OriginalHash string `json:"OriginalHash"` + ContentHash *string `json:"ContentHash"` // Maybe null +} + +type progressReader struct { + reader io.Reader + total int64 + current int64 + callback driver.UpdateProgress +} + +type RenameRequest struct { + Name string `json:"Name"` // PGP encrypted name + NameSignatureEmail string `json:"NameSignatureEmail"` // User's signature email + Hash string `json:"Hash"` // New name hash + OriginalHash string `json:"OriginalHash"` // Current name hash +} + +type RenameResponse struct { + Code int `json:"Code"` +} diff --git a/drivers/proton_drive/util.go b/drivers/proton_drive/util.go new file mode 100644 index 00000000000..7bd52fcaa33 --- /dev/null +++ b/drivers/proton_drive/util.go @@ -0,0 +1,918 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +func (d *ProtonDrive) loadCachedCredentials() (*common.ReusableCredentialData, error) { + if d.credentialCacheFile == "" { + return nil, nil + } + + if _, err := os.Stat(d.credentialCacheFile); os.IsNotExist(err) { + return nil, nil + } + + data, err := os.ReadFile(d.credentialCacheFile) + if err != nil { + return nil, fmt.Errorf("failed to read credential cache file: %w", err) + } + + var credentials common.ReusableCredentialData + if err := json.Unmarshal(data, &credentials); err != nil { + return nil, fmt.Errorf("failed to parse cached credentials: %w", err) + } + + if credentials.UID == "" || credentials.AccessToken == "" || + credentials.RefreshToken == "" || credentials.SaltedKeyPass == "" { + return nil, fmt.Errorf("cached credentials are incomplete") + } + + return &credentials, nil +} + +func (d *ProtonDrive) searchByPath(ctx context.Context, fullPath string, isFolder bool) (*proton.Link, error) { + if fullPath == "/" { + return d.protonDrive.RootLink, nil + } + + cleanPath := strings.Trim(fullPath, "/") + pathParts := strings.Split(cleanPath, "/") + + currentLink := d.protonDrive.RootLink + + for i, part := range pathParts { + isLastPart := i == len(pathParts)-1 + searchForFolder := !isLastPart || isFolder + + entries, err := d.protonDrive.ListDirectory(ctx, currentLink.LinkID) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + + } + + found := false + for _, entry := range entries { + // entry.Name is already decrypted! + if entry.Name == part && entry.IsFolder == searchForFolder { + currentLink = entry.Link + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("path not found: %s (looking for part: %s)", fullPath, part) + } + } + + return currentLink, nil +} + +func (pr *progressReader) Read(p []byte) (int, error) { + n, err := pr.reader.Read(p) + pr.current += int64(n) + + if pr.callback != nil { + percentage := float64(pr.current) / float64(pr.total) * 100 + pr.callback(percentage) + } + + return n, err +} + +func (d *ProtonDrive) uploadFile(ctx context.Context, parentLinkID, fileName string, file *os.File, size int64, up driver.UpdateProgress) error { + + fileInfo, err := file.Stat() + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + + _, err = d.protonDrive.GetLink(ctx, parentLinkID) + if err != nil { + return fmt.Errorf("failed to get parent link: %w", err) + } + + reader := &progressReader{ + reader: bufio.NewReader(file), + total: size, + current: 0, + callback: up, + } + + _, _, err = d.protonDrive.UploadFileByReader(ctx, parentLinkID, fileName, fileInfo.ModTime(), reader, 0) + if err != nil { + return fmt.Errorf("failed to upload file: %w", err) + } + + return nil +} + +func (d *ProtonDrive) ensureTempServer() error { + if d.tempServer != nil { + + // Already running + return nil + } + + listener, err := net.Listen("tcp", ":0") + if err != nil { + return err + } + d.tempServerPort = listener.Addr().(*net.TCPAddr).Port + + mux := http.NewServeMux() + mux.HandleFunc("/temp/", d.handleTempDownload) + + d.tempServer = &http.Server{ + Handler: mux, + } + + go func() { + d.tempServer.Serve(listener) + }() + + return nil +} + +func (d *ProtonDrive) handleTempDownload(w http.ResponseWriter, r *http.Request) { + token := strings.TrimPrefix(r.URL.Path, "/temp/") + + d.tokenMutex.RLock() + info, exists := d.downloadTokens[token] + d.tokenMutex.RUnlock() + + if !exists { + http.Error(w, "Invalid or expired token", http.StatusNotFound) + return + } + + link, err := d.protonDrive.GetLink(r.Context(), info.LinkID) + if err != nil { + http.Error(w, "Failed to get file link", http.StatusInternalServerError) + return + } + + // Get file size for range calculations + _, _, attrs, err := d.protonDrive.DownloadFile(r.Context(), link, 0) + if err != nil { + http.Error(w, "Failed to get file info", http.StatusInternalServerError) + return + } + + fileSize := attrs.Size + + rangeHeader := r.Header.Get("Range") + if rangeHeader != "" { + + // Parse range header like "bytes=0-1023" or "bytes=1024-" + ranges, err := parseRange(rangeHeader, fileSize) + if err != nil { + http.Error(w, "Invalid range", http.StatusRequestedRangeNotSatisfiable) + return + } + + if len(ranges) == 1 { + + // Single range request, small + start, end := ranges[0].start, ranges[0].end + contentLength := end - start + 1 + + // Start download from offset + reader, _, _, err := d.protonDrive.DownloadFile(r.Context(), link, start) + if err != nil { + http.Error(w, "Failed to start download", http.StatusInternalServerError) + return + } + defer reader.Close() + + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fileSize)) + w.Header().Set("Content-Length", fmt.Sprintf("%d", contentLength)) + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(link.Name))) + + // Partial content... + // Setting fileName is more cosmetical here + //.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", link.Name)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", info.FileName)) + w.Header().Set("Accept-Ranges", "bytes") + + w.WriteHeader(http.StatusPartialContent) + + io.CopyN(w, reader, contentLength) + return + } + } + + // Full file download (non-range request) + reader, _, _, err := d.protonDrive.DownloadFile(r.Context(), link, 0) + if err != nil { + http.Error(w, "Failed to start download", http.StatusInternalServerError) + return + } + defer reader.Close() + + // Set headers for full content + w.Header().Set("Content-Length", fmt.Sprintf("%d", fileSize)) + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(link.Name))) + + // Setting fileName is needed since ProtonDrive fileName is more like a random string + //w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", link.Name)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", info.FileName)) + + w.Header().Set("Accept-Ranges", "bytes") + + // Stream the full file + io.Copy(w, reader) +} + +func (d *ProtonDrive) generateDownloadToken(linkID, fileName string) string { + token := fmt.Sprintf("%d_%s", time.Now().UnixNano(), linkID[:8]) + + d.tokenMutex.Lock() + if d.downloadTokens == nil { + d.downloadTokens = make(map[string]*downloadInfo) + } + + d.downloadTokens[token] = &downloadInfo{ + LinkID: linkID, + FileName: fileName, + } + + d.tokenMutex.Unlock() + + go func() { + + // Token expires in 1 hour + time.Sleep(1 * time.Hour) + d.tokenMutex.Lock() + + delete(d.downloadTokens, token) + d.tokenMutex.Unlock() + }() + + return token +} + +func parseRange(rangeHeader string, size int64) ([]httpRange, error) { + if !strings.HasPrefix(rangeHeader, "bytes=") { + return nil, fmt.Errorf("invalid range header") + } + + rangeSpec := strings.TrimPrefix(rangeHeader, "bytes=") + ranges := strings.Split(rangeSpec, ",") + + var result []httpRange + for _, r := range ranges { + r = strings.TrimSpace(r) + if strings.Contains(r, "-") { + parts := strings.Split(r, "-") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid range format") + } + + var start, end int64 + var err error + + if parts[0] == "" { + + // Suffix range (e.g., "-500") + if parts[1] == "" { + return nil, fmt.Errorf("invalid range format") + } + end = size - 1 + start, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + start = size - start + if start < 0 { + start = 0 + } + } else if parts[1] == "" { + + // Prefix range (e.g., "500-") + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, err + } + end = size - 1 + } else { + // Full range (e.g., "0-1023") + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, err + } + end, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + } + + if start >= size || end >= size || start > end { + return nil, fmt.Errorf("range out of bounds") + } + + result = append(result, httpRange{start: start, end: end}) + } + } + + return result, nil +} + +func (d *ProtonDrive) encryptFileName(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Temporary file (request) + tempReq := proton.CreateFileReq{ + SignatureAddress: d.MainShare.Creator, + } + + // Encrypt the filename + err = tempReq.SetName(name, d.DefaultAddrKR, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to encrypt filename: %w", err) + } + + return tempReq.Name, nil +} + +func (d *ProtonDrive) generateFileNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func (d *ProtonDrive) getOriginalNameHash(link *proton.Link) (string, error) { + if link == nil { + return "", fmt.Errorf("link cannot be nil") + } + + if link.Hash == "" { + return "", fmt.Errorf("link hash is empty") + } + + return link.Hash, nil +} + +func (d *ProtonDrive) getLink(ctx context.Context, linkID string) (*proton.Link, error) { + if linkID == "" { + return nil, fmt.Errorf("linkID cannot be empty") + } + + link, err := d.c.GetLink(ctx, d.MainShare.ShareID, linkID) + if err != nil { + return nil, err + } + + return &link, nil +} + +func (d *ProtonDrive) getLinkKR(ctx context.Context, link *proton.Link) (*crypto.KeyRing, error) { + if link == nil { + return nil, fmt.Errorf("link cannot be nil") + } + + // Root Link or Root Dir + if link.ParentLinkID == "" { + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + return link.GetKeyRing(d.MainShareKR, signatureVerificationKR) + } + + // Get parent keyring recursively + parentLink, err := d.getLink(ctx, link.ParentLinkID) + if err != nil { + return nil, err + } + + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return nil, err + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + + return link.GetKeyRing(parentNodeKR, signatureVerificationKR) +} + +var ( + ErrKeyPassOrSaltedKeyPassMustBeNotNil = errors.New("either keyPass or saltedKeyPass must be not nil") + ErrFailedToUnlockUserKeys = errors.New("failed to unlock user keys") +) + +func getAccountKRs(ctx context.Context, c *proton.Client, keyPass, saltedKeyPass []byte) (*crypto.KeyRing, map[string]*crypto.KeyRing, map[string]proton.Address, []byte, error) { + + user, err := c.GetUser(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("user %#v", user) + + addrsArr, err := c.GetAddresses(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("addr %#v", addr) + + if saltedKeyPass == nil { + if keyPass == nil { + return nil, nil, nil, nil, ErrKeyPassOrSaltedKeyPassMustBeNotNil + } + + // Due to limitations, salts are stored using cacheCredentialToFile + salts, err := c.GetSalts(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("salts %#v", salts) + + saltedKeyPass, err = salts.SaltForKey(keyPass, user.Keys.Primary().ID) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("saltedKeyPass ok") + } + + userKR, addrKRs, err := proton.Unlock(user, addrsArr, saltedKeyPass, nil) + if err != nil { + return nil, nil, nil, nil, err + + } else if userKR.CountDecryptionEntities() == 0 { + return nil, nil, nil, nil, ErrFailedToUnlockUserKeys + } + + addrs := make(map[string]proton.Address) + for _, addr := range addrsArr { + addrs[addr.Email] = addr + } + + return userKR, addrKRs, addrs, saltedKeyPass, nil +} + +func (d *ProtonDrive) getSignatureVerificationKeyring(emailAddresses []string, verificationAddrKRs ...*crypto.KeyRing) (*crypto.KeyRing, error) { + ret, err := crypto.NewKeyRing(nil) + if err != nil { + return nil, err + } + + for _, emailAddress := range emailAddresses { + if addr, ok := d.addrData[emailAddress]; ok { + if addrKR, exists := d.addrKRs[addr.ID]; exists { + err = d.addKeysFromKR(ret, addrKR) + if err != nil { + return nil, err + } + } + } + } + + for _, kr := range verificationAddrKRs { + err = d.addKeysFromKR(ret, kr) + if err != nil { + return nil, err + } + } + + if ret.CountEntities() == 0 { + return nil, fmt.Errorf("no keyring for signature verification") + } + + return ret, nil +} + +func (d *ProtonDrive) addKeysFromKR(kr *crypto.KeyRing, newKRs ...*crypto.KeyRing) error { + for i := range newKRs { + for _, key := range newKRs[i].GetKeys() { + err := kr.AddKey(key) + if err != nil { + return err + } + } + } + return nil +} + +func (d *ProtonDrive) DirectRename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + //fmt.Printf("DEBUG DirectRename: path=%s, newName=%s", srcObj.GetPath(), newName) + + if d.MainShare == nil || d.DefaultAddrKR == nil { + return nil, fmt.Errorf("missing required fields: MainShare=%v, DefaultAddrKR=%v", + d.MainShare != nil, d.DefaultAddrKR != nil) + } + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), srcObj.IsDir()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + parentLinkID := srcLink.ParentLinkID + if parentLinkID == "" { + return nil, fmt.Errorf("cannot rename root folder") + } + + encryptedName, err := d.encryptFileName(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateFileNameHash(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + renameReq := RenameRequest{ + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + } + + err = d.executeRenameAPI(ctx, srcLink.LinkID, renameReq) + if err != nil { + return nil, fmt.Errorf("rename API call failed: %w", err) + } + + return &model.Object{ + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) executeRenameAPI(ctx context.Context, linkID string, req RenameRequest) error { + + renameURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/rename", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal rename request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", renameURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.credentials.UID) + httpReq.Header.Set("Authorization", "Bearer "+d.credentials.AccessToken) + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute rename request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("rename failed with status %d", resp.StatusCode) + } + + var renameResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&renameResp); err != nil { + return fmt.Errorf("failed to decode rename response: %w", err) + } + + if renameResp.Code != 1000 { + return fmt.Errorf("rename failed with code %d", renameResp.Code) + } + + return nil +} + +func (d *ProtonDrive) executeMoveAPI(ctx context.Context, linkID string, req MoveRequest) error { + //fmt.Printf("DEBUG Move Request - Name: %s\n", req.Name) + //fmt.Printf("DEBUG Move Request - Hash: %s\n", req.Hash) + //fmt.Printf("DEBUG Move Request - OriginalHash: %s\n", req.OriginalHash) + //fmt.Printf("DEBUG Move Request - ParentLinkID: %s\n", req.ParentLinkID) + + //fmt.Printf("DEBUG Move Request - Name length: %d\n", len(req.Name)) + //fmt.Printf("DEBUG Move Request - NameSignatureEmail: %s\n", req.NameSignatureEmail) + //fmt.Printf("DEBUG Move Request - ContentHash: %v\n", req.ContentHash) + //fmt.Printf("DEBUG Move Request - NodePassphrase length: %d\n", len(req.NodePassphrase)) + //fmt.Printf("DEBUG Move Request - NodePassphraseSignature length: %d\n", len(req.NodePassphraseSignature)) + + //fmt.Printf("DEBUG Move Request - SrcLinkID: %s\n", linkID) + //fmt.Printf("DEBUG Move Request - DstParentLinkID: %s\n", req.ParentLinkID) + //fmt.Printf("DEBUG Move Request - ShareID: %s\n", d.MainShare.ShareID) + + srcLink, _ := d.getLink(ctx, linkID) + if srcLink != nil && srcLink.ParentLinkID == req.ParentLinkID { + return fmt.Errorf("cannot move to same parent directory") + } + + moveURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/move", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal move request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", moveURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Authorization", "Bearer "+d.credentials.AccessToken) + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.credentials.UID) + httpReq.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute move request: %w", err) + } + defer resp.Body.Close() + + var moveResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&moveResp); err != nil { + return fmt.Errorf("failed to decode move response: %w", err) + } + + if moveResp.Code != 1000 { + return fmt.Errorf("move operation failed with code: %d", moveResp.Code) + } + + return nil +} + +func (d *ProtonDrive) DirectMove(ctx context.Context, srcObj model.Obj, dstDir model.Obj) (model.Obj, error) { + //fmt.Printf("DEBUG DirectMove: srcPath=%s, dstPath=%s", srcObj.GetPath(), dstDir.GetPath()) + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), srcObj.IsDir()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + var dstParentLinkID string + if dstDir.GetPath() == "/" { + dstParentLinkID = d.RootLink.LinkID + } else { + dstLink, err := d.searchByPath(ctx, dstDir.GetPath(), true) + if err != nil { + return nil, fmt.Errorf("failed to find destination: %w", err) + } + dstParentLinkID = dstLink.LinkID + } + + if srcObj.IsDir() { + + // Check if destination is a descendant of source + if err := d.checkCircularMove(ctx, srcLink.LinkID, dstParentLinkID); err != nil { + return nil, err + } + } + + // Encrypt the filename for the new location + encryptedName, err := d.encryptFileName(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateNameHash(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + // Re-encrypt node passphrase for new parent context + reencryptedPassphrase, err := d.reencryptNodePassphrase(ctx, srcLink, dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to re-encrypt node passphrase: %w", err) + } + + moveReq := MoveRequest{ + ParentLinkID: dstParentLinkID, + NodePassphrase: reencryptedPassphrase, + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + ContentHash: nil, + + // *** Causes rejection *** + /* NodePassphraseSignature: srcLink.NodePassphraseSignature, */ + } + + //fmt.Printf("DEBUG MoveRequest validation:\n") + //fmt.Printf(" Name length: %d\n", len(moveReq.Name)) + //fmt.Printf(" Hash: %s\n", moveReq.Hash) + //fmt.Printf(" OriginalHash: %s\n", moveReq.OriginalHash) + //fmt.Printf(" NodePassphrase length: %d\n", len(moveReq.NodePassphrase)) + /* fmt.Printf(" NodePassphraseSignature length: %d\n", len(moveReq.NodePassphraseSignature)) */ + //fmt.Printf(" NameSignatureEmail: %s\n", moveReq.NameSignatureEmail) + + err = d.executeMoveAPI(ctx, srcLink.LinkID, moveReq) + if err != nil { + return nil, fmt.Errorf("move API call failed: %w", err) + } + + return &model.Object{ + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) reencryptNodePassphrase(ctx context.Context, srcLink *proton.Link, dstParentLinkID string) (string, error) { + // Get source parent link with metadata + srcParentLink, err := d.getLink(ctx, srcLink.ParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get source parent link: %w", err) + } + + // Get source parent keyring using link object + srcParentKR, err := d.getLinkKR(ctx, srcParentLink) + if err != nil { + return "", fmt.Errorf("failed to get source parent keyring: %w", err) + } + + // Get destination parent link with metadata + dstParentLink, err := d.getLink(ctx, dstParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get destination parent link: %w", err) + } + + // Get destination parent keyring using link object + dstParentKR, err := d.getLinkKR(ctx, dstParentLink) + if err != nil { + return "", fmt.Errorf("failed to get destination parent keyring: %w", err) + } + + // Re-encrypt the node passphrase from source parent context to destination parent context + reencryptedPassphrase, err := reencryptKeyPacket(srcParentKR, dstParentKR, d.DefaultAddrKR, srcLink.NodePassphrase) + if err != nil { + return "", fmt.Errorf("failed to re-encrypt key packet: %w", err) + } + + return reencryptedPassphrase, nil +} + +func (d *ProtonDrive) generateNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Get signature verification keyring + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func reencryptKeyPacket(srcKR, dstKR, _ *crypto.KeyRing, passphrase string) (string, error) { // addrKR (3) + oldSplitMessage, err := crypto.NewPGPSplitMessageFromArmored(passphrase) + if err != nil { + return "", err + } + + sessionKey, err := srcKR.DecryptSessionKey(oldSplitMessage.KeyPacket) + if err != nil { + return "", err + } + + newKeyPacket, err := dstKR.EncryptSessionKey(sessionKey) + if err != nil { + return "", err + } + + newSplitMessage := crypto.NewPGPSplitMessage(newKeyPacket, oldSplitMessage.DataPacket) + + return newSplitMessage.GetArmored() +} + +func (d *ProtonDrive) checkCircularMove(ctx context.Context, srcLinkID, dstParentLinkID string) error { + currentLinkID := dstParentLinkID + + for currentLinkID != "" && currentLinkID != d.RootLink.LinkID { + if currentLinkID == srcLinkID { + return fmt.Errorf("cannot move folder into itself or its subfolder") + } + + currentLink, err := d.getLink(ctx, currentLinkID) + if err != nil { + return err + } + currentLinkID = currentLink.ParentLinkID + } + + return nil +} diff --git a/go.mod b/go.mod index 51f9beec443..5772806a7b4 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 + github.com/ProtonMail/gopenpgp/v2 v2.7.4 github.com/SheltonZhu/115driver v1.1.2 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 @@ -38,6 +39,8 @@ require ( github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hekmon/transmissionrpc/v3 v3.0.0 + github.com/henrybear327/Proton-API-Bridge v1.0.0 + github.com/henrybear327/go-proton-api v1.0.0 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 @@ -81,7 +84,21 @@ require ( gorm.io/gorm v1.25.11 ) -require github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect +require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect + github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect + github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect + github.com/ProtonMail/go-srp v0.0.7 // indirect + github.com/PuerkitoBio/goquery v1.8.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/bradenaw/juniper v0.15.2 // indirect + github.com/cronokirby/saferith v0.33.0 // indirect + github.com/emersion/go-message v0.18.0 // indirect + github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect + github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect + github.com/relvacode/iso8601 v1.3.0 // indirect +) require ( github.com/STARRY-S/zip v0.2.1 // indirect @@ -265,4 +282,8 @@ require ( lukechampine.com/blake3 v1.1.7 // indirect ) +replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 + +replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed + replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.1.2 diff --git a/go.sum b/go.sum index 1b088a7186e..4b21f881331 100644 --- a/go.sum +++ b/go.sum @@ -34,14 +34,33 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= +github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 h1:ikwCzeqoqN6wvBHOB9OI6dde/jbV7EoTMpUcxtYl5Po= github.com/KirCute/ftpserverlib-pasvportmap v1.25.0/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= github.com/KirCute/sftpd-alist v0.0.12 h1:GNVM5QLbQLAfXP4wGUlXFA2IO6fVek0n0IsGnOuISdg= github.com/KirCute/sftpd-alist v0.0.12/go.mod h1:2wNK7yyW2XfjyJq10OY6xB4COLac64hOwfV6clDJn6s= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= +github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= +github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= +github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= +github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= +github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= +github.com/ProtonMail/gopenpgp/v2 v2.7.4 h1:Vz/8+HViFFnf2A6XX8JOvZMrA6F5puwNvvF21O1mRlo= +github.com/ProtonMail/gopenpgp/v2 v2.7.4/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= @@ -67,6 +86,9 @@ github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAP github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -132,6 +154,9 @@ github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW4U= +github.com/bradenaw/juniper v0.15.2/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= @@ -162,6 +187,7 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= @@ -197,6 +223,12 @@ github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj6 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= +github.com/emersion/go-message v0.18.0 h1:7LxAXHRpSeoO/Wom3ZApVZYG7c3d17yCScYce8WiXA8= +github.com/emersion/go-message v0.18.0/go.mod h1:Zi69ACvzaoV/MBnrxfVBPV3xWEuCmC2nEN39oJF4B8A= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= @@ -337,6 +369,10 @@ github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= +github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0= +github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= +github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= +github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -529,6 +565,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= +github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= +github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -658,6 +696,8 @@ go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGX go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -735,6 +775,7 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -743,6 +784,7 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= @@ -791,6 +833,7 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -803,6 +846,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -819,6 +863,7 @@ golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= From 35d322443b1ba8b3fa9c2ed7421111808d86b6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 11 Oct 2025 04:14:13 -0700 Subject: [PATCH 570/659] feat(driver): Add URL signing support (#9347) Introduces the ability to sign generated URLs for enhanced security and access control. This feature is activated by configuring a `PrivateKey`, `UID`, and `ValidDuration` in the driver settings. If a private key is provided, the driver will sign the output URLs, making them time-limited based on the `ValidDuration`. The `ValidDuration` defaults to 30 minutes if not specified. The core signing logic is encapsulated in the new `sign.go` file. The `driver.go` file integrates this signing process before returning the final URL. --- drivers/123_open/driver.go | 19 ++++++++++++++++++- drivers/123_open/meta.go | 7 +++++-- drivers/123_open/sign.go | 27 +++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 drivers/123_open/sign.go diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go index 39ed146eb03..ace86bb981a 100644 --- a/drivers/123_open/driver.go +++ b/drivers/123_open/driver.go @@ -11,6 +11,7 @@ import ( "github.com/go-resty/resty/v2" "net/http" "strconv" + "time" ) type Open123 struct { @@ -89,8 +90,24 @@ func (d *Open123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return nil, fmt.Errorf("get link failed: %s", result.Message) } + linkURL := result.Data.URL + if d.PrivateKey != "" { + if d.UID == 0 { + return nil, fmt.Errorf("uid is required when private key is set") + } + duration := time.Duration(d.ValidDuration) + if duration <= 0 { + duration = 30 + } + signedURL, err := SignURL(linkURL, d.PrivateKey, d.UID, duration*time.Minute) + if err != nil { + return nil, err + } + linkURL = signedURL + } + return &model.Link{ - URL: result.Data.URL, + URL: linkURL, }, nil } diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go index d99bb75ba2c..d0b117aa7c0 100644 --- a/drivers/123_open/meta.go +++ b/drivers/123_open/meta.go @@ -8,8 +8,11 @@ import ( type Addition struct { driver.RootID - ClientID string `json:"client_id" required:"true" label:"Client ID"` - ClientSecret string `json:"client_secret" required:"true" label:"Client Secret"` + ClientID string `json:"client_id" required:"true" label:"Client ID"` + ClientSecret string `json:"client_secret" required:"true" label:"Client Secret"` + PrivateKey string `json:"private_key"` + UID uint64 `json:"uid" type:"number"` + ValidDuration int64 `json:"valid_duration" type:"number" default:"30" help:"minutes"` } var config = driver.Config{ diff --git a/drivers/123_open/sign.go b/drivers/123_open/sign.go new file mode 100644 index 00000000000..549d7ad8fdf --- /dev/null +++ b/drivers/123_open/sign.go @@ -0,0 +1,27 @@ +package _123Open + +import ( + "crypto/md5" + "fmt" + "math/rand" + "net/url" + "time" +) + +func SignURL(originURL, privateKey string, uid uint64, validDuration time.Duration) (string, error) { + if privateKey == "" { + return originURL, nil + } + parsed, err := url.Parse(originURL) + if err != nil { + return "", err + } + ts := time.Now().Add(validDuration).Unix() + randInt := rand.Int() + signature := fmt.Sprintf("%d-%d-%d-%x", ts, randInt, uid, md5.Sum([]byte(fmt.Sprintf("%s-%d-%d-%d-%s", + parsed.Path, ts, randInt, uid, privateKey)))) + query := parsed.Query() + query.Add("auth_key", signature) + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} From a6bd90a9b2c3b595990400ab061e4ab3d6720378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 16 Oct 2025 02:22:54 -0700 Subject: [PATCH 571/659] feat(driver/s3): Add OSS Archive Support (#9350) * feat(s3): Add support for S3 object storage classes Introduces a new 'storage_class' configuration option for S3 providers. Users can now specify the desired storage class (e.g., Standard, GLACIER, DEEP_ARCHIVE) for objects uploaded to S3-compatible services like AWS S3 and Tencent COS. The input storage class string is normalized to match AWS SDK constants, supporting various common aliases. If an unknown storage class is provided, it will be used as a raw value with a warning. This enhancement provides greater control over storage costs and data access patterns. * feat(storage): Support for displaying file storage classes Adds storage class information to file metadata and API responses. This change introduces the ability to store file storage classes in file metadata and display them in API responses. This allows users to view a file's storage tier (e.g., S3 Standard, Glacier), enhancing data management capabilities. Implementation details include: - Introducing the StorageClassProvider interface and the ObjWrapStorageClass structure to uniformly handle and communicate object storage class information. - Updated file metadata structures (e.g., ArchiveObj, FileInfo, RespFile) to include a StorageClass field. - Modified relevant API response functions (e.g., GetFileInfo, GetFileList) to populate and return storage classes. - Integrated functionality for retrieving object storage classes from underlying storage systems (e.g., S3) and wrapping them in lists. * feat(driver/s3): Added the "Other" interface and implemented it by the S3 driver. A new `driver.Other` interface has been added and defined in the `other.go` file. The S3 driver has been updated to implement this new interface, extending its functionality. * feat(s3): Add S3 object archive and thaw task management This commit introduces comprehensive support for S3 object archive and thaw operations, managed asynchronously through a new task system. - **S3 Transition Task System**: - Adds a new `S3Transition` task configuration, including workers, max retries, and persistence options. - Initializes `S3TransitionTaskManager` to handle asynchronous S3 archive/thaw requests. - Registers dedicated API routes for monitoring S3 transition tasks. - **Integrate S3 Archive/Thaw with Other API**: - Modifies the `Other` API handler to intercept `archive` and `thaw` methods for S3 storage drivers. - Dispatches these operations as `S3TransitionTask` instances to the task manager for background processing. - Returns a task ID to the client for tracking the status of the dispatched operation. - **Refactor `other` package for improved API consistency**: - Exports previously internal structs such as `archiveRequest`, `thawRequest`, `objectDescriptor`, `archiveResponse`, `thawResponse`, and `restoreStatus` by making their names public. - Makes helper functions like `decodeOtherArgs`, `normalizeStorageClass`, and `normalizeRestoreTier` public. - Introduces new constants for various S3 `Other` API methods. --- drivers/s3/driver.go | 36 +++- drivers/s3/meta.go | 1 + drivers/s3/other.go | 286 ++++++++++++++++++++++++++++++++ drivers/s3/util.go | 11 +- internal/bootstrap/task.go | 12 ++ internal/conf/config.go | 6 + internal/fs/other.go | 37 +++++ internal/fs/s3_transition.go | 310 +++++++++++++++++++++++++++++++++++ internal/model/obj.go | 25 +++ internal/model/object.go | 19 +++ server/handles/archive.go | 22 +-- server/handles/fsread.go | 106 ++++++------ server/handles/task.go | 1 + 13 files changed, 807 insertions(+), 65 deletions(-) create mode 100644 drivers/s3/other.go create mode 100644 internal/fs/s3_transition.go diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index b741148983e..7825ca6f935 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -15,6 +15,7 @@ import ( "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/server/common" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -32,6 +33,33 @@ type S3 struct { cron *cron.Cron } +var storageClassLookup = map[string]string{ + "standard": s3.ObjectStorageClassStandard, + "reduced_redundancy": s3.ObjectStorageClassReducedRedundancy, + "glacier": s3.ObjectStorageClassGlacier, + "standard_ia": s3.ObjectStorageClassStandardIa, + "onezone_ia": s3.ObjectStorageClassOnezoneIa, + "intelligent_tiering": s3.ObjectStorageClassIntelligentTiering, + "deep_archive": s3.ObjectStorageClassDeepArchive, + "outposts": s3.ObjectStorageClassOutposts, + "glacier_ir": s3.ObjectStorageClassGlacierIr, + "snow": s3.ObjectStorageClassSnow, + "express_onezone": s3.ObjectStorageClassExpressOnezone, +} + +func (d *S3) resolveStorageClass() *string { + value := strings.TrimSpace(d.StorageClass) + if value == "" { + return nil + } + normalized := strings.ToLower(strings.ReplaceAll(value, "-", "_")) + if v, ok := storageClassLookup[normalized]; ok { + return aws.String(v) + } + log.Warnf("s3: unknown storage class %q, using raw value", d.StorageClass) + return aws.String(value) +} + func (d *S3) Config() driver.Config { return d.config } @@ -179,8 +207,14 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up }), ContentType: &contentType, } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } _, err := uploader.UploadWithContext(ctx, input) return err } -var _ driver.Driver = (*S3)(nil) +var ( + _ driver.Driver = (*S3)(nil) + _ driver.Other = (*S3)(nil) +) diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 4de4b60a690..89d723b60b5 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -21,6 +21,7 @@ type Addition struct { ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` + StorageClass string `json:"storage_class" type:"select" options:",standard,standard_ia,onezone_ia,intelligent_tiering,glacier,glacier_ir,deep_archive,archive" help:"Storage class for new objects. AWS and Tencent COS support different subsets (COS uses ARCHIVE/DEEP_ARCHIVE)."` } func init() { diff --git a/drivers/s3/other.go b/drivers/s3/other.go new file mode 100644 index 00000000000..e299dae05d1 --- /dev/null +++ b/drivers/s3/other.go @@ -0,0 +1,286 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +const ( + OtherMethodArchive = "archive" + OtherMethodArchiveStatus = "archive_status" + OtherMethodThaw = "thaw" + OtherMethodThawStatus = "thaw_status" +) + +type ArchiveRequest struct { + StorageClass string `json:"storage_class"` +} + +type ThawRequest struct { + Days int64 `json:"days"` + Tier string `json:"tier"` +} + +type ObjectDescriptor struct { + Path string `json:"path"` + Bucket string `json:"bucket"` + Key string `json:"key"` +} + +type ArchiveResponse struct { + Action string `json:"action"` + Object ObjectDescriptor `json:"object"` + StorageClass string `json:"storage_class"` + RequestID string `json:"request_id,omitempty"` + VersionID string `json:"version_id,omitempty"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` +} + +type ThawResponse struct { + Action string `json:"action"` + Object ObjectDescriptor `json:"object"` + RequestID string `json:"request_id,omitempty"` + Status *RestoreStatus `json:"status,omitempty"` +} + +type RestoreStatus struct { + Ongoing bool `json:"ongoing"` + Expiry string `json:"expiry,omitempty"` + Raw string `json:"raw"` +} + +func (d *S3) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + if args.Obj == nil { + return nil, fmt.Errorf("missing object reference") + } + if args.Obj.IsDir() { + return nil, errs.NotSupport + } + + switch strings.ToLower(strings.TrimSpace(args.Method)) { + case "archive": + return d.archive(ctx, args) + case "archive_status": + return d.archiveStatus(ctx, args) + case "thaw": + return d.thaw(ctx, args) + case "thaw_status": + return d.thawStatus(ctx, args) + default: + return nil, errs.NotSupport + } +} + +func (d *S3) archive(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + payload := ArchiveRequest{} + if err := DecodeOtherArgs(args.Data, &payload); err != nil { + return nil, fmt.Errorf("parse archive request: %w", err) + } + if payload.StorageClass == "" { + return nil, fmt.Errorf("storage_class is required") + } + storageClass := NormalizeStorageClass(payload.StorageClass) + input := &s3.CopyObjectInput{ + Bucket: &d.Bucket, + Key: &key, + CopySource: aws.String(url.PathEscape(d.Bucket + "/" + key)), + MetadataDirective: aws.String(s3.MetadataDirectiveCopy), + StorageClass: aws.String(storageClass), + } + copyReq, output := d.client.CopyObjectRequest(input) + copyReq.SetContext(ctx) + if err := copyReq.Send(); err != nil { + return nil, err + } + + resp := ArchiveResponse{ + Action: "archive", + Object: d.describeObject(args.Obj, key), + StorageClass: storageClass, + RequestID: copyReq.RequestID, + } + if output.VersionId != nil { + resp.VersionID = aws.StringValue(output.VersionId) + } + if result := output.CopyObjectResult; result != nil { + resp.ETag = aws.StringValue(result.ETag) + if result.LastModified != nil { + resp.LastModified = result.LastModified.UTC().Format(time.RFC3339) + } + } + if status, err := d.describeObjectStatus(ctx, key); err == nil { + if status.StorageClass != "" { + resp.StorageClass = status.StorageClass + } + } + return resp, nil +} + +func (d *S3) archiveStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + status, err := d.describeObjectStatus(ctx, key) + if err != nil { + return nil, err + } + return ArchiveResponse{ + Action: "archive_status", + Object: d.describeObject(args.Obj, key), + StorageClass: status.StorageClass, + }, nil +} + +func (d *S3) thaw(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + payload := ThawRequest{Days: 1} + if err := DecodeOtherArgs(args.Data, &payload); err != nil { + return nil, fmt.Errorf("parse thaw request: %w", err) + } + if payload.Days <= 0 { + payload.Days = 1 + } + restoreRequest := &s3.RestoreRequest{ + Days: aws.Int64(payload.Days), + } + if tier := NormalizeRestoreTier(payload.Tier); tier != "" { + restoreRequest.GlacierJobParameters = &s3.GlacierJobParameters{Tier: aws.String(tier)} + } + input := &s3.RestoreObjectInput{ + Bucket: &d.Bucket, + Key: &key, + RestoreRequest: restoreRequest, + } + restoreReq, _ := d.client.RestoreObjectRequest(input) + restoreReq.SetContext(ctx) + if err := restoreReq.Send(); err != nil { + return nil, err + } + status, _ := d.describeObjectStatus(ctx, key) + resp := ThawResponse{ + Action: "thaw", + Object: d.describeObject(args.Obj, key), + RequestID: restoreReq.RequestID, + } + if status != nil { + resp.Status = status.Restore + } + return resp, nil +} + +func (d *S3) thawStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + status, err := d.describeObjectStatus(ctx, key) + if err != nil { + return nil, err + } + return ThawResponse{ + Action: "thaw_status", + Object: d.describeObject(args.Obj, key), + Status: status.Restore, + }, nil +} + +func (d *S3) describeObject(obj model.Obj, key string) ObjectDescriptor { + return ObjectDescriptor{ + Path: obj.GetPath(), + Bucket: d.Bucket, + Key: key, + } +} + +type objectStatus struct { + StorageClass string + Restore *RestoreStatus +} + +func (d *S3) describeObjectStatus(ctx context.Context, key string) (*objectStatus, error) { + head, err := d.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{Bucket: &d.Bucket, Key: &key}) + if err != nil { + return nil, err + } + status := &objectStatus{ + StorageClass: aws.StringValue(head.StorageClass), + Restore: parseRestoreHeader(head.Restore), + } + return status, nil +} + +func parseRestoreHeader(header *string) *RestoreStatus { + if header == nil { + return nil + } + value := strings.TrimSpace(*header) + if value == "" { + return nil + } + status := &RestoreStatus{Raw: value} + parts := strings.Split(value, ",") + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if strings.HasPrefix(part, "ongoing-request=") { + status.Ongoing = strings.Contains(part, "\"true\"") + } + if strings.HasPrefix(part, "expiry-date=") { + expiry := strings.Trim(part[len("expiry-date="):], "\"") + if expiry != "" { + if t, err := time.Parse(time.RFC1123, expiry); err == nil { + status.Expiry = t.UTC().Format(time.RFC3339) + } else { + status.Expiry = expiry + } + } + } + } + return status +} + +func DecodeOtherArgs(data interface{}, target interface{}) error { + if data == nil { + return nil + } + raw, err := json.Marshal(data) + if err != nil { + return err + } + return json.Unmarshal(raw, target) +} + +func NormalizeStorageClass(value string) string { + normalized := strings.ToLower(strings.TrimSpace(strings.ReplaceAll(value, "-", "_"))) + if normalized == "" { + return value + } + if v, ok := storageClassLookup[normalized]; ok { + return v + } + return value +} + +func NormalizeRestoreTier(value string) string { + normalized := strings.ToLower(strings.TrimSpace(value)) + switch normalized { + case "", "default": + return "" + case "bulk": + return s3.TierBulk + case "standard": + return s3.TierStandard + case "expedited": + return s3.TierExpedited + default: + return value + } +} diff --git a/drivers/s3/util.go b/drivers/s3/util.go index e02945a07d2..9d2b285ce1d 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -109,13 +109,13 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } - file := model.Object{ + file := &model.Object{ //Id: *object.Key, Name: name, Size: *object.Size, Modified: *object.LastModified, } - files = append(files, &file) + files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass))) } if listObjectsResult.IsTruncated == nil { return nil, errors.New("IsTruncated nil") @@ -164,13 +164,13 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } - file := model.Object{ + file := &model.Object{ //Id: *object.Key, Name: name, Size: *object.Size, Modified: *object.LastModified, } - files = append(files, &file) + files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass))) } if !aws.BoolValue(listObjectsResult.IsTruncated) { break @@ -202,6 +202,9 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { CopySource: aws.String(url.PathEscape(d.Bucket + "/" + srcKey)), Key: &dstKey, } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } _, err := d.client.CopyObject(input) return err } diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go index c67e3029b61..8f05b84a347 100644 --- a/internal/bootstrap/task.go +++ b/internal/bootstrap/task.go @@ -37,6 +37,18 @@ func InitTaskManager() { if len(tool.TransferTaskManager.GetAll()) == 0 { //prevent offline downloaded files from being deleted CleanTempDir() } + workers := conf.Conf.Tasks.S3Transition.Workers + if workers < 0 { + workers = 0 + } + fs.S3TransitionTaskManager = tache.NewManager[*fs.S3TransitionTask]( + tache.WithWorks(workers), + tache.WithPersistFunction( + db.GetTaskDataFunc("s3_transition", conf.Conf.Tasks.S3Transition.TaskPersistant), + db.UpdateTaskDataFunc("s3_transition", conf.Conf.Tasks.S3Transition.TaskPersistant), + ), + tache.WithMaxRetry(conf.Conf.Tasks.S3Transition.MaxRetry), + ) fs.ArchiveDownloadTaskManager = tache.NewManager[*fs.ArchiveDownloadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant), db.UpdateTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Decompress.MaxRetry)) op.RegisterSettingChangingCallback(func() { fs.ArchiveDownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers))) diff --git a/internal/conf/config.go b/internal/conf/config.go index cdb86fee3ad..cf7cde0bcf0 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -60,6 +60,7 @@ type TasksConfig struct { Copy TaskConfig `json:"copy" envPrefix:"COPY_"` Decompress TaskConfig `json:"decompress" envPrefix:"DECOMPRESS_"` DecompressUpload TaskConfig `json:"decompress_upload" envPrefix:"DECOMPRESS_UPLOAD_"` + S3Transition TaskConfig `json:"s3_transition" envPrefix:"S3_TRANSITION_"` AllowRetryCanceled bool `json:"allow_retry_canceled" env:"ALLOW_RETRY_CANCELED"` } @@ -184,6 +185,11 @@ func DefaultConfig() *Config { Workers: 5, MaxRetry: 2, }, + S3Transition: TaskConfig{ + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, + }, AllowRetryCanceled: false, }, Cors: Cors{ diff --git a/internal/fs/other.go b/internal/fs/other.go index 85b7b1d17bf..14f8f63d3ad 100644 --- a/internal/fs/other.go +++ b/internal/fs/other.go @@ -2,10 +2,15 @@ package fs import ( "context" + "encoding/json" + stdpath "path" + "strings" + "github.com/alist-org/alist/v3/drivers/s3" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" "github.com/pkg/errors" ) @@ -53,6 +58,38 @@ func other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { if err != nil { return nil, errors.WithMessage(err, "failed get storage") } + originalPath := args.Path + + if _, ok := storage.(*s3.S3); ok { + method := strings.ToLower(strings.TrimSpace(args.Method)) + if method == s3.OtherMethodArchive || method == s3.OtherMethodThaw { + if S3TransitionTaskManager == nil { + return nil, errors.New("s3 transition task manager is not initialized") + } + var payload json.RawMessage + if args.Data != nil { + raw, err := json.Marshal(args.Data) + if err != nil { + return nil, errors.WithMessage(err, "failed to encode request payload") + } + payload = raw + } + taskCreator, _ := ctx.Value("user").(*model.User) + tsk := &S3TransitionTask{ + TaskExtension: task.TaskExtension{Creator: taskCreator}, + status: "queued", + StorageMountPath: storage.GetStorage().MountPath, + ObjectPath: actualPath, + DisplayPath: originalPath, + ObjectName: stdpath.Base(actualPath), + Transition: method, + Payload: payload, + } + S3TransitionTaskManager.Add(tsk) + return map[string]string{"task_id": tsk.GetID()}, nil + } + } + args.Path = actualPath return op.Other(ctx, storage, args) } diff --git a/internal/fs/s3_transition.go b/internal/fs/s3_transition.go new file mode 100644 index 00000000000..395f3c5be8c --- /dev/null +++ b/internal/fs/s3_transition.go @@ -0,0 +1,310 @@ +package fs + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/s3" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" + "github.com/xhofe/tache" +) + +const s3TransitionPollInterval = 15 * time.Second + +// S3TransitionTask represents an asynchronous S3 archive/thaw request that is +// tracked via the task manager so that clients can monitor the progress of the +// operation. +type S3TransitionTask struct { + task.TaskExtension + status string + + StorageMountPath string `json:"storage_mount_path"` + ObjectPath string `json:"object_path"` + DisplayPath string `json:"display_path"` + ObjectName string `json:"object_name"` + Transition string `json:"transition"` + Payload json.RawMessage `json:"payload,omitempty"` + + TargetStorageClass string `json:"target_storage_class,omitempty"` + RequestID string `json:"request_id,omitempty"` + VersionID string `json:"version_id,omitempty"` + + storage driver.Driver `json:"-"` +} + +// S3TransitionTaskManager holds asynchronous S3 archive/thaw tasks. +var S3TransitionTaskManager *tache.Manager[*S3TransitionTask] + +var _ task.TaskExtensionInfo = (*S3TransitionTask)(nil) + +func (t *S3TransitionTask) GetName() string { + action := strings.ToLower(t.Transition) + if action == "" { + action = "transition" + } + display := t.DisplayPath + if display == "" { + display = t.ObjectPath + } + if display == "" { + display = t.ObjectName + } + return fmt.Sprintf("s3 %s %s", action, display) +} + +func (t *S3TransitionTask) GetStatus() string { + return t.status +} + +func (t *S3TransitionTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + start := time.Now() + t.SetStartTime(start) + defer func() { t.SetEndTime(time.Now()) }() + + if err := t.ensureStorage(); err != nil { + t.status = fmt.Sprintf("locate storage failed: %v", err) + return err + } + + payload, err := t.decodePayload() + if err != nil { + t.status = fmt.Sprintf("decode payload failed: %v", err) + return err + } + + method := strings.ToLower(strings.TrimSpace(t.Transition)) + switch method { + case s3.OtherMethodArchive: + t.status = "submitting archive request" + t.SetProgress(0) + resp, err := op.Other(t.Ctx(), t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodArchive, + Data: payload, + }) + if err != nil { + t.status = fmt.Sprintf("archive request failed: %v", err) + return err + } + archiveResp, ok := toArchiveResponse(resp) + if ok { + if t.TargetStorageClass == "" { + t.TargetStorageClass = archiveResp.StorageClass + } + t.RequestID = archiveResp.RequestID + t.VersionID = archiveResp.VersionID + if archiveResp.StorageClass != "" { + t.status = fmt.Sprintf("archive requested, waiting for %s", archiveResp.StorageClass) + } else { + t.status = "archive requested" + } + } else if sc := t.extractTargetStorageClass(); sc != "" { + t.TargetStorageClass = sc + t.status = fmt.Sprintf("archive requested, waiting for %s", sc) + } else { + t.status = "archive requested" + } + if t.TargetStorageClass != "" { + t.TargetStorageClass = s3.NormalizeStorageClass(t.TargetStorageClass) + } + t.SetProgress(25) + return t.waitForArchive() + case s3.OtherMethodThaw: + t.status = "submitting thaw request" + t.SetProgress(0) + resp, err := op.Other(t.Ctx(), t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodThaw, + Data: payload, + }) + if err != nil { + t.status = fmt.Sprintf("thaw request failed: %v", err) + return err + } + thawResp, ok := toThawResponse(resp) + if ok { + t.RequestID = thawResp.RequestID + if thawResp.Status != nil && !thawResp.Status.Ongoing { + t.SetProgress(100) + t.status = thawCompletionMessage(thawResp.Status) + return nil + } + } + t.status = "thaw requested" + t.SetProgress(25) + return t.waitForThaw() + default: + return errors.Errorf("unsupported transition method: %s", t.Transition) + } +} + +func (t *S3TransitionTask) ensureStorage() error { + if t.storage != nil { + return nil + } + storage, err := op.GetStorageByMountPath(t.StorageMountPath) + if err != nil { + return err + } + t.storage = storage + return nil +} + +func (t *S3TransitionTask) decodePayload() (interface{}, error) { + if len(t.Payload) == 0 { + return nil, nil + } + var payload interface{} + if err := json.Unmarshal(t.Payload, &payload); err != nil { + return nil, err + } + return payload, nil +} + +func (t *S3TransitionTask) extractTargetStorageClass() string { + if len(t.Payload) == 0 { + return "" + } + var req s3.ArchiveRequest + if err := json.Unmarshal(t.Payload, &req); err != nil { + return "" + } + return s3.NormalizeStorageClass(req.StorageClass) +} + +func (t *S3TransitionTask) waitForArchive() error { + ticker := time.NewTicker(s3TransitionPollInterval) + defer ticker.Stop() + + ctx := t.Ctx() + for { + select { + case <-ctx.Done(): + t.status = "archive canceled" + return ctx.Err() + case <-ticker.C: + resp, err := op.Other(ctx, t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodArchiveStatus, + }) + if err != nil { + t.status = fmt.Sprintf("archive status error: %v", err) + return err + } + archiveResp, ok := toArchiveResponse(resp) + if !ok { + t.status = fmt.Sprintf("unexpected archive status response: %T", resp) + return errors.Errorf("unexpected archive status response: %T", resp) + } + currentClass := strings.TrimSpace(archiveResp.StorageClass) + target := strings.TrimSpace(t.TargetStorageClass) + if target == "" { + target = currentClass + t.TargetStorageClass = currentClass + } + if currentClass == "" { + t.status = "waiting for storage class update" + t.SetProgress(50) + continue + } + if strings.EqualFold(currentClass, target) { + t.SetProgress(100) + t.status = fmt.Sprintf("archive complete (%s)", currentClass) + return nil + } + t.status = fmt.Sprintf("storage class %s (target %s)", currentClass, target) + t.SetProgress(75) + } + } +} + +func (t *S3TransitionTask) waitForThaw() error { + ticker := time.NewTicker(s3TransitionPollInterval) + defer ticker.Stop() + + ctx := t.Ctx() + for { + select { + case <-ctx.Done(): + t.status = "thaw canceled" + return ctx.Err() + case <-ticker.C: + resp, err := op.Other(ctx, t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodThawStatus, + }) + if err != nil { + t.status = fmt.Sprintf("thaw status error: %v", err) + return err + } + thawResp, ok := toThawResponse(resp) + if !ok { + t.status = fmt.Sprintf("unexpected thaw status response: %T", resp) + return errors.Errorf("unexpected thaw status response: %T", resp) + } + status := thawResp.Status + if status == nil { + t.status = "waiting for thaw status" + t.SetProgress(50) + continue + } + if status.Ongoing { + t.status = fmt.Sprintf("thaw in progress (%s)", status.Raw) + t.SetProgress(75) + continue + } + t.SetProgress(100) + t.status = thawCompletionMessage(status) + return nil + } + } +} + +func thawCompletionMessage(status *s3.RestoreStatus) string { + if status == nil { + return "thaw complete" + } + if status.Expiry != "" { + return fmt.Sprintf("thaw complete, expires %s", status.Expiry) + } + return "thaw complete" +} + +func toArchiveResponse(v interface{}) (s3.ArchiveResponse, bool) { + switch resp := v.(type) { + case s3.ArchiveResponse: + return resp, true + case *s3.ArchiveResponse: + if resp != nil { + return *resp, true + } + } + return s3.ArchiveResponse{}, false +} + +func toThawResponse(v interface{}) (s3.ThawResponse, bool) { + switch resp := v.(type) { + case s3.ThawResponse: + return resp, true + case *s3.ThawResponse: + if resp != nil { + return *resp, true + } + } + return s3.ThawResponse{}, false +} + +// Ensure compatibility with persistence when tasks are restored. +func (t *S3TransitionTask) OnRestore() { + // The storage handle is not persisted intentionally; it will be lazily + // re-fetched on the next Run invocation. + t.storage = nil +} diff --git a/internal/model/obj.go b/internal/model/obj.go index 93fa7a96475..ed4e0451ec7 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -20,6 +20,10 @@ type ObjUnwrap interface { Unwrap() Obj } +type StorageClassProvider interface { + StorageClass() string +} + type Obj interface { GetSize() int64 GetName() string @@ -141,6 +145,13 @@ func WrapObjsName(objs []Obj) { } } +func WrapObjStorageClass(obj Obj, storageClass string) Obj { + if storageClass == "" { + return obj + } + return &ObjWrapStorageClass{Obj: obj, storageClass: storageClass} +} + func UnwrapObj(obj Obj) Obj { if unwrap, ok := obj.(ObjUnwrap); ok { obj = unwrap.Unwrap() @@ -168,6 +179,20 @@ func GetUrl(obj Obj) (url string, ok bool) { return url, false } +func GetStorageClass(obj Obj) (string, bool) { + if provider, ok := obj.(StorageClassProvider); ok { + value := provider.StorageClass() + if value == "" { + return "", false + } + return value, true + } + if unwrap, ok := obj.(ObjUnwrap); ok { + return GetStorageClass(unwrap.Unwrap()) + } + return "", false +} + func GetRawObject(obj Obj) *Object { switch v := obj.(type) { case *ObjThumbURL: diff --git a/internal/model/object.go b/internal/model/object.go index c8c10bb9d92..1617662cf9b 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -11,6 +11,11 @@ type ObjWrapName struct { Obj } +type ObjWrapStorageClass struct { + storageClass string + Obj +} + func (o *ObjWrapName) Unwrap() Obj { return o.Obj } @@ -19,6 +24,20 @@ func (o *ObjWrapName) GetName() string { return o.Name } +func (o *ObjWrapStorageClass) Unwrap() Obj { + return o.Obj +} + +func (o *ObjWrapStorageClass) StorageClass() string { + return o.storageClass +} + +func (o *ObjWrapStorageClass) SetPath(path string) { + if setter, ok := o.Obj.(SetPath); ok { + setter.SetPath(path) + } +} + type Object struct { ID string Path string diff --git a/server/handles/archive.go b/server/handles/archive.go index 0bb8d94a728..5787897cc90 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -44,17 +44,19 @@ type ArchiveContentResp struct { } func toObjsRespWithoutSignAndThumb(obj model.Obj) ObjResp { + storageClass, _ := model.GetStorageClass(obj) return ObjResp{ - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: "", - Thumb: "", - Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: "", + Thumb: "", + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + StorageClass: storageClass, } } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 8cf3c9b0290..676d64b166e 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -33,18 +33,19 @@ type DirReq struct { } type ObjResp struct { - Id string `json:"id"` - Path string `json:"path"` - Name string `json:"name"` - Size int64 `json:"size"` - IsDir bool `json:"is_dir"` - Modified time.Time `json:"modified"` - Created time.Time `json:"created"` - Sign string `json:"sign"` - Thumb string `json:"thumb"` - Type int `json:"type"` - HashInfoStr string `json:"hashinfo"` - HashInfo map[*utils.HashType]string `json:"hash_info"` + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + StorageClass string `json:"storage_class,omitempty"` } type FsListResp struct { @@ -57,19 +58,20 @@ type FsListResp struct { } type ObjLabelResp struct { - Id string `json:"id"` - Path string `json:"path"` - Name string `json:"name"` - Size int64 `json:"size"` - IsDir bool `json:"is_dir"` - Modified time.Time `json:"modified"` - Created time.Time `json:"created"` - Sign string `json:"sign"` - Thumb string `json:"thumb"` - Type int `json:"type"` - HashInfoStr string `json:"hashinfo"` - HashInfo map[*utils.HashType]string `json:"hash_info"` - LabelList []model.Label `json:"label_list"` + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + LabelList []model.Label `json:"label_list"` + StorageClass string `json:"storage_class,omitempty"` } func FsList(c *gin.Context) { @@ -256,20 +258,22 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjLabelResp { labels = labelsByName[obj.GetName()] } thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) resp = append(resp, ObjLabelResp{ - Id: obj.GetID(), - Path: obj.GetPath(), - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: common.Sign(obj, parent, encrypt), - Thumb: thumb, - Type: utils.GetObjType(obj.GetName(), obj.IsDir()), - LabelList: labels, + Id: obj.GetID(), + Path: obj.GetPath(), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: common.Sign(obj, parent, encrypt), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + LabelList: labels, + StorageClass: storageClass, }) } return resp @@ -374,20 +378,22 @@ func FsGet(c *gin.Context) { } parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ - Id: obj.GetID(), - Path: obj.GetPath(), - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), - Type: utils.GetFileType(obj.GetName()), - Thumb: thumb, + Id: obj.GetID(), + Path: obj.GetPath(), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), + Type: utils.GetFileType(obj.GetName()), + Thumb: thumb, + StorageClass: storageClass, }, RawURL: rawURL, Readme: getReadme(meta, reqPath), diff --git a/server/handles/task.go b/server/handles/task.go index 6d49f9e5027..a4dbce0f1fa 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -220,6 +220,7 @@ func SetupTaskRoute(g *gin.RouterGroup) { taskRoute(g.Group("/copy"), fs.CopyTaskManager) taskRoute(g.Group("/offline_download"), tool.DownloadTaskManager) taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager) + taskRoute(g.Group("/s3_transition"), fs.S3TransitionTaskManager) taskRoute(g.Group("/decompress"), fs.ArchiveDownloadTaskManager) taskRoute(g.Group("/decompress_upload"), fs.ArchiveContentUploadTaskManager) } From e2016dd03174366442727f30cb62029fb330b9a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 16 Oct 2025 02:23:11 -0700 Subject: [PATCH 572/659] refactor(webdav): Use ResolvePath instead of JoinPath (#9344) - Changed the path concatenation method between `reqPath` and `src` and `dst` to use `ResolvePath` - Updated the implementation of path handling in multiple functions - Improved the consistency and reliability of path resolution --- server/webdav.go | 2 +- server/webdav/path.go | 22 ++++++++++++++++++++++ server/webdav/webdav.go | 20 ++++++++++---------- 3 files changed, 33 insertions(+), 11 deletions(-) create mode 100644 server/webdav/path.go diff --git a/server/webdav.go b/server/webdav.go index e0980139e4f..ac520070686 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -113,7 +113,7 @@ func WebDAVAuth(c *gin.Context) { reqPath = "/" } reqPath, _ = url.PathUnescape(reqPath) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = webdav.ResolvePath(user, reqPath) if err != nil { c.Status(http.StatusForbidden) c.Abort() diff --git a/server/webdav/path.go b/server/webdav/path.go new file mode 100644 index 00000000000..9a18da9551f --- /dev/null +++ b/server/webdav/path.go @@ -0,0 +1,22 @@ +package webdav + +import ( + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// ResolvePath normalizes the provided raw path and resolves it against the user's base path +// before delegating to the user-aware JoinPath permission checks. +func ResolvePath(user *model.User, raw string) (string, error) { + cleaned := utils.FixAndCleanPath(raw) + basePath := utils.FixAndCleanPath(user.BasePath) + + if cleaned != "/" && basePath != "/" && !utils.IsSubPath(basePath, cleaned) { + cleaned = path.Join(basePath, strings.TrimPrefix(cleaned, "/")) + } + + return user.JoinPath(cleaned) +} diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 93211e8a77e..00c0471f743 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -194,7 +194,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status } ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -222,7 +222,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta // TODO: check locks for read-only access?? ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return http.StatusForbidden, err } @@ -282,7 +282,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -321,7 +321,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, // comments in http.checkEtag. ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return http.StatusForbidden, err } @@ -375,7 +375,7 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -439,11 +439,11 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status ctx := r.Context() user := ctx.Value("user").(*model.User) - src, err = user.JoinPath(src) + src, err = ResolvePath(user, src) if err != nil { return 403, err } - dst, err = user.JoinPath(dst) + dst, err = ResolvePath(user, dst) if err != nil { return 403, err } @@ -540,7 +540,7 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus if err != nil { return status, err } - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -623,7 +623,7 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status userAgent := r.Header.Get("User-Agent") ctx = context.WithValue(ctx, "userAgent", userAgent) user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -801,7 +801,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } From 4c8401855c777e2a4f869b11a8fffbe400320323 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 23 Oct 2025 09:29:33 -0700 Subject: [PATCH 573/659] feat: Add new driver bitqiu support (#9355) * feat(bitqiu): Add Bitqiu cloud drive support - Implement the new Bitqiu cloud drive. - Add core driver logic, metadata handling, and utility functions. - Register the Bitqiu driver for use. * feat(driver): Implement GetLink, CreateDir, and Move operations - Implement `GetLink` method to retrieve download links for files. - Implement `CreateDir` method to create new directories. - Implement `Move` method to relocate files and directories. - Add new API endpoints and data structures for download and directory creation responses. - Integrate retry logic with re-authentication for API calls in implemented methods. - Update HTTP request headers to include `x-requested-with`. * feat(bitqiu): Add rename, copy, and delete operations - Implement `Rename` operation with retry logic and API calls. - Implement `Copy` operation, including asynchronous handling, polling for completion, and status checks. - Implement `Remove` operation with retry logic and API calls. - Add new API endpoint URLs for rename, copy, and delete, and a new copy success code. - Introduce `AsyncManagerData`, `AsyncTask`, and `AsyncTaskInfo` types to support async copy status monitoring. - Add utility functions `updateObjectName` and `parentPathOf` for object manipulation. - Integrate login retry mechanism for all file operations. * feat(bitqiu-upload): Implement chunked file upload support - Implement multi-part chunked upload logic for the BitQiu service. - Introduce `UploadInitData` and `ChunkUploadResponse` structs for structured API communication. - Refactor the `Save` method to orchestrate initial upload, chunked data transfer, and finalization. - Add `uploadFileInChunks` function to handle sequential uploading of file parts. - Add `completeChunkUpload` function to finalize the chunked upload process on the server. - Ensure proper temporary file cleanup using `defer tmpFile.Close()`. * feat(driver): Implement automatic root folder ID retrieval - Add `userInfoURL` constant for fetching user information. - Implement `ensureRootFolderID` function to retrieve and set the driver's root folder ID if not already present. - Integrate `ensureRootFolderID` into the driver's `Init` process. - Define `UserInfoData` struct to parse the `rootDirId` from user information responses. * feat(client): Implement configurable user agent * Introduce a configurable `UserAgent` field in the client's settings. * Add a `userAgent()` method to retrieve the user agent, prioritizing the custom setting or using a predefined default. * Apply the determined user agent to all outbound HTTP requests made by the `BitQiu` client. --- drivers/all.go | 1 + drivers/bitqiu/driver.go | 767 +++++++++++++++++++++++++++++++++++++++ drivers/bitqiu/meta.go | 28 ++ drivers/bitqiu/types.go | 107 ++++++ drivers/bitqiu/util.go | 102 ++++++ 5 files changed, 1005 insertions(+) create mode 100644 drivers/bitqiu/driver.go create mode 100644 drivers/bitqiu/meta.go create mode 100644 drivers/bitqiu/types.go create mode 100644 drivers/bitqiu/util.go diff --git a/drivers/all.go b/drivers/all.go index 5c0f1ca04f4..efeb6f7716a 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -21,6 +21,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_netdisk" _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" + _ "github.com/alist-org/alist/v3/drivers/bitqiu" _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" diff --git a/drivers/bitqiu/driver.go b/drivers/bitqiu/driver.go new file mode 100644 index 00000000000..048377fee93 --- /dev/null +++ b/drivers/bitqiu/driver.go @@ -0,0 +1,767 @@ +package bitqiu + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http/cookiejar" + "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + streamPkg "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" +) + +const ( + baseURL = "https://pan.bitqiu.com" + loginURL = baseURL + "/loginServer/login" + userInfoURL = baseURL + "/user/getInfo" + listURL = baseURL + "/apiToken/cfi/fs/resources/pages" + uploadInitializeURL = baseURL + "/apiToken/cfi/fs/upload/v2/initialize" + uploadCompleteURL = baseURL + "/apiToken/cfi/fs/upload/v2/complete" + downloadURL = baseURL + "/download/getUrl" + createDirURL = baseURL + "/resource/create" + moveResourceURL = baseURL + "/resource/remove" + renameResourceURL = baseURL + "/resource/rename" + copyResourceURL = baseURL + "/apiToken/cfi/fs/async/copy" + copyManagerURL = baseURL + "/apiToken/cfi/fs/async/manager" + deleteResourceURL = baseURL + "/resource/delete" + + successCode = "10200" + uploadSuccessCode = "30010" + copySubmittedCode = "10300" + orgChannel = "default|default|default" +) + +const ( + copyPollInterval = time.Second + copyPollMaxAttempts = 60 + chunkSize = int64(1 << 20) +) + +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36" + +type BitQiu struct { + model.Storage + Addition + + client *resty.Client + userID string +} + +func (d *BitQiu) Config() driver.Config { + return config +} + +func (d *BitQiu) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BitQiu) Init(ctx context.Context) error { + if d.Addition.UserPlatform == "" { + d.Addition.UserPlatform = uuid.NewString() + op.MustSaveDriverStorage(d) + } + + if d.client == nil { + jar, err := cookiejar.New(nil) + if err != nil { + return err + } + d.client = base.NewRestyClient() + d.client.SetBaseURL(baseURL) + d.client.SetCookieJar(jar) + } + d.client.SetHeader("user-agent", d.userAgent()) + + return d.login(ctx) +} + +func (d *BitQiu) Drop(ctx context.Context) error { + d.client = nil + d.userID = "" + return nil +} + +func (d *BitQiu) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + parentID := d.resolveParentID(dir) + dirPath := "" + if dir != nil { + dirPath = dir.GetPath() + } + pageSize := d.pageSize() + orderType := d.orderType() + desc := d.orderDesc() + + var results []model.Obj + page := 1 + for { + form := map[string]string{ + "parentId": parentID, + "limit": strconv.Itoa(pageSize), + "orderType": orderType, + "desc": desc, + "model": "1", + "userId": d.userID, + "currentPage": strconv.Itoa(page), + "page": strconv.Itoa(page), + "org_channel": orgChannel, + } + var resp Response[ResourcePage] + if err := d.postForm(ctx, listURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != successCode { + if resp.Code == "10401" || resp.Code == "10404" { + if err := d.login(ctx); err != nil { + return nil, err + } + continue + } + return nil, fmt.Errorf("list failed: %s", resp.Message) + } + + objs, err := utils.SliceConvert(resp.Data.Data, func(item Resource) (model.Obj, error) { + return item.toObject(parentID, dirPath) + }) + if err != nil { + return nil, err + } + results = append(results, objs...) + + if !resp.Data.HasNext || len(resp.Data.Data) == 0 { + break + } + page++ + } + + return results, nil +} + +func (d *BitQiu) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + form := map[string]string{ + "fileIds": file.GetID(), + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[DownloadData] + if err := d.postForm(ctx, downloadURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + if resp.Data.URL == "" { + return nil, fmt.Errorf("empty download url returned") + } + return &model.Link{URL: resp.Data.URL}, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("get link failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("get link failed: retry limit reached") +} + +func (d *BitQiu) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + parentID := d.resolveParentID(parentDir) + parentPath := "" + if parentDir != nil { + parentPath = parentDir.GetPath() + } + form := map[string]string{ + "parentId": parentID, + "name": dirName, + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[CreateDirData] + if err := d.postForm(ctx, createDirURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + newParentID := parentID + if resp.Data.ParentID != "" { + newParentID = resp.Data.ParentID + } + name := resp.Data.Name + if name == "" { + name = dirName + } + resource := Resource{ + ResourceID: resp.Data.DirID, + ResourceType: 1, + Name: name, + ParentID: newParentID, + } + obj, err := resource.toObject(newParentID, parentPath) + if err != nil { + return nil, err + } + if o, ok := obj.(*Object); ok { + o.ParentID = newParentID + } + return obj, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("create folder failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("create folder failed: retry limit reached") +} + +func (d *BitQiu) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + targetParentID := d.resolveParentID(dstDir) + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "parentId": targetParentID, + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["dirIds"] = srcObj.GetID() + } else { + form["fileIds"] = srcObj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, moveResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + dstPath := "" + if dstDir != nil { + dstPath = dstDir.GetPath() + } + if setter, ok := srcObj.(model.SetPath); ok { + setter.SetPath(path.Join(dstPath, srcObj.GetName())) + } + if o, ok := srcObj.(*Object); ok { + o.ParentID = targetParentID + } + return srcObj, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("move failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("move failed: retry limit reached") +} + +func (d *BitQiu) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + form := map[string]string{ + "resourceId": srcObj.GetID(), + "name": newName, + "type": "0", + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["type"] = "1" + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, renameResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + return updateObjectName(srcObj, newName), nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("rename failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("rename failed: retry limit reached") +} + +func (d *BitQiu) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + targetParentID := d.resolveParentID(dstDir) + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "parentId": targetParentID, + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["dirIds"] = srcObj.GetID() + } else { + form["fileIds"] = srcObj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, copyResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode, copySubmittedCode: + return d.waitForCopiedObject(ctx, srcObj, dstDir) + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("copy failed: %s", resp.Message) + } + } + + return nil, fmt.Errorf("copy failed: retry limit reached") +} + +func (d *BitQiu) Remove(ctx context.Context, obj model.Obj) error { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return err + } + } + + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "org_channel": orgChannel, + } + if obj.IsDir() { + form["dirIds"] = obj.GetID() + } else { + form["fileIds"] = obj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, deleteResourceURL, form, &resp); err != nil { + return err + } + switch resp.Code { + case successCode: + return nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return err + } + default: + return fmt.Errorf("remove failed: %s", resp.Message) + } + } + return fmt.Errorf("remove failed: retry limit reached") +} + +func (d *BitQiu) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + up(0) + tmpFile, md5sum, err := streamPkg.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return nil, err + } + defer tmpFile.Close() + + parentID := d.resolveParentID(dstDir) + parentPath := "" + if dstDir != nil { + parentPath = dstDir.GetPath() + } + form := map[string]string{ + "parentId": parentID, + "name": file.GetName(), + "size": strconv.FormatInt(file.GetSize(), 10), + "hash": md5sum, + "sampleMd5": md5sum, + "org_channel": orgChannel, + } + var resp Response[json.RawMessage] + if err = d.postForm(ctx, uploadInitializeURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != uploadSuccessCode { + switch resp.Code { + case successCode: + var initData UploadInitData + if err := json.Unmarshal(resp.Data, &initData); err != nil { + return nil, fmt.Errorf("parse upload init response failed: %w", err) + } + serverCode, err := d.uploadFileInChunks(ctx, tmpFile, file.GetSize(), md5sum, initData, up) + if err != nil { + return nil, err + } + obj, err := d.completeChunkUpload(ctx, initData, parentID, parentPath, file.GetName(), file.GetSize(), md5sum, serverCode) + if err != nil { + return nil, err + } + up(100) + return obj, nil + default: + return nil, fmt.Errorf("upload failed: %s", resp.Message) + } + } + + var resource Resource + if err := json.Unmarshal(resp.Data, &resource); err != nil { + return nil, fmt.Errorf("parse upload response failed: %w", err) + } + obj, err := resource.toObject(parentID, parentPath) + if err != nil { + return nil, err + } + up(100) + return obj, nil +} + +func (d *BitQiu) uploadFileInChunks(ctx context.Context, tmpFile model.File, size int64, md5sum string, initData UploadInitData, up driver.UpdateProgress) (string, error) { + if d.client == nil { + return "", fmt.Errorf("client not initialized") + } + if size <= 0 { + return "", fmt.Errorf("invalid file size") + } + buf := make([]byte, chunkSize) + offset := int64(0) + var finishedFlag string + + for offset < size { + chunkLen := chunkSize + remaining := size - offset + if remaining < chunkLen { + chunkLen = remaining + } + + reader := io.NewSectionReader(tmpFile, offset, chunkLen) + chunkBuf := buf[:chunkLen] + if _, err := io.ReadFull(reader, chunkBuf); err != nil { + return "", fmt.Errorf("read chunk failed: %w", err) + } + + headers := map[string]string{ + "accept": "*/*", + "content-type": "application/octet-stream", + "appid": initData.AppID, + "token": initData.Token, + "userid": strconv.FormatInt(initData.UserID, 10), + "serialnumber": initData.SerialNumber, + "hash": md5sum, + "len": strconv.FormatInt(chunkLen, 10), + "offset": strconv.FormatInt(offset, 10), + "user-agent": d.userAgent(), + } + + var chunkResp ChunkUploadResponse + req := d.client.R(). + SetContext(ctx). + SetHeaders(headers). + SetBody(chunkBuf). + SetResult(&chunkResp) + + if _, err := req.Post(initData.UploadURL); err != nil { + return "", err + } + if chunkResp.ErrCode != 0 { + return "", fmt.Errorf("chunk upload failed with code %d", chunkResp.ErrCode) + } + finishedFlag = chunkResp.FinishedFlag + offset += chunkLen + up(float64(offset) * 100 / float64(size)) + } + + if finishedFlag == "" { + return "", fmt.Errorf("upload finished without server code") + } + return finishedFlag, nil +} + +func (d *BitQiu) completeChunkUpload(ctx context.Context, initData UploadInitData, parentID, parentPath, name string, size int64, md5sum, serverCode string) (model.Obj, error) { + form := map[string]string{ + "currentPage": "1", + "limit": "1", + "userId": strconv.FormatInt(initData.UserID, 10), + "status": "0", + "parentId": parentID, + "name": name, + "fileUid": initData.FileUID, + "fileSid": initData.FileSID, + "size": strconv.FormatInt(size, 10), + "serverCode": serverCode, + "snapTime": "", + "hash": md5sum, + "sampleMd5": md5sum, + "org_channel": orgChannel, + } + + var resp Response[Resource] + if err := d.postForm(ctx, uploadCompleteURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != successCode { + return nil, fmt.Errorf("complete upload failed: %s", resp.Message) + } + + return resp.Data.toObject(parentID, parentPath) +} + +func (d *BitQiu) login(ctx context.Context) error { + if d.client == nil { + return fmt.Errorf("client not initialized") + } + + form := map[string]string{ + "passport": d.Username, + "password": utils.GetMD5EncodeStr(d.Password), + "remember": "0", + "captcha": "", + "org_channel": orgChannel, + } + var resp Response[LoginData] + if err := d.postForm(ctx, loginURL, form, &resp); err != nil { + return err + } + if resp.Code != successCode { + return fmt.Errorf("login failed: %s", resp.Message) + } + d.userID = strconv.FormatInt(resp.Data.UserID, 10) + return d.ensureRootFolderID(ctx) +} + +func (d *BitQiu) ensureRootFolderID(ctx context.Context) error { + rootID := d.Addition.GetRootId() + if rootID != "" && rootID != "0" { + return nil + } + + form := map[string]string{ + "org_channel": orgChannel, + } + var resp Response[UserInfoData] + if err := d.postForm(ctx, userInfoURL, form, &resp); err != nil { + return err + } + if resp.Code != successCode { + return fmt.Errorf("get user info failed: %s", resp.Message) + } + if resp.Data.RootDirID == "" { + return fmt.Errorf("get user info failed: empty root dir id") + } + if d.Addition.RootFolderID != resp.Data.RootDirID { + d.Addition.RootFolderID = resp.Data.RootDirID + op.MustSaveDriverStorage(d) + } + return nil +} + +func (d *BitQiu) postForm(ctx context.Context, url string, form map[string]string, result interface{}) error { + if d.client == nil { + return fmt.Errorf("client not initialized") + } + req := d.client.R(). + SetContext(ctx). + SetHeaders(d.commonHeaders()). + SetFormData(form) + if result != nil { + req = req.SetResult(result) + } + _, err := req.Post(url) + return err +} + +func (d *BitQiu) waitForCopiedObject(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + expectedName := srcObj.GetName() + expectedIsDir := srcObj.IsDir() + var lastListErr error + + for attempt := 0; attempt < copyPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, copyPollInterval); err != nil { + return nil, err + } + } + + if err := d.checkCopyFailure(ctx); err != nil { + return nil, err + } + + obj, err := d.findObjectInDir(ctx, dstDir, expectedName, expectedIsDir) + if err != nil { + lastListErr = err + continue + } + if obj != nil { + return obj, nil + } + } + if lastListErr != nil { + return nil, lastListErr + } + return nil, fmt.Errorf("copy task timed out waiting for completion") +} + +func (d *BitQiu) checkCopyFailure(ctx context.Context) error { + form := map[string]string{ + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[AsyncManagerData] + if err := d.postForm(ctx, copyManagerURL, form, &resp); err != nil { + return err + } + switch resp.Code { + case successCode: + if len(resp.Data.FailTasks) > 0 { + return fmt.Errorf("copy failed: %s", resp.Data.FailTasks[0].ErrorMessage()) + } + return nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return err + } + default: + return fmt.Errorf("query copy status failed: %s", resp.Message) + } + } + return fmt.Errorf("query copy status failed: retry limit reached") +} + +func (d *BitQiu) findObjectInDir(ctx context.Context, dir model.Obj, name string, isDir bool) (model.Obj, error) { + objs, err := d.List(ctx, dir, model.ListArgs{}) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name && obj.IsDir() == isDir { + return obj, nil + } + } + return nil, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (d *BitQiu) commonHeaders() map[string]string { + headers := map[string]string{ + "accept": "application/json, text/plain, */*", + "accept-language": "en-US,en;q=0.9", + "cache-control": "no-cache", + "pragma": "no-cache", + "user-platform": d.Addition.UserPlatform, + "x-kl-saas-ajax-request": "Ajax_Request", + "x-requested-with": "XMLHttpRequest", + "referer": baseURL + "/", + "origin": baseURL, + "user-agent": d.userAgent(), + } + return headers +} + +func (d *BitQiu) userAgent() string { + if ua := strings.TrimSpace(d.Addition.UserAgent); ua != "" { + return ua + } + return defaultUserAgent +} + +func (d *BitQiu) resolveParentID(dir model.Obj) string { + if dir != nil && dir.GetID() != "" { + return dir.GetID() + } + if root := d.Addition.GetRootId(); root != "" { + return root + } + return config.DefaultRoot +} + +func (d *BitQiu) pageSize() int { + if size, err := strconv.Atoi(d.Addition.PageSize); err == nil && size > 0 { + return size + } + return 24 +} + +func (d *BitQiu) orderType() string { + if d.Addition.OrderType != "" { + return d.Addition.OrderType + } + return "updateTime" +} + +func (d *BitQiu) orderDesc() string { + if d.Addition.OrderDesc { + return "1" + } + return "0" +} + +var _ driver.Driver = (*BitQiu)(nil) +var _ driver.PutResult = (*BitQiu)(nil) diff --git a/drivers/bitqiu/meta.go b/drivers/bitqiu/meta.go new file mode 100644 index 00000000000..63cb03344c4 --- /dev/null +++ b/drivers/bitqiu/meta.go @@ -0,0 +1,28 @@ +package bitqiu + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + UserPlatform string `json:"user_platform" help:"Optional device identifier; auto-generated if empty."` + OrderType string `json:"order_type" type:"select" options:"updateTime,createTime,name,size" default:"updateTime"` + OrderDesc bool `json:"order_desc"` + PageSize string `json:"page_size" default:"24" help:"Number of entries to request per page."` + UserAgent string `json:"user_agent" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"` +} + +var config = driver.Config{ + Name: "BitQiu", + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BitQiu{} + }) +} diff --git a/drivers/bitqiu/types.go b/drivers/bitqiu/types.go new file mode 100644 index 00000000000..8fbec989135 --- /dev/null +++ b/drivers/bitqiu/types.go @@ -0,0 +1,107 @@ +package bitqiu + +import "encoding/json" + +type Response[T any] struct { + Code string `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type LoginData struct { + UserID int64 `json:"userId"` +} + +type ResourcePage struct { + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` + TotalCount int `json:"totalCount"` + TotalPageCount int `json:"totalPageCount"` + Data []Resource `json:"data"` + HasNext bool `json:"hasNext"` +} + +type Resource struct { + ResourceID string `json:"resourceId"` + ResourceUID string `json:"resourceUid"` + ResourceType int `json:"resourceType"` + ParentID string `json:"parentId"` + Name string `json:"name"` + ExtName string `json:"extName"` + Size *json.Number `json:"size"` + CreateTime *string `json:"createTime"` + UpdateTime *string `json:"updateTime"` + FileMD5 string `json:"fileMd5"` +} + +type DownloadData struct { + URL string `json:"url"` + MD5 string `json:"md5"` + Size int64 `json:"size"` +} + +type UserInfoData struct { + RootDirID string `json:"rootDirId"` +} + +type CreateDirData struct { + DirID string `json:"dirId"` + Name string `json:"name"` + ParentID string `json:"parentId"` +} + +type AsyncManagerData struct { + WaitTasks []AsyncTask `json:"waitTaskList"` + RunningTasks []AsyncTask `json:"runningTaskList"` + SuccessTasks []AsyncTask `json:"successTaskList"` + FailTasks []AsyncTask `json:"failTaskList"` + TaskList []AsyncTask `json:"taskList"` +} + +type AsyncTask struct { + TaskID string `json:"taskId"` + Status int `json:"status"` + ErrorMsg string `json:"errorMsg"` + Message string `json:"message"` + Result *AsyncTaskInfo `json:"result"` + TargetName string `json:"targetName"` + TargetDirID string `json:"parentId"` +} + +type AsyncTaskInfo struct { + Resource Resource `json:"resource"` + DirID string `json:"dirId"` + FileID string `json:"fileId"` + Name string `json:"name"` + ParentID string `json:"parentId"` +} + +func (t AsyncTask) ErrorMessage() string { + if t.ErrorMsg != "" { + return t.ErrorMsg + } + if t.Message != "" { + return t.Message + } + return "unknown error" +} + +type UploadInitData struct { + Name string `json:"name"` + Size int64 `json:"size"` + Token string `json:"token"` + FileUID string `json:"fileUid"` + FileSID string `json:"fileSid"` + ParentID string `json:"parentId"` + UserID int64 `json:"userId"` + SerialNumber string `json:"serialNumber"` + UploadURL string `json:"uploadUrl"` + AppID string `json:"appId"` +} + +type ChunkUploadResponse struct { + ErrCode int `json:"errCode"` + Offset int64 `json:"offset"` + Finished int `json:"finished"` + FinishedFlag string `json:"finishedFlag"` +} diff --git a/drivers/bitqiu/util.go b/drivers/bitqiu/util.go new file mode 100644 index 00000000000..bccd6815e9b --- /dev/null +++ b/drivers/bitqiu/util.go @@ -0,0 +1,102 @@ +package bitqiu + +import ( + "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Object struct { + model.Object + ParentID string +} + +func (r Resource) toObject(parentID, parentPath string) (model.Obj, error) { + id := r.ResourceID + if id == "" { + id = r.ResourceUID + } + obj := &Object{ + Object: model.Object{ + ID: id, + Name: r.Name, + IsFolder: r.ResourceType == 1, + }, + ParentID: parentID, + } + if r.Size != nil { + if size, err := (*r.Size).Int64(); err == nil { + obj.Size = size + } + } + if ct := parseBitQiuTime(r.CreateTime); !ct.IsZero() { + obj.Ctime = ct + } + if mt := parseBitQiuTime(r.UpdateTime); !mt.IsZero() { + obj.Modified = mt + } + if r.FileMD5 != "" { + obj.HashInfo = utils.NewHashInfo(utils.MD5, strings.ToLower(r.FileMD5)) + } + obj.SetPath(path.Join(parentPath, obj.Name)) + return obj, nil +} + +func parseBitQiuTime(value *string) time.Time { + if value == nil { + return time.Time{} + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return time.Time{} + } + if ts, err := time.ParseInLocation("2006-01-02 15:04:05", trimmed, time.Local); err == nil { + return ts + } + return time.Time{} +} + +func updateObjectName(obj model.Obj, newName string) model.Obj { + newPath := path.Join(parentPathOf(obj.GetPath()), newName) + + switch o := obj.(type) { + case *Object: + o.Name = newName + o.Object.Name = newName + o.SetPath(newPath) + return o + case *model.Object: + o.Name = newName + o.SetPath(newPath) + return o + } + + if setter, ok := obj.(model.SetPath); ok { + setter.SetPath(newPath) + } + + return &model.Object{ + ID: obj.GetID(), + Path: newPath, + Name: newName, + Size: obj.GetSize(), + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + } +} + +func parentPathOf(p string) string { + if p == "" { + return "" + } + dir := path.Dir(p) + if dir == "." { + return "" + } + return dir +} From b4d9beb49cba399842a54fcc33bc95a4a09b7bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Thu, 23 Oct 2025 09:31:15 -0700 Subject: [PATCH 574/659] fix(Mediatrack): Add support for X-Device-Fingerprint header (#9354) Introduce a `DeviceFingerprint` field to the request metadata. This field is used to conditionally set the `X-Device-Fingerprint` HTTP header in outgoing requests if its value is not empty. --- drivers/mediatrack/meta.go | 5 +++-- drivers/mediatrack/util.go | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/mediatrack/meta.go b/drivers/mediatrack/meta.go index 47f112c3573..ade8ae1c8fe 100644 --- a/drivers/mediatrack/meta.go +++ b/drivers/mediatrack/meta.go @@ -9,8 +9,9 @@ type Addition struct { AccessToken string `json:"access_token" required:"true"` ProjectID string `json:"project_id"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` - OrderDesc bool `json:"order_desc"` + OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` + OrderDesc bool `json:"order_desc"` + DeviceFingerprint string `json:"device_fingerprint" required:"true"` } var config = driver.Config{ diff --git a/drivers/mediatrack/util.go b/drivers/mediatrack/util.go index 37ca0b3d09c..f5b751111a1 100644 --- a/drivers/mediatrack/util.go +++ b/drivers/mediatrack/util.go @@ -17,6 +17,9 @@ import ( func (d *MediaTrack) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) + if d.DeviceFingerprint != "" { + req.SetHeader("X-Device-Fingerprint", d.DeviceFingerprint) + } if callback != nil { callback(req) } From 0cbc7ebc92b2d64299ab6014be4c3feffeb1c967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 11 Nov 2025 20:25:26 +0800 Subject: [PATCH 575/659] feat(driver): Added support for Gitee driver (#9368) * feat(driver): Added support for Gitee driver - Implemented core driver functions including initialization, file listing, and file linking - Added Gitee-specific API interaction and object mapping - Registered Gitee driver in the driver registry * feat(driver): Added cookie-based authentication support for Gitee driver - Extended request handling to include `Cookie` header if provided - Updated metadata to include `cookie` field with appropriate documentation - Adjusted file link generation to propagate `Cookie` headers in requests --- drivers/all.go | 1 + drivers/gitee/driver.go | 224 ++++++++++++++++++++++++++++++++++++++++ drivers/gitee/meta.go | 29 ++++++ drivers/gitee/types.go | 60 +++++++++++ drivers/gitee/util.go | 44 ++++++++ 5 files changed, 358 insertions(+) create mode 100644 drivers/gitee/driver.go create mode 100644 drivers/gitee/meta.go create mode 100644 drivers/gitee/types.go create mode 100644 drivers/gitee/util.go diff --git a/drivers/all.go b/drivers/all.go index efeb6f7716a..3eb7e813bcc 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -31,6 +31,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" + _ "github.com/alist-org/alist/v3/drivers/gitee" _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/github_releases" _ "github.com/alist-org/alist/v3/drivers/gofile" diff --git a/drivers/gitee/driver.go b/drivers/gitee/driver.go new file mode 100644 index 00000000000..78a400941b9 --- /dev/null +++ b/drivers/gitee/driver.go @@ -0,0 +1,224 @@ +package gitee + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type Gitee struct { + model.Storage + Addition + client *resty.Client +} + +func (d *Gitee) Config() driver.Config { + return config +} + +func (d *Gitee) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Gitee) Init(ctx context.Context) error { + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + d.Endpoint = strings.TrimSpace(d.Endpoint) + if d.Endpoint == "" { + d.Endpoint = "https://gitee.com/api/v5" + } + d.Endpoint = strings.TrimSuffix(d.Endpoint, "/") + d.Owner = strings.TrimSpace(d.Owner) + d.Repo = strings.TrimSpace(d.Repo) + d.Token = strings.TrimSpace(d.Token) + d.DownloadProxy = strings.TrimSpace(d.DownloadProxy) + if d.Owner == "" || d.Repo == "" { + return errors.New("owner and repo are required") + } + d.client = base.NewRestyClient(). + SetBaseURL(d.Endpoint). + SetHeader("Accept", "application/json") + repo, err := d.getRepo() + if err != nil { + return err + } + d.Ref = strings.TrimSpace(d.Ref) + if d.Ref == "" { + d.Ref = repo.DefaultBranch + } + return nil +} + +func (d *Gitee) Drop(ctx context.Context) error { + return nil +} + +func (d *Gitee) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + relPath := d.relativePath(dir.GetPath()) + contents, err := d.listContents(relPath) + if err != nil { + return nil, err + } + objs := make([]model.Obj, 0, len(contents)) + for i := range contents { + objs = append(objs, contents[i].toModelObj()) + } + return objs, nil +} + +func (d *Gitee) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var downloadURL string + if obj, ok := file.(*Object); ok { + downloadURL = obj.DownloadURL + if downloadURL == "" { + relPath := d.relativePath(file.GetPath()) + content, err := d.getContent(relPath) + if err != nil { + return nil, err + } + if content.DownloadURL == "" { + return nil, errors.New("empty download url") + } + obj.DownloadURL = content.DownloadURL + downloadURL = content.DownloadURL + } + } else { + relPath := d.relativePath(file.GetPath()) + content, err := d.getContent(relPath) + if err != nil { + return nil, err + } + if content.DownloadURL == "" { + return nil, errors.New("empty download url") + } + downloadURL = content.DownloadURL + } + url := d.applyProxy(downloadURL) + return &model.Link{ + URL: url, + Header: http.Header{ + "Cookie": {d.Cookie}, + }, + }, nil +} + +func (d *Gitee) newRequest() *resty.Request { + req := d.client.R() + if d.Token != "" { + req.SetQueryParam("access_token", d.Token) + } + if d.Ref != "" { + req.SetQueryParam("ref", d.Ref) + } + return req +} + +func (d *Gitee) apiPath(path string) string { + escapedOwner := url.PathEscape(d.Owner) + escapedRepo := url.PathEscape(d.Repo) + if path == "" { + return fmt.Sprintf("/repos/%s/%s/contents", escapedOwner, escapedRepo) + } + return fmt.Sprintf("/repos/%s/%s/contents/%s", escapedOwner, escapedRepo, encodePath(path)) +} + +func (d *Gitee) listContents(path string) ([]Content, error) { + res, err := d.newRequest().Get(d.apiPath(path)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var contents []Content + if err := utils.Json.Unmarshal(res.Body(), &contents); err != nil { + var single Content + if err2 := utils.Json.Unmarshal(res.Body(), &single); err2 == nil && single.Type != "" { + if single.Type != "dir" { + return nil, errs.NotFolder + } + return []Content{}, nil + } + return nil, err + } + for i := range contents { + contents[i].Path = joinPath(path, contents[i].Name) + } + return contents, nil +} + +func (d *Gitee) getContent(path string) (*Content, error) { + res, err := d.newRequest().Get(d.apiPath(path)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var content Content + if err := utils.Json.Unmarshal(res.Body(), &content); err != nil { + return nil, err + } + if content.Type == "" { + return nil, errors.New("invalid response") + } + if content.Path == "" { + content.Path = path + } + return &content, nil +} + +func (d *Gitee) relativePath(full string) string { + full = utils.FixAndCleanPath(full) + root := utils.FixAndCleanPath(d.RootFolderPath) + if root == "/" { + return strings.TrimPrefix(full, "/") + } + if utils.PathEqual(full, root) { + return "" + } + prefix := utils.PathAddSeparatorSuffix(root) + if strings.HasPrefix(full, prefix) { + return strings.TrimPrefix(full, prefix) + } + return strings.TrimPrefix(full, "/") +} + +func (d *Gitee) applyProxy(raw string) string { + if raw == "" || d.DownloadProxy == "" { + return raw + } + proxy := d.DownloadProxy + if !strings.HasSuffix(proxy, "/") { + proxy += "/" + } + return proxy + strings.TrimLeft(raw, "/") +} + +func encodePath(p string) string { + if p == "" { + return "" + } + parts := strings.Split(p, "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) + } + return strings.Join(parts, "/") +} + +func joinPath(base, name string) string { + if base == "" { + return name + } + return strings.TrimPrefix(stdpath.Join(base, name), "./") +} diff --git a/drivers/gitee/meta.go b/drivers/gitee/meta.go new file mode 100644 index 00000000000..2f926d635f3 --- /dev/null +++ b/drivers/gitee/meta.go @@ -0,0 +1,29 @@ +package gitee + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Endpoint string `json:"endpoint" type:"string" help:"Gitee API endpoint, default https://gitee.com/api/v5"` + Token string `json:"token" type:"string"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"Branch, tag or commit SHA, defaults to repository default branch"` + DownloadProxy string `json:"download_proxy" type:"string" help:"Prefix added before download URLs, e.g. https://mirror.example.com/"` + Cookie string `json:"cookie" type:"string" help:"Cookie returned from user info request"` +} + +var config = driver.Config{ + Name: "Gitee", + LocalSort: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Gitee{} + }) +} diff --git a/drivers/gitee/types.go b/drivers/gitee/types.go new file mode 100644 index 00000000000..c10536a5d10 --- /dev/null +++ b/drivers/gitee/types.go @@ -0,0 +1,60 @@ +package gitee + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Links struct { + Self string `json:"self"` + Html string `json:"html"` +} + +type Content struct { + Type string `json:"type"` + Size *int64 `json:"size"` + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + URL string `json:"url"` + HtmlURL string `json:"html_url"` + DownloadURL string `json:"download_url"` + Links Links `json:"_links"` +} + +func (c Content) toModelObj() model.Obj { + size := int64(0) + if c.Size != nil { + size = *c.Size + } + return &Object{ + Object: model.Object{ + ID: c.Path, + Name: c.Name, + Size: size, + Modified: time.Unix(0, 0), + IsFolder: c.Type == "dir", + }, + DownloadURL: c.DownloadURL, + HtmlURL: c.HtmlURL, + } +} + +type Object struct { + model.Object + DownloadURL string + HtmlURL string +} + +func (o *Object) URL() string { + return o.DownloadURL +} + +type Repo struct { + DefaultBranch string `json:"default_branch"` +} + +type ErrResp struct { + Message string `json:"message"` +} diff --git a/drivers/gitee/util.go b/drivers/gitee/util.go new file mode 100644 index 00000000000..fbef972ad3d --- /dev/null +++ b/drivers/gitee/util.go @@ -0,0 +1,44 @@ +package gitee + +import ( + "fmt" + "net/url" + + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +func (d *Gitee) getRepo() (*Repo, error) { + req := d.client.R() + if d.Token != "" { + req.SetQueryParam("access_token", d.Token) + } + if d.Cookie != "" { + req.SetHeader("Cookie", d.Cookie) + } + escapedOwner := url.PathEscape(d.Owner) + escapedRepo := url.PathEscape(d.Repo) + res, err := req.Get(fmt.Sprintf("/repos/%s/%s", escapedOwner, escapedRepo)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var repo Repo + if err := utils.Json.Unmarshal(res.Body(), &repo); err != nil { + return nil, err + } + if repo.DefaultBranch == "" { + return nil, fmt.Errorf("failed to fetch default branch") + } + return &repo, nil +} + +func toErr(res *resty.Response) error { + var errMsg ErrResp + if err := utils.Json.Unmarshal(res.Body(), &errMsg); err == nil && errMsg.Message != "" { + return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) + } + return fmt.Errorf(res.Status()) +} From ce41587095bb3a680b6bc926aacd0eecb61722a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 11 Nov 2025 20:26:51 +0800 Subject: [PATCH 576/659] feat(cloud189): Added sanitization for file and folder names (#9366) - Introduced `sanitizeName` function to remove four-byte characters (e.g., emojis) from names before upload or creation. - Added `StripEmoji` option in driver configurations for cloud189 and cloud189pc. - Updated file and folder operations (upload, rename, and creation) to use sanitized names. - Ensured compatibility with both cloud189 and cloud189pc implementations. --- drivers/189/driver.go | 6 ++++-- drivers/189/meta.go | 7 ++++--- drivers/189/util.go | 33 ++++++++++++++++++++++++++++++--- drivers/189pc/driver.go | 16 +++++++++++++--- drivers/189pc/meta.go | 7 ++++--- drivers/189pc/utils.go | 37 +++++++++++++++++++++++++++++++++---- 6 files changed, 88 insertions(+), 18 deletions(-) diff --git a/drivers/189/driver.go b/drivers/189/driver.go index 6fc4932640c..0c8db1134ee 100644 --- a/drivers/189/driver.go +++ b/drivers/189/driver.go @@ -80,9 +80,10 @@ func (d *Cloud189) Link(ctx context.Context, file model.Obj, args model.LinkArgs } func (d *Cloud189) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + safeName := d.sanitizeName(dirName) form := map[string]string{ "parentFolderId": parentDir.GetID(), - "folderName": dirName, + "folderName": safeName, } _, err := d.request("https://cloud.189.cn/api/open/file/createFolder.action", http.MethodPost, func(req *resty.Request) { req.SetFormData(form) @@ -126,9 +127,10 @@ func (d *Cloud189) Rename(ctx context.Context, srcObj model.Obj, newName string) idKey = "folderId" nameKey = "destFolderName" } + safeName := d.sanitizeName(newName) form := map[string]string{ idKey: srcObj.GetID(), - nameKey: newName, + nameKey: safeName, } _, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetFormData(form) diff --git a/drivers/189/meta.go b/drivers/189/meta.go index ad621fb440d..da81406e683 100644 --- a/drivers/189/meta.go +++ b/drivers/189/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` + StripEmoji bool `json:"strip_emoji" help:"Remove four-byte characters (e.g., emoji) before upload"` driver.RootID } diff --git a/drivers/189/util.go b/drivers/189/util.go index 16a5aa3996e..ee2b5061dd8 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -11,9 +11,11 @@ import ( "io" "math" "net/http" + "path" "strconv" "strings" "time" + "unicode/utf8" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -222,13 +224,37 @@ func (d *Cloud189) getFiles(fileId string) ([]model.Obj, error) { return res, nil } +func (d *Cloud189) sanitizeName(name string) string { + if !d.StripEmoji { + return name + } + b := strings.Builder{} + for _, r := range name { + if utf8.RuneLen(r) == 4 { + continue + } + b.WriteRune(r) + } + sanitized := b.String() + if sanitized == "" { + ext := path.Ext(name) + if ext != "" { + sanitized = "file" + ext + } else { + sanitized = "file" + } + } + return sanitized +} + func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { + safeName := d.sanitizeName(file.GetName()) res, err := d.client.R().SetMultipartFormData(map[string]string{ "parentId": dstDir.GetID(), "sessionKey": "??", "opertype": "1", - "fname": file.GetName(), - }).SetMultipartField("Filedata", file.GetName(), file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") + "fname": safeName, + }).SetMultipartField("Filedata", safeName, file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") if err != nil { return err } @@ -313,9 +339,10 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F const DEFAULT int64 = 10485760 var count = int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + safeName := d.sanitizeName(file.GetName()) res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ "parentFolderId": dstDir.GetID(), - "fileName": encode(file.GetName()), + "fileName": encode(safeName), "fileSize": strconv.FormatInt(file.GetSize(), 10), "sliceSize": strconv.FormatInt(DEFAULT, 10), "lazyCheck": "1", diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index c91caf2fb4f..9462cef6662 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -205,10 +205,11 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s fullUrl += "/createFolder.action" var newFolder Cloud189Folder + safeName := y.sanitizeName(dirName) _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ - "folderName": dirName, + "folderName": safeName, "relativePath": "", }) if isFamily { @@ -225,6 +226,7 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s if err != nil { return nil, err } + newFolder.Name = safeName return &newFolder, nil } @@ -258,21 +260,29 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin } var newObj model.Obj + safeName := y.sanitizeName(newName) switch f := srcObj.(type) { case *Cloud189File: fullUrl += "/renameFile.action" queryParam["fileId"] = srcObj.GetID() - queryParam["destFileName"] = newName + queryParam["destFileName"] = safeName newObj = &Cloud189File{Icon: f.Icon} // 复用预览 case *Cloud189Folder: fullUrl += "/renameFolder.action" queryParam["folderId"] = srcObj.GetID() - queryParam["destFolderName"] = newName + queryParam["destFolderName"] = safeName newObj = &Cloud189Folder{} default: return nil, errs.NotSupport } + switch obj := newObj.(type) { + case *Cloud189File: + obj.Name = safeName + case *Cloud189Folder: + obj.Name = safeName + } + _, err := y.request(fullUrl, method, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(queryParam) }, nil, newObj, isFamily) diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 1891c5c0ccd..d6edc063593 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - VCode string `json:"validate_code"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + VCode string `json:"validate_code"` + StripEmoji bool `json:"strip_emoji" help:"Remove four-byte characters (e.g., emoji) before upload"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index a8b444cb4b7..ca89251e278 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -12,11 +12,13 @@ import ( "net/http/cookiejar" "net/url" "os" + "path" "regexp" "sort" "strconv" "strings" "time" + "unicode/utf8" "golang.org/x/sync/semaphore" @@ -57,6 +59,29 @@ const ( CHANNEL_ID = "web_cloud.189.cn" ) +func (y *Cloud189PC) sanitizeName(name string) string { + if !y.StripEmoji { + return name + } + b := strings.Builder{} + for _, r := range name { + if utf8.RuneLen(r) == 4 { + continue + } + b.WriteRune(r) + } + sanitized := b.String() + if sanitized == "" { + ext := path.Ext(name) + if ext != "" { + sanitized = "file" + ext + } else { + sanitized = "file" + } + } + return sanitized +} + func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { dateOfGmt := getHttpDateStr() sessionKey := y.getTokenInfo().SessionKey @@ -475,10 +500,11 @@ func (y *Cloud189PC) refreshSession() (err error) { func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { size := file.GetSize() sliceSize := partSize(size) + safeName := y.sanitizeName(file.GetName()) params := Params{ "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(file.GetName()), + "fileName": url.QueryEscape(safeName), "fileSize": fmt.Sprint(file.GetSize()), "sliceSize": fmt.Sprint(sliceSize), "lazyCheck": "1", @@ -596,7 +622,8 @@ func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream m return nil, errors.New("invalid hash") } - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, stream.GetName(), fmt.Sprint(stream.GetSize()), isFamily) + safeName := y.sanitizeName(stream.GetName()) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, safeName, fmt.Sprint(stream.GetSize()), isFamily) if err != nil { return nil, err } @@ -615,6 +642,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode tmpF *os.File err error ) + safeName := y.sanitizeName(file.GetName()) size := file.GetSize() if _, ok := cache.(io.ReaderAt); !ok && size > 0 { tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") @@ -697,7 +725,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode //step.2 预上传 params := Params{ "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(file.GetName()), + "fileName": url.QueryEscape(safeName), "fileSize": fmt.Sprint(file.GetSize()), "fileMd5": fileMd5Hex, "sliceSize": fmt.Sprint(sliceSize), @@ -833,9 +861,10 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model return nil, err } rateLimited := driver.NewLimitedUploadStream(ctx, io.NopCloser(tempFile)) + safeName := y.sanitizeName(file.GetName()) // 创建上传会话 - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize()), isFamily) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, safeName, fmt.Sprint(file.GetSize()), isFamily) if err != nil { return nil, err } From 3cddb6b7edd77b422b5c9c66462b58538d6c206b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 11 Nov 2025 20:27:20 +0800 Subject: [PATCH 577/659] fix(driver): Handle Lanzou anti-crawler challenge by recalculating cookies (#9364) - Detect and solve `acw_sc__v2` challenge to bypass anti-crawler validation - Refactored request header initialization logic for clarity --- drivers/lanzou/util.go | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index e66252bcc79..be53963c157 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -430,17 +430,35 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( file.Time = timeFindReg.FindString(sharePageData) // 重定向获取真实链接 - res, err := base.NoRedirectClient.R().SetHeaders(map[string]string{ + headers := map[string]string{ "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", - }).Get(downloadUrl) + } + res, err := base.NoRedirectClient.R().SetHeaders(headers).Get(downloadUrl) if err != nil { return nil, err } + rPageData := res.String() + if findAcwScV2Reg.MatchString(rPageData) { + log.Debug("lanzou: detected acw_sc__v2 challenge, recalculating cookie") + acwScV2, err := CalcAcwScV2(rPageData) + if err != nil { + return nil, err + } + // retry with calculated cookie to bypass anti-crawler validation + res, err = base.NoRedirectClient.R(). + SetHeaders(headers). + SetCookie(&http.Cookie{Name: "acw_sc__v2", Value: acwScV2}). + Get(downloadUrl) + if err != nil { + return nil, err + } + rPageData = res.String() + } + file.Url = res.Header().Get("location") // 触发验证 - rPageData := res.String() if res.StatusCode() != 302 { param, err = htmlJsonToMap(rPageData) if err != nil { From 129895beecb87c278d227a2038a2056fa6bbf804 Mon Sep 17 00:00:00 2001 From: vxtls <187420201+vxtls@users.noreply.github.com> Date: Sat, 15 Nov 2025 12:33:58 -0500 Subject: [PATCH 578/659] fix(misskey): folderId format validation and root directory handling --- drivers/misskey/util.go | 57 +++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go index f8baeafa6cb..d301955ec42 100644 --- a/drivers/misskey/util.go +++ b/drivers/misskey/util.go @@ -4,6 +4,7 @@ import ( "context" "errors" "io" + "net/http" "time" "github.com/go-resty/resty/v2" @@ -56,23 +57,27 @@ func setBody(body interface{}) base.ReqCallback { } func handleFolderId(dir model.Obj) interface{} { - if dir.GetID() == "" { - return nil + if isRootFolder(dir) { + return nil // Root folder doesn't need folderId } return dir.GetID() } +func isRootFolder(dir model.Obj) bool { + return dir.GetID() == "" +} + // API layer methods func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { var files []MFile var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} } - err := d.request("/files", "POST", setBody(body), &files) + err := d.request("/files", http.MethodPost, setBody(body), &files) if err != nil { return []model.Obj{}, err } @@ -84,12 +89,12 @@ func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) { var folders []MFolder var body map[string]string - if dir.GetPath() != "/" { + if !isRootFolder(dir) { body = map[string]string{"folderId": dir.GetID()} } else { body = map[string]string{} } - err := d.request("/folders", "POST", setBody(body), &folders) + err := d.request("/folders", http.MethodPost, setBody(body), &folders) if err != nil { return []model.Obj{}, err } @@ -106,7 +111,7 @@ func (d *Misskey) list(dir model.Obj) ([]model.Obj, error) { func (d *Misskey) link(file model.Obj) (*model.Link, error) { var mFile MFile - err := d.request("/files/show", "POST", setBody(map[string]string{"fileId": file.GetID()}), &mFile) + err := d.request("/files/show", http.MethodPost, setBody(map[string]string{"fileId": file.GetID()}), &mFile) if err != nil { return nil, err } @@ -117,7 +122,7 @@ func (d *Misskey) link(file model.Obj) (*model.Link, error) { func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error) { var folder MFolder - err := d.request("/folders/create", "POST", setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) + err := d.request("/folders/create", http.MethodPost, setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) if err != nil { return nil, err } @@ -127,11 +132,11 @@ func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { if srcObj.IsDir() { var folder MFolder - err := d.request("/folders/update", "POST", setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) + err := d.request("/folders/update", http.MethodPost, setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) return mFolder2Object(folder), err } else { var file MFile - err := d.request("/files/update", "POST", setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) + err := d.request("/files/update", http.MethodPost, setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) return mFile2Object(file), err } } @@ -139,11 +144,11 @@ func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { func (d *Misskey) rename(srcObj model.Obj, newName string) (model.Obj, error) { if srcObj.IsDir() { var folder MFolder - err := d.request("/folders/update", "POST", setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) + err := d.request("/folders/update", http.MethodPost, setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) return mFolder2Object(folder), err } else { var file MFile - err := d.request("/files/update", "POST", setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) + err := d.request("/files/update", http.MethodPost, setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) return mFile2Object(file), err } } @@ -171,7 +176,7 @@ func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { if err != nil { return nil, err } - err = d.request("/files/upload-from-url", "POST", setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) + err = d.request("/files/upload-from-url", http.MethodPost, setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) if err != nil { return nil, err } @@ -181,10 +186,10 @@ func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { func (d *Misskey) remove(obj model.Obj) error { if obj.IsDir() { - err := d.request("/folders/delete", "POST", setBody(map[string]string{"folderId": obj.GetID()}), nil) + err := d.request("/folders/delete", http.MethodPost, setBody(map[string]string{"folderId": obj.GetID()}), nil) return err } else { - err := d.request("/files/delete", "POST", setBody(map[string]string{"fileId": obj.GetID()}), nil) + err := d.request("/files/delete", http.MethodPost, setBody(map[string]string{"fileId": obj.GetID()}), nil) return err } } @@ -196,16 +201,24 @@ func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileSt Reader: stream, UpdateProgress: up, }) + + // Build form data, only add folderId if not root folder + formData := map[string]string{ + "name": stream.GetName(), + "comment": "", + "isSensitive": "false", + "force": "false", + } + + folderId := handleFolderId(dstDir) + if folderId != nil { + formData["folderId"] = folderId.(string) + } + req := base.RestyClient.R(). SetContext(ctx). SetFileReader("file", stream.GetName(), reader). - SetFormData(map[string]string{ - "folderId": handleFolderId(dstDir).(string), - "name": stream.GetName(), - "comment": "", - "isSensitive": "false", - "force": "false", - }). + SetFormData(formData). SetResult(&file). SetAuthToken(d.AccessToken) From 998022e38be0de08e709a00f9769e8e22b0eaa81 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sun, 21 Dec 2025 15:23:12 +0800 Subject: [PATCH 579/659] feat(settings): Add `SetToken` endpoint for updating token settings - Introduced `SetToken` handler to allow manual token updates via API - Added `SetTokenReq` struct for request validation --- server/handles/setting.go | 21 ++++++++++++++++++++- server/router.go | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/server/handles/setting.go b/server/handles/setting.go index e0dbb490033..f209b7c5dc5 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -29,9 +29,13 @@ func getRoleOptions() string { return strings.Join(names, ",") } +type SetTokenReq struct { + Token string `json:"token" form:"token" binding:"required"` +} + func ResetToken(c *gin.Context) { token := random.Token() - item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} + item := model.SettingItem{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} if err := op.SaveSettingItem(&item); err != nil { common.ErrorResp(c, err, 500) return @@ -40,6 +44,21 @@ func ResetToken(c *gin.Context) { common.SuccessResp(c, token) } +func SetToken(c *gin.Context) { + var req SetTokenReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + item := model.SettingItem{Key: conf.Token, Value: req.Token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} + if err := op.SaveSettingItem(&item); err != nil { + common.ErrorResp(c, err, 500) + return + } + sign.Instance() + common.SuccessResp(c, req.Token) +} + func GetSetting(c *gin.Context) { key := c.Query("key") keys := c.Query("keys") diff --git a/server/router.go b/server/router.go index 4d79c1fde52..63503838af7 100644 --- a/server/router.go +++ b/server/router.go @@ -154,6 +154,7 @@ func admin(g *gin.RouterGroup) { setting.POST("/save", handles.SaveSettings) setting.POST("/delete", handles.DeleteSetting) setting.POST("/reset_token", handles.ResetToken) + setting.POST("/set_token", handles.SetToken) setting.POST("/set_aria2", handles.SetAria2) setting.POST("/set_qbit", handles.SetQbittorrent) setting.POST("/set_transmission", handles.SetTransmission) From 5f244090708b3dd7f1a6b5cf7170ad71b561b795 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sun, 21 Dec 2025 15:59:26 +0800 Subject: [PATCH 580/659] feat(proxy): Added configurable signature for down proxy URLs - Introduced `DownProxySign` field with default `true` in the `Proxy` struct - Added `BuildDownProxyURL` utility to generate proxy URLs with an optional signature - Updated proxy handling logic across multiple modules to support the new signature configuration --- internal/model/storage.go | 9 +++++---- internal/op/driver.go | 5 +++++ server/common/proxy.go | 9 +++++++++ server/handles/down.go | 7 +------ server/handles/fsread.go | 9 +++++---- server/webdav/webdav.go | 6 +----- 6 files changed, 26 insertions(+), 19 deletions(-) diff --git a/internal/model/storage.go b/internal/model/storage.go index e3c7e1f9731..4d9c062518d 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -28,10 +28,11 @@ type Sort struct { } type Proxy struct { - WebProxy bool `json:"web_proxy"` - WebdavPolicy string `json:"webdav_policy"` - ProxyRange bool `json:"proxy_range"` - DownProxyUrl string `json:"down_proxy_url"` + WebProxy bool `json:"web_proxy"` + WebdavPolicy string `json:"webdav_policy"` + ProxyRange bool `json:"proxy_range"` + DownProxyUrl string `json:"down_proxy_url"` + DownProxySign bool `json:"down_proxy_sign" gorm:"default:true"` } func (s *Storage) GetStorage() *Storage { diff --git a/internal/op/driver.go b/internal/op/driver.go index 41b6f6d42c7..4099fbbf5dd 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -117,6 +117,11 @@ func getMainItems(config driver.Config) []driver.Item { Name: "down_proxy_url", Type: conf.TypeText, }) + items = append(items, driver.Item{ + Name: "down_proxy_sign", + Type: conf.TypeBool, + Default: "true", + }) if config.LocalSort { items = append(items, []driver.Item{{ Name: "order_by", diff --git a/server/common/proxy.go b/server/common/proxy.go index ca7f6325d7d..97bf84efa12 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -13,6 +13,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" + "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" @@ -129,6 +130,14 @@ func ProxyRange(link *model.Link, size int64) { } } +func BuildDownProxyURL(downProxyURL, path string, useSign bool) string { + base := strings.Split(downProxyURL, "\n")[0] + if useSign { + return fmt.Sprintf("%s%s?sign=%s", base, utils.EncodePath(path, true), sign.Sign(path)) + } + return fmt.Sprintf("%s%s", base, utils.EncodePath(path, true)) +} + type InterceptResponseWriter struct { http.ResponseWriter io.Writer diff --git a/server/handles/down.go b/server/handles/down.go index 2c5c2fafc51..59c75530d3b 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -6,14 +6,12 @@ import ( "io" stdpath "path" "strconv" - "strings" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" @@ -62,10 +60,7 @@ func Proxy(c *gin.Context) { if downProxyUrl != "" { _, ok := c.GetQuery("d") if !ok { - URL := fmt.Sprintf("%s%s?sign=%s", - strings.Split(downProxyUrl, "\n")[0], - utils.EncodePath(rawPath, true), - sign.Sign(rawPath)) + URL := common.BuildDownProxyURL(downProxyUrl, rawPath, storage.GetStorage().DownProxySign) c.Redirect(302, URL) return } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 676d64b166e..15dd9f1ce7e 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -340,10 +340,11 @@ func FsGet(c *gin.Context) { query = "?sign=" + sign.Sign(reqPath) } if storage.GetStorage().DownProxyUrl != "" { - rawURL = fmt.Sprintf("%s%s?sign=%s", - strings.Split(storage.GetStorage().DownProxyUrl, "\n")[0], - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)) + rawURL = common.BuildDownProxyURL( + storage.GetStorage().DownProxyUrl, + reqPath, + storage.GetStorage().DownProxySign, + ) } else { rawURL = fmt.Sprintf("%s/p%s%s", common.GetApiUrl(c.Request), diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index 00c0471f743..df1d2045140 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -21,7 +21,6 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" ) @@ -253,10 +252,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) } } else if storage.GetStorage().WebdavProxy() && downProxyUrl != "" { - u := fmt.Sprintf("%s%s?sign=%s", - strings.Split(downProxyUrl, "\n")[0], - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)) + u := common.BuildDownProxyURL(downProxyUrl, reqPath, storage.GetStorage().DownProxySign) w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") http.Redirect(w, r, u, http.StatusFound) } else { From e5662efad3efb8fea8a6057e537af439d4c7997b Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sun, 21 Dec 2025 19:01:18 +0800 Subject: [PATCH 581/659] feat(driver): Enhanced Baidu Netdisk upload logic with dynamic URL retrieval - Added support for dynamically retrieving upload URLs, with fallback in case of failure - Improved token refresh and error handling during uploads - Prevented uploading of empty files and added error message for invalid operations - Refactored large file uploads with segment-level progress and retry handling logic - Introduced constants for upload settings, enabling better configurability - Improved logs to include the driver name for better debugging context --- drivers/baidu_netdisk/driver.go | 220 ++++++++++++++++++++++---------- drivers/baidu_netdisk/meta.go | 12 ++ drivers/baidu_netdisk/types.go | 29 +++++ drivers/baidu_netdisk/util.go | 91 ++++++++++++- 4 files changed, 276 insertions(+), 76 deletions(-) diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index c33e0b32b05..64539dc7d89 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -5,23 +5,26 @@ import ( "crypto/md5" "encoding/hex" "errors" + "fmt" "io" "net/url" "os" stdpath "path" "strconv" + "strings" + "sync" "time" - "golang.org/x/sync/semaphore" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/singleflight" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -31,8 +34,16 @@ type BaiduNetdisk struct { uploadThread int vipType int // 会员类型，0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) + + upClient *resty.Client // 上传文件使用的http客户端 + uploadUrlG singleflight.Group[string] + uploadUrlMu sync.RWMutex + uploadUrl string // 上传域名 + uploadUrlUpdateTime time.Time // 上传域名上次更新时间 } +var ErrUploadIDExpired = errors.New("uploadid expired") + func (d *BaiduNetdisk) Config() driver.Config { return config } @@ -42,19 +53,26 @@ func (d *BaiduNetdisk) GetAddition() driver.Additional { } func (d *BaiduNetdisk) Init(ctx context.Context) error { + d.upClient = base.NewRestyClient(). + SetTimeout(UPLOAD_TIMEOUT). + SetRetryCount(UPLOAD_RETRY_COUNT). + SetRetryWaitTime(UPLOAD_RETRY_WAIT_TIME). + SetRetryMaxWaitTime(UPLOAD_RETRY_MAX_WAIT_TIME) d.uploadThread, _ = strconv.Atoi(d.UploadThread) - if d.uploadThread < 1 || d.uploadThread > 32 { - d.uploadThread, d.UploadThread = 3, "3" + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" } if _, err := url.Parse(d.UploadAPI); d.UploadAPI == "" || err != nil { - d.UploadAPI = "https://d.pcs.baidu.com" + d.UploadAPI = UPLOAD_FALLBACK_API } res, err := d.get("/xpan/nas", map[string]string{ "method": "uinfo", }, nil) - log.Debugf("[baidu] get uinfo: %s", string(res)) + log.Debugf("[baidu_netdisk] get uinfo: %s", string(res)) if err != nil { return err } @@ -181,6 +199,11 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo // **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间，而不是文件时间。 // 而实际上云盘存储的时间是文件时间，所以此处需要覆盖时间，保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // 百度网盘不允许上传空文件 + if stream.GetSize() < 1 { + return nil, ErrBaiduEmptyFilesNotAllowed + } + // rapid upload if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil @@ -245,7 +268,7 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F } if tmpF != nil { if written != streamSize { - return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) } _, err = tmpF.Seek(0, io.SeekStart) if err != nil { @@ -259,82 +282,97 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 预上传 - // 尝试获取之前的进度 + // step.1 尝试读取已保存进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - params := map[string]string{ - "method": "precreate", - } - form := map[string]string{ - "path": path, - "size": strconv.FormatInt(streamSize, 10), - "isdir": "0", - "autoinit": "1", - "rtype": "3", - "block_list": blockListStr, - "content-md5": contentMd5, - "slice-md5": sliceMd5, - } - joinTime(form, ctime, mtime) - - log.Debugf("[baidu_netdisk] precreate data: %s", form) - _, err = d.postForm("/xpan/file", params, form, &precreateResp) + // 没有进度，走预上传 + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) if err != nil { return nil, err } - log.Debugf("%+v", precreateResp) if precreateResp.ReturnType == 2 { //rapid upload, since got md5 match from baidu server // 修复时间，具体原因见 Put 方法注释的 **注意** - precreateResp.File.Ctime = ctime - precreateResp.File.Mtime = mtime return fileToObj(precreateResp.File), nil } } + // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(1), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - sem := semaphore.NewWeighted(3) - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + // 获取上传域名 + uploadUrl := d.getUploadUrl(path, precreateResp.Uploadid) + // 并发上传 + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + cacheReaderAt, okReaderAt := cache.(io.ReaderAt) + if !okReaderAt { + return nil, fmt.Errorf("cache object must implement io.ReaderAt interface for upload operations") } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*sliceSize, sliceSize - if partseq+1 == count { - byteSize = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - if err = sem.Acquire(ctx, 1); err != nil { - return err - } - defer sem.Release(1) - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), + totalParts := len(precreateResp.BlockList) + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue } - err := d.uploadSlice(ctx, params, stream.GetName(), - driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) - if err != nil { - return err + + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize } - up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) - precreateResp.BlockList[i] = -1 - return nil - }) - } - if err = threadG.Wait(); err != nil { - // 如果属于用户主动取消，则保存上传进度 + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": precreateResp.Uploadid, + "partseq": strconv.Itoa(partseq), + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + err := d.uploadSlice(ctx, uploadUrl, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 当前goroutine还没退出，+1才是真正成功的数量 + success := threadG.Success() + 1 + progress := float64(success) * 100 / float64(totalParts) + up(progress) + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + // 保存进度（所有错误都会保存） + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + if errors.Is(err, context.Canceled) { - precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") + // 重新 precreate（所有分片都要重传） + newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime) + if err2 != nil { + return nil, err2 + } + if newPre.ReturnType == 2 { + return fileToObj(newPre.File), nil + } + precreateResp = newPre + // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + continue uploadLoop } return nil, err } @@ -348,23 +386,67 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F // 修复时间，具体原因见 Put 方法注释的 **注意** newFile.Ctime = ctime newFile.Mtime = mtime + // 上传成功清理进度 + base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5) return fileToObj(newFile), nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { - res, err := base.RestyClient.R(). +// precreate 执行预上传操作，支持首次上传和 uploadid 过期重试 +func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + params := map[string]string{"method": "precreate"} + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + } + + // 只有在首次上传时才包含 content-md5 和 slice-md5 + if contentMd5 != "" && sliceMd5 != "" { + form["content-md5"] = contentMd5 + form["slice-md5"] = sliceMd5 + } + + joinTime(form, ctime, mtime) + + var precreateResp PrecreateResp + _, err := d.postForm("/xpan/file", params, form, &precreateResp) + if err != nil { + return nil, err + } + + // 修复时间，具体原因见 Put 方法注释的 **注意** + if precreateResp.ReturnType == 2 { + precreateResp.File.Ctime = ctime + precreateResp.File.Mtime = mtime + } + + return &precreateResp, nil +} + +func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file io.Reader) error { + res, err := d.upClient.R(). SetContext(ctx). SetQueryParams(params). SetFileReader("file", fileName, file). - Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") + Post(uploadUrl + "/rest/2.0/pcs/superfile2") if err != nil { return err } log.Debugln(res.RawResponse.Status + res.String()) errCode := utils.Json.Get(res.Body(), "error_code").ToInt() errNo := utils.Json.Get(res.Body(), "errno").ToInt() + respStr := res.String() + lower := strings.ToLower(respStr) + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String()) + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", res.String()) } return nil } diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index 7577c747fe3..b75650ef063 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -1,6 +1,8 @@ package baidu_netdisk import ( + "time" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/op" ) @@ -17,11 +19,21 @@ type Addition struct { AccessToken string UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + UseDynamicUploadAPI bool `json:"use_dynamic_upload_api" default:"true" help:"dynamically get upload api domain, when enabled, the 'Upload API' setting will be used as a fallback if failed to get"` CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` } +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 + UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) + UPLOAD_TIMEOUT = time.Minute * 30 // 上传请求超时时间 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second * 1 + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 +) + var config = driver.Config{ Name: "BaiduNetdisk", DefaultRoot: "/", diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index ed9b09df8ee..a158956d09b 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,6 +1,7 @@ package baidu_netdisk import ( + "errors" "path" "strconv" "time" @@ -9,6 +10,10 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" ) +var ( + ErrBaiduEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu netdisk") +) + type TokenErrResp struct { ErrorDescription string `json:"error_description"` Error string `json:"error"` @@ -189,3 +194,27 @@ type PrecreateResp struct { // return_type=2 File File `json:"info"` } + +type UploadServerResp struct { + BakServer []any `json:"bak_server"` + BakServers []struct { + Server string `json:"server"` + } `json:"bak_servers"` + ClientIP string `json:"client_ip"` + ErrorCode int `json:"error_code"` + ErrorMsg string `json:"error_msg"` + Expire int `json:"expire"` + Host string `json:"host"` + Newno string `json:"newno"` + QuicServer []any `json:"quic_server"` + QuicServers []struct { + Server string `json:"server"` + } `json:"quic_servers"` + RequestID int64 `json:"request_id"` + Server []any `json:"server"` + ServerTime int `json:"server_time"` + Servers []struct { + Server string `json:"server"` + } `json:"servers"` + Sl int `json:"sl"` +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index 1249b3f470f..c5a7334315d 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -73,7 +73,7 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno != 0 { if utils.SliceContains([]int{111, -6}, errno) { - log.Info("refreshing baidu_netdisk token.") + log.Info("[baidu_netdisk] refreshing baidu_netdisk token.") err2 := d.refreshToken() if err2 != nil { return retry.Unrecoverable(err2) @@ -284,10 +284,10 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { // 非会员固定为 4MB if d.vipType == 0 { if d.CustomUploadPartSize != 0 { - log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") + log.Warnf("[baidu_netdisk] CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") } if filesize > MaxSliceNum*DefaultSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return DefaultSliceSize @@ -295,17 +295,17 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { if d.CustomUploadPartSize != 0 { if d.CustomUploadPartSize < DefaultSliceSize { - log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) return DefaultSliceSize } if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) return VipSliceSize } if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize { - log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) return SVipSliceSize } @@ -335,12 +335,89 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { } if filesize > MaxSliceNum*maxSliceSize { - log.Warnf("File size(%d) is too large, may cause upload failure", filesize) + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) } return maxSliceSize } +// getUploadUrl 从开放平台获取上传域名/地址，并发请求会被合并，结果会被缓存1h。 +// 如果获取失败，则返回 Upload API设置项。 +func (d *BaiduNetdisk) getUploadUrl(path, uploadId string) string { + if !d.UseDynamicUploadAPI { + return d.UploadAPI + } + getCachedUrlFunc := func() string { + d.uploadUrlMu.RLock() + defer d.uploadUrlMu.RUnlock() + if d.uploadUrl != "" && time.Since(d.uploadUrlUpdateTime) < UPLOAD_URL_EXPIRE_TIME { + return d.uploadUrl + } + return "" + } + // 检查地址缓存 + if uploadUrl := getCachedUrlFunc(); uploadUrl != "" { + return uploadUrl + } + + uploadUrlGetFunc := func() (string, error) { + // 双重检查缓存 + if uploadUrl := getCachedUrlFunc(); uploadUrl != "" { + return uploadUrl, nil + } + + uploadUrl, err := d.requestForUploadUrl(path, uploadId) + if err != nil { + return "", err + } + + d.uploadUrlMu.Lock() + defer d.uploadUrlMu.Unlock() + d.uploadUrl = uploadUrl + d.uploadUrlUpdateTime = time.Now() + return uploadUrl, nil + } + + uploadUrl, err, _ := d.uploadUrlG.Do("", uploadUrlGetFunc) + if err != nil { + fallback := d.UploadAPI + log.Warnf("[baidu_netdisk] get upload URL failed (%v), will use fallback URL: %s", err, fallback) + return fallback + } + return uploadUrl +} + +// requestForUploadUrl 请求获取上传地址。 +// 实测此接口不需要认证，传method和upload_version就行，不过还是按文档规范调用。 +// https://pan.baidu.com/union/doc/Mlvw5hfnr +func (d *BaiduNetdisk) requestForUploadUrl(path, uploadId string) (string, error) { + params := map[string]string{ + "method": "locateupload", + "appid": "250528", + "path": path, + "uploadid": uploadId, + "upload_version": "2.0", + } + apiUrl := "https://d.pcs.baidu.com/rest/2.0/pcs/file" + var resp UploadServerResp + _, err := d.request(apiUrl, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(params) + }, &resp) + if err != nil { + return "", err + } + var uploadUrl string + if len(resp.Servers) > 0 { + uploadUrl = resp.Servers[0].Server + } else if len(resp.BakServers) > 0 { + uploadUrl = resp.BakServers[0].Server + } + if uploadUrl == "" { + return "", errors.New("upload URL is empty") + } + return uploadUrl, nil +} + // func encodeURIComponent(str string) string { // r := url.QueryEscape(str) // r = strings.ReplaceAll(r, "+", "%20") From 66a52b820dede9ccbc55eee4e1f300d9e11aec22 Mon Sep 17 00:00:00 2001 From: qianshi Date: Fri, 30 Jan 2026 14:55:09 +0800 Subject: [PATCH 582/659] fix(tls): harden defaults and warn on insecure mode default TLS verification to enabled to avoid insecure skips use the global TLS skip flag for WebDAV and LDAP transports log a clear warning when insecure TLS is explicitly enabled --- drivers/webdav/meta.go | 1 - drivers/webdav/util.go | 3 ++- internal/bootstrap/config.go | 9 +++++++++ internal/conf/config.go | 2 +- server/handles/ldap_login.go | 2 +- 5 files changed, 13 insertions(+), 4 deletions(-) diff --git a/drivers/webdav/meta.go b/drivers/webdav/meta.go index 2294d482a6e..d66499bc3f9 100644 --- a/drivers/webdav/meta.go +++ b/drivers/webdav/meta.go @@ -11,7 +11,6 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootPath - TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false"` } var config = driver.Config{ diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 23dc909ff88..dfd6e5b2457 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -6,6 +6,7 @@ import ( "net/http/cookiejar" "github.com/alist-org/alist/v3/drivers/webdav/odrvcookie" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/gowebdav" ) @@ -20,7 +21,7 @@ func (d *WebDav) setClient() error { c := gowebdav.NewClient(d.Address, d.Username, d.Password) c.SetTransport(&http.Transport{ Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: d.TlsInsecureSkipVerify}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, }) if d.isSharepoint() { cookie, err := odrvcookie.GetCookie(d.Username, d.Password, d.Address) diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index db3e20942b6..ac36059a076 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -70,6 +70,15 @@ func InitConfig() { if !conf.Conf.Force { confFromEnv() } + if conf.Conf.TlsInsecureSkipVerify { + log.Warn("SECURITY WARNING / 安全警告:") + log.Warn("TLS certificate verification is disabled.") + log.Warn("TLS 证书校验已被禁用。") + log.Warn("This exposes all storage traffic to MitM attacks and may leak credentials or allow data tampering.") + log.Warn("这会使所有存储通信暴露于中间人攻击（MitM），可能导致凭据泄露和数据被篡改。") + log.Warn("Only use this setting if you fully understand the risks.") + log.Warn("仅在你完全理解风险的情况下使用该配置。") + } // convert abs path if !filepath.IsAbs(conf.Conf.TempDir) { absPath, err := filepath.Abs(conf.Conf.TempDir) diff --git a/internal/conf/config.go b/internal/conf/config.go index cf7cde0bcf0..15bd7feba0a 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -156,7 +156,7 @@ func DefaultConfig() *Config { }, MaxConnections: 0, MaxConcurrency: 64, - TlsInsecureSkipVerify: true, + TlsInsecureSkipVerify: false, Tasks: TasksConfig{ Download: TaskConfig{ Workers: 5, diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go index 2a85dc03d0b..fb8417b68b5 100644 --- a/server/handles/ldap_login.go +++ b/server/handles/ldap_login.go @@ -150,7 +150,7 @@ func dial(ldapServer string) (*ldap.Conn, error) { } if tlsEnabled { - return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: true}) + return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) } else { return ldap.Dial("tcp", ldapServer) } From 1328690016c3ffdc06e33949d3595113b59645eb Mon Sep 17 00:00:00 2001 From: qianshi Date: Fri, 30 Jan 2026 16:04:25 +0800 Subject: [PATCH 583/659] fix(fs): block path traversal in handlers --- internal/errs/operate.go | 1 + pkg/utils/path.go | 35 ++++++++++++++++++++++++ pkg/utils/path_test.go | 46 +++++++++++++++++++++++++++++++ server/handles/archive.go | 9 +++++-- server/handles/fsbatch.go | 21 +++++++++++++-- server/handles/fsmanage.go | 55 +++++++++++++++++++++++++++++++++----- 6 files changed, 157 insertions(+), 10 deletions(-) diff --git a/internal/errs/operate.go b/internal/errs/operate.go index 92fbd6a1a49..d2df47ddb9c 100644 --- a/internal/errs/operate.go +++ b/internal/errs/operate.go @@ -4,4 +4,5 @@ import "errors" var ( PermissionDenied = errors.New("permission denied") + InvalidName = errors.New("invalid file name") ) diff --git a/pkg/utils/path.go b/pkg/utils/path.go index 6f3a55fc3d3..fe4ff2fd96a 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -101,3 +101,38 @@ func JoinBasePath(basePath, reqPath string) (string, error) { func GetFullPath(mountPath, path string) string { return stdpath.Join(GetActualMountPath(mountPath), path) } + +// ValidateNameComponent validates a single path component. +// It rejects empty names, dot segments, separators, ".." sequences, and NUL bytes. +func ValidateNameComponent(name string) error { + if name == "" { + return errs.InvalidName + } + if name == "." || name == ".." { + return errs.InvalidName + } + if strings.Contains(name, "/") || strings.Contains(name, "\\") { + return errs.InvalidName + } + if strings.Contains(name, "..") { + return errs.InvalidName + } + if strings.ContainsRune(name, 0) { + return errs.InvalidName + } + return nil +} + +// JoinUnderBase safely joins baseDir with a single name component and ensures the +// result stays under baseDir after normalization. +func JoinUnderBase(baseDir, name string) (string, error) { + if err := ValidateNameComponent(name); err != nil { + return "", err + } + base := FixAndCleanPath(baseDir) + joined := FixAndCleanPath(stdpath.Join(base, name)) + if !IsSubPath(base, joined) { + return "", errs.InvalidName + } + return joined, nil +} diff --git a/pkg/utils/path_test.go b/pkg/utils/path_test.go index f42f2f8bb5d..def286b8038 100644 --- a/pkg/utils/path_test.go +++ b/pkg/utils/path_test.go @@ -20,3 +20,49 @@ func TestFixAndCleanPath(t *testing.T) { } } } + +func TestValidateNameComponent(t *testing.T) { + validNames := []string{ + "file.txt", + "abc", + "file_name-1", + } + for _, name := range validNames { + if err := ValidateNameComponent(name); err != nil { + t.Fatalf("expected valid name %q, got error: %v", name, err) + } + } + + invalidNames := []string{ + "", + ".", + "..", + "a/b", + `a\b`, + "a..b", + string([]byte{'a', 0, 'b'}), + } + for _, name := range invalidNames { + if err := ValidateNameComponent(name); err == nil { + t.Fatalf("expected invalid name %q to be rejected", name) + } + } +} + +func TestJoinUnderBase(t *testing.T) { + base := "/lanzou-y/shared/test1" + out, err := JoinUnderBase(base, "file.txt") + if err != nil { + t.Fatalf("expected join success, got error: %v", err) + } + if out != "/lanzou-y/shared/test1/file.txt" { + t.Fatalf("unexpected join result: %s", out) + } + + if _, err := JoinUnderBase(base, "../admin/screts.txt"); err == nil { + t.Fatalf("expected traversal to be rejected") + } + if _, err := JoinUnderBase(base, "sub/child"); err == nil { + t.Fatalf("expected nested path to be rejected") + } +} diff --git a/server/handles/archive.go b/server/handles/archive.go index 5787897cc90..844947408be 100644 --- a/server/handles/archive.go +++ b/server/handles/archive.go @@ -254,11 +254,16 @@ func FsArchiveDecompress(c *gin.Context) { return } user := c.MustGet("user").(*model.User) + srcDir, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } srcPaths := make([]string, 0, len(req.Name)) for _, name := range req.Name { - srcPath, err := user.JoinPath(stdpath.Join(req.SrcDir, name)) + srcPath, err := utils.JoinUnderBase(srcDir, name) if err != nil { - common.ErrorResp(c, err, 403) + common.ErrorResp(c, err, 400) return } if !common.CheckPathLimitWithRoles(user, srcPath) { diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 7ff07c6df28..35cec645e1b 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -10,6 +10,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/generic" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" @@ -185,7 +186,15 @@ func FsBatchRename(c *gin.Context) { if renameObject.SrcName == "" || renameObject.NewName == "" { continue } - filePath := fmt.Sprintf("%s/%s", reqPath, renameObject.SrcName) + if err := utils.ValidateNameComponent(renameObject.NewName); err != nil { + common.ErrorResp(c, err, 400) + return + } + filePath, err := utils.JoinUnderBase(reqPath, renameObject.SrcName) + if err != nil { + common.ErrorResp(c, err, 400) + return + } if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { common.ErrorResp(c, err, 500) return @@ -247,8 +256,16 @@ func FsRegexRename(c *gin.Context) { for _, file := range files { if srcRegexp.MatchString(file.GetName()) { - filePath := fmt.Sprintf("%s/%s", reqPath, file.GetName()) + filePath, err := utils.JoinUnderBase(reqPath, file.GetName()) + if err != nil { + common.ErrorResp(c, err, 500) + return + } newFileName := srcRegexp.ReplaceAllString(file.GetName(), req.NewNameRegex) + if err := utils.ValidateNameComponent(newFileName); err != nil { + common.ErrorResp(c, err, 400) + return + } if err := fs.Rename(c, filePath, newFileName); err != nil { common.ErrorResp(c, err, 500) return diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 87be6e4106b..dcb5a7b9a09 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -103,14 +103,29 @@ func FsMove(c *gin.Context) { } if !req.Overwrite { for _, name := range req.Names { - if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + dstPath, err := utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } } } for i, name := range req.Names { - err := fs.Move(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + err = fs.Move(c, srcPath, dstDir, len(req.Names) > i+1) if err != nil { common.ErrorResp(c, err, 500) return @@ -155,7 +170,12 @@ func FsCopy(c *gin.Context) { } if !req.Overwrite { for _, name := range req.Names { - if res, _ := fs.Get(c, stdpath.Join(dstDir, name), &fs.GetArgs{NoLog: true}); res != nil { + dstPath, err := utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) return } @@ -163,7 +183,17 @@ func FsCopy(c *gin.Context) { } var addedTasks []task.TaskExtensionInfo for i, name := range req.Names { - t, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + t, err := fs.Copy(c, srcPath, dstDir, len(req.Names) > i+1) if t != nil { addedTasks = append(addedTasks, t) } @@ -204,8 +234,16 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + if err := utils.ValidateNameComponent(req.Name); err != nil { + common.ErrorResp(c, err, 400) + return + } if !req.Overwrite { - dstPath := stdpath.Join(stdpath.Dir(reqPath), req.Name) + dstPath, err := utils.JoinUnderBase(stdpath.Dir(reqPath), req.Name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } if dstPath != reqPath { if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", req.Name), 403) @@ -251,7 +289,12 @@ func FsRemove(c *gin.Context) { return } for _, name := range req.Names { - err := fs.Remove(c, stdpath.Join(reqDir, name)) + removePath, err := utils.JoinUnderBase(reqDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + err = fs.Remove(c, removePath) if err != nil { common.ErrorResp(c, err, 500) return From c258b93dbf01f8a343dca7130cc24367cf533081 Mon Sep 17 00:00:00 2001 From: qianshi Date: Sat, 31 Jan 2026 15:07:03 +0800 Subject: [PATCH 584/659] fix(archive): prevent Zip Slip in extraction --- internal/archive/archives/archives.go | 46 +++++++-- internal/archive/archives/utils.go | 10 +- internal/archive/rardecode/rardecode.go | 59 ++++++++++-- internal/archive/rardecode/utils.go | 40 ++++---- internal/archive/tool/helper.go | 113 +++++++++++++++++------ internal/archive/tool/securepath.go | 62 +++++++++++++ internal/archive/tool/securepath_test.go | 48 ++++++++++ 7 files changed, 310 insertions(+), 68 deletions(-) create mode 100644 internal/archive/tool/securepath.go create mode 100644 internal/archive/tool/securepath_test.go diff --git a/internal/archive/archives/archives.go b/internal/archive/archives/archives.go index 0a42cd0c512..e3a48b15757 100644 --- a/internal/archive/archives/archives.go +++ b/internal/archive/archives/archives.go @@ -1,10 +1,12 @@ package archives import ( + "fmt" "io" "io/fs" "os" stdpath "path" + "path/filepath" "strings" "github.com/alist-org/alist/v3/internal/archive/tool" @@ -106,7 +108,7 @@ func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args } if stat.IsDir() { isDir = true - outputPath = stdpath.Join(outputPath, stat.Name()) + outputPath = filepath.Join(outputPath, stat.Name()) err = os.Mkdir(outputPath, 0700) if err != nil { return filterPassword(err) @@ -118,18 +120,46 @@ func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args if err != nil { return err } + if p == path { + if d.IsDir() { + return nil + } + } relPath := strings.TrimPrefix(p, path+"/") - dstPath := stdpath.Join(outputPath, relPath) + if relPath == "" || relPath == "." { + if d.IsDir() { + return nil + } + } + dstPath, err := tool.SecureJoin(outputPath, relPath) + if err != nil { + return err + } if d.IsDir() { - err = os.MkdirAll(dstPath, 0700) - } else { - dir := stdpath.Dir(dstPath) - err = decompress(fsys, p, dir, func(_ float64) {}) + return os.MkdirAll(dstPath, 0700) } - return err + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, p) + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + return decompress(fsys, p, dstPath, func(_ float64) {}) }) } else { - err = decompress(fsys, path, outputPath, up) + entryName := stdpath.Base(path) + dstPath, e := tool.SecureJoin(outputPath, entryName) + if e != nil { + return e + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = decompress(fsys, path, dstPath, up) } return filterPassword(err) } diff --git a/internal/archive/archives/utils.go b/internal/archive/archives/utils.go index 2f499a10feb..5249862ce4b 100644 --- a/internal/archive/archives/utils.go +++ b/internal/archive/archives/utils.go @@ -1,12 +1,13 @@ package archives import ( + "fmt" "io" fs2 "io/fs" "os" - stdpath "path" "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" @@ -59,7 +60,7 @@ func filterPassword(err error) error { return err } -func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgress) error { +func decompress(fsys fs2.FS, filePath, dstPath string, up model.UpdateProgress) error { rc, err := fsys.Open(filePath) if err != nil { return err @@ -69,7 +70,10 @@ func decompress(fsys fs2.FS, filePath, targetPath string, up model.UpdateProgres if err != nil { return err } - f, err := os.OpenFile(stdpath.Join(targetPath, stat.Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if !stat.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, filePath) + } + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/rardecode/rardecode.go b/internal/archive/rardecode/rardecode.go index cd31d1a40e0..2848c704bee 100644 --- a/internal/archive/rardecode/rardecode.go +++ b/internal/archive/rardecode/rardecode.go @@ -1,15 +1,18 @@ package rardecode import ( + "fmt" + "io" + "os" + stdpath "path" + "path/filepath" + "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" "github.com/nwaples/rardecode/v2" - "io" - "os" - stdpath "path" - "strings" ) type RarDecoder struct{} @@ -85,7 +88,11 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg if header.IsDir { name = name + "/" } - err = decompress(reader, header, name, outputPath) + dstPath, e := tool.SecureJoin(outputPath, name) + if e != nil { + return e + } + err = decompress(reader, header, dstPath) if err != nil { return err } @@ -94,6 +101,7 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg innerPath := strings.TrimPrefix(args.InnerPath, "/") innerBase := stdpath.Base(innerPath) createdBaseDir := false + var baseDirPath string for { var header *rardecode.FileHeader header, err = reader.Next() @@ -108,22 +116,55 @@ func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, arg name = name + "/" } if name == innerPath { - err = _decompress(reader, header, outputPath, up) + if header.IsDir { + if !createdBaseDir { + baseDirPath, err = tool.SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + if err = os.MkdirAll(baseDirPath, 0700); err != nil { + return err + } + createdBaseDir = true + } + continue + } + if !header.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, header.Name) + } + dstPath, e := tool.SecureJoin(outputPath, stdpath.Base(innerPath)) + if e != nil { + return e + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(reader, header, dstPath, up) if err != nil { return err } break } else if strings.HasPrefix(name, innerPath+"/") { - targetPath := stdpath.Join(outputPath, innerBase) if !createdBaseDir { - err = os.Mkdir(targetPath, 0700) + baseDirPath, err = tool.SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + err = os.MkdirAll(baseDirPath, 0700) if err != nil { return err } createdBaseDir = true } restPath := strings.TrimPrefix(name, innerPath+"/") - err = decompress(reader, header, restPath, targetPath) + if restPath == "" || restPath == "." { + continue + } + dstPath, e := tool.SecureJoin(baseDirPath, restPath) + if e != nil { + return e + } + err = decompress(reader, header, dstPath) if err != nil { return err } diff --git a/internal/archive/rardecode/utils.go b/internal/archive/rardecode/utils.go index 5790ec58a22..e3612b363df 100644 --- a/internal/archive/rardecode/utils.go +++ b/internal/archive/rardecode/utils.go @@ -2,18 +2,20 @@ package rardecode import ( "fmt" - "github.com/alist-org/alist/v3/internal/archive/tool" - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/nwaples/rardecode/v2" "io" "io/fs" "os" stdpath "path" + "path/filepath" "sort" "strings" "time" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/nwaples/rardecode/v2" ) type VolumeFile struct { @@ -179,27 +181,21 @@ func getReader(ss []*stream.SeekableStream, password string) (*rardecode.Reader, return &rc.Reader, nil } -func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, filePath, outputPath string) error { - targetPath := outputPath - dir, base := stdpath.Split(filePath) - if dir != "" { - targetPath = stdpath.Join(targetPath, dir) - err := os.MkdirAll(targetPath, 0700) - if err != nil { - return err - } +func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, dstPath string) error { + if header.IsDir { + return os.MkdirAll(dstPath, 0700) } - if base != "" { - err := _decompress(reader, header, targetPath, func(_ float64) {}) - if err != nil { - return err - } + if !header.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, header.Name) } - return nil + if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + return _decompress(reader, header, dstPath, func(_ float64) {}) } -func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, targetPath string, up model.UpdateProgress) error { - f, err := os.OpenFile(stdpath.Join(targetPath, stdpath.Base(header.Name)), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) +func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, dstPath string, up model.UpdateProgress) error { + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/tool/helper.go b/internal/archive/tool/helper.go index 20da34467b0..80254b8fd04 100644 --- a/internal/archive/tool/helper.go +++ b/internal/archive/tool/helper.go @@ -1,10 +1,12 @@ package tool import ( + "fmt" "io" "io/fs" "os" stdpath "path" + "path/filepath" "strings" "github.com/alist-org/alist/v3/internal/model" @@ -119,7 +121,30 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode if args.InnerPath == "/" { for i, file := range files { name := file.Name() - err = decompress(file, name, outputPath, args.Password) + info := file.FileInfo() + if info.IsDir() { + var dirPath string + dirPath, err = SecureJoin(outputPath, name) + if err != nil { + return err + } + if err = os.MkdirAll(dirPath, 0700); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(outputPath, name) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, func(_ float64) {}) if err != nil { return err } @@ -129,25 +154,80 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode innerPath := strings.TrimPrefix(args.InnerPath, "/") innerBase := stdpath.Base(innerPath) createdBaseDir := false + var baseDirPath string for _, file := range files { name := file.Name() if name == innerPath { - err = _decompress(file, outputPath, args.Password, up) + info := file.FileInfo() + if info.IsDir() { + if !createdBaseDir { + baseDirPath, err = SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + if err = os.MkdirAll(baseDirPath, 0700); err != nil { + return err + } + createdBaseDir = true + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(outputPath, stdpath.Base(innerPath)) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, up) if err != nil { return err } break } else if strings.HasPrefix(name, innerPath+"/") { - targetPath := stdpath.Join(outputPath, innerBase) if !createdBaseDir { - err = os.Mkdir(targetPath, 0700) + baseDirPath, err = SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + err = os.MkdirAll(baseDirPath, 0700) if err != nil { return err } createdBaseDir = true } restPath := strings.TrimPrefix(name, innerPath+"/") - err = decompress(file, restPath, targetPath, args.Password) + if restPath == "" || restPath == "." { + continue + } + info := file.FileInfo() + if info.IsDir() { + var dirPath string + dirPath, err = SecureJoin(baseDirPath, restPath) + if err != nil { + return err + } + if err = os.MkdirAll(dirPath, 0700); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(baseDirPath, restPath) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, func(_ float64) {}) if err != nil { return err } @@ -157,26 +237,7 @@ func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args mode return nil } -func decompress(file SubFile, filePath, outputPath, password string) error { - targetPath := outputPath - dir, base := stdpath.Split(filePath) - if dir != "" { - targetPath = stdpath.Join(targetPath, dir) - err := os.MkdirAll(targetPath, 0700) - if err != nil { - return err - } - } - if base != "" { - err := _decompress(file, targetPath, password, func(_ float64) {}) - if err != nil { - return err - } - } - return nil -} - -func _decompress(file SubFile, targetPath, password string, up model.UpdateProgress) error { +func _decompress(file SubFile, dstPath, password string, up model.UpdateProgress) error { if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { encrypt.SetPassword(password) } @@ -185,7 +246,7 @@ func _decompress(file SubFile, targetPath, password string, up model.UpdateProgr return err } defer func() { _ = rc.Close() }() - f, err := os.OpenFile(stdpath.Join(targetPath, file.FileInfo().Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } diff --git a/internal/archive/tool/securepath.go b/internal/archive/tool/securepath.go new file mode 100644 index 00000000000..45f5e749d53 --- /dev/null +++ b/internal/archive/tool/securepath.go @@ -0,0 +1,62 @@ +package tool + +import ( + "errors" + "fmt" + "path" + "path/filepath" + "strings" +) + +// ErrArchiveIllegalPath indicates an archive entry path is unsafe for extraction. +var ErrArchiveIllegalPath = errors.New("archive entry has illegal path") + +// SecureJoin returns a safe extraction path for an archive entry. +// It rejects absolute paths, traversal, Windows drive/UNC paths, and NUL bytes. +func SecureJoin(baseDir, entryName string) (string, error) { + if strings.Contains(entryName, "\x00") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + normalized := strings.ReplaceAll(entryName, "\\", "/") + if strings.HasPrefix(normalized, "//") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + cleaned := path.Clean(normalized) + + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + if strings.HasPrefix(cleaned, "/") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + rel := filepath.FromSlash(cleaned) + if filepath.IsAbs(rel) || filepath.VolumeName(rel) != "" { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + if strings.HasPrefix(rel, `\\`) { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + base := filepath.Clean(baseDir) + dst := filepath.Join(base, rel) + + baseAbs, err := filepath.Abs(base) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + dstAbs, err := filepath.Abs(dst) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + + relCheck, err := filepath.Rel(baseAbs, dstAbs) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + if relCheck == ".." || strings.HasPrefix(relCheck, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + return dst, nil +} diff --git a/internal/archive/tool/securepath_test.go b/internal/archive/tool/securepath_test.go new file mode 100644 index 00000000000..78be52ee5d5 --- /dev/null +++ b/internal/archive/tool/securepath_test.go @@ -0,0 +1,48 @@ +package tool + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestSecureJoin(t *testing.T) { + baseDir := t.TempDir() + tests := []struct { + name string + entry string + wantErr bool + }{ + {name: "ok", entry: "a/b/c.txt", wantErr: false}, + {name: "parent", entry: "../evil.txt", wantErr: true}, + {name: "parent-backslash", entry: "..\\evil.txt", wantErr: true}, + {name: "abs", entry: "/tmp/evil.txt", wantErr: true}, + {name: "drive", entry: "C:\\evil.txt", wantErr: true}, + {name: "unc", entry: "\\\\server\\share\\evil.txt", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dst, err := SecureJoin(baseDir, tc.entry) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %q, got nil", tc.entry) + } + if !strings.Contains(err.Error(), tc.entry) { + t.Fatalf("error should include entry name %q, got %q", tc.entry, err.Error()) + } + return + } + if err != nil { + t.Fatalf("unexpected error for %q: %v", tc.entry, err) + } + rel, err := filepath.Rel(baseDir, dst) + if err != nil { + t.Fatalf("Rel failed: %v", err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + t.Fatalf("path escaped baseDir: %q", dst) + } + }) + } +} From 42fce722a90c1fa33695969df2fb54faae089d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Tue, 3 Feb 2026 22:33:50 +0800 Subject: [PATCH 585/659] feat: add doubao new driver (#9416) * feat(driver): add doubao_new driver * fix(driver): improve doubao_new move with csrf * feat(driver): add doubao_new remove task polling --- drivers/all.go | 1 + drivers/doubao_new/driver.go | 600 +++++++++++++++++++++++ drivers/doubao_new/meta.go | 36 ++ drivers/doubao_new/types.go | 182 +++++++ drivers/doubao_new/util.go | 909 +++++++++++++++++++++++++++++++++++ server/handles/down.go | 14 + 6 files changed, 1742 insertions(+) create mode 100644 drivers/doubao_new/driver.go create mode 100644 drivers/doubao_new/meta.go create mode 100644 drivers/doubao_new/types.go create mode 100644 drivers/doubao_new/util.go diff --git a/drivers/all.go b/drivers/all.go index 3eb7e813bcc..3b0435595d2 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -27,6 +27,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" _ "github.com/alist-org/alist/v3/drivers/doubao" + _ "github.com/alist-org/alist/v3/drivers/doubao_new" _ "github.com/alist-org/alist/v3/drivers/doubao_share" _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go new file mode 100644 index 00000000000..7846551790a --- /dev/null +++ b/drivers/doubao_new/driver.go @@ -0,0 +1,600 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type DoubaoNew struct { + model.Storage + Addition + TtLogid string +} + +func (d *DoubaoNew) Config() driver.Config { + return config +} + +func (d *DoubaoNew) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *DoubaoNew) Init(ctx context.Context) error { + // TODO login / refresh token + //op.MustSaveDriverStorage(d) + return nil +} + +func (d *DoubaoNew) Drop(ctx context.Context) error { + return nil +} + +func (d *DoubaoNew) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + nodes, err := d.listAllChildren(ctx, dir.GetID()) + if err != nil { + return nil, err + } + + objs := make([]model.Obj, 0, len(nodes)) + for _, node := range nodes { + size := parseSize(node.Extra.Size) + isFolder := node.Type == 0 + obj := &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: dir.GetID(), + Name: node.Name, + Size: size, + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: isFolder, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + } + objs = append(objs, obj) + } + + return objs, nil +} + +func (d *DoubaoNew) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, ok := file.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + if obj.IsFolder { + return nil, errs.LinkIsDir + } + if args.Type == "preview" || args.Type == "thumb" { + if link, err := d.previewLink(ctx, obj, args); err == nil { + return link, nil + } + } + auth := d.resolveAuthorization() + dpop := d.resolveDpop() + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + + query := url.Values{} + query.Set("authorization", auth) + query.Set("dpop", dpop) + + downloadURL := DownloadBaseURL + "/space/api/box/stream/download/all/" + obj.ObjToken + "/?" + query.Encode() + + headers := http.Header{ + "Referer": []string{"https://www.doubao.com/"}, + "User-Agent": []string{base.UserAgent}, + } + + return &model.Link{ + URL: downloadURL, + Header: headers, + }, nil +} + +func (d *DoubaoNew) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + node, err := d.createFolder(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + return &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: parentDir.GetID(), + Name: node.Name, + Size: parseSize(node.Extra.Size), + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: true, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + }, nil +} + +func (d *DoubaoNew) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if dstDir == nil { + return nil, errors.New("nil destination dir") + } + srcToken := srcObj.GetID() + if srcToken == "" { + if obj, ok := srcObj.(*Object); ok { + srcToken = obj.ObjToken + } + } + if srcToken == "" { + return nil, errors.New("missing source token") + } + if err := d.moveObj(ctx, srcToken, dstDir.GetID()); err != nil { + return nil, err + } + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Path = dstDir.GetID() + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if srcObj.IsDir() { + if err := d.renameFolder(ctx, srcObj.GetID(), newName); err != nil { + return nil, err + } + } else { + fileToken := "" + if obj, ok := srcObj.(*Object); ok { + fileToken = obj.ObjToken + } + if fileToken == "" { + fileToken = srcObj.GetID() + } + if err := d.renameFile(ctx, fileToken, newName); err != nil { + return nil, err + } + } + + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Name = newName + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Remove(ctx context.Context, obj model.Obj) error { + if obj == nil { + return errors.New("nil object") + } + token := obj.GetID() + if token == "" { + if o, ok := obj.(*Object); ok { + token = o.ObjToken + } + } + if token == "" { + return errors.New("missing object token") + } + return d.removeObj(ctx, []string{token}) +} + +func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if file == nil { + return nil, errors.New("nil file") + } + if file.GetSize() <= 0 { + return nil, errors.New("invalid file size") + } + + uploadPrep, err := d.prepareUpload(ctx, file.GetName(), file.GetSize(), dstDir.GetID()) + if err != nil { + return nil, err + } + if uploadPrep.BlockSize <= 0 { + return nil, errors.New("invalid block size from prepare") + } + + tmpFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer tmpFile.Close() + + blockSize := uploadPrep.BlockSize + totalSize := file.GetSize() + numBlocks := int((totalSize + blockSize - 1) / blockSize) + blocks := make([]UploadBlock, 0, numBlocks) + blockMeta := make(map[int]UploadBlock, numBlocks) + + for seq := 0; seq < numBlocks; seq++ { + offset := int64(seq) * blockSize + length := blockSize + if remain := totalSize - offset; remain < length { + length = remain + } + buf := make([]byte, int(length)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + buf = buf[:n] + sum := sha256.Sum256(buf) + hash := base64.StdEncoding.EncodeToString(sum[:]) + checksum := adler32String(buf) + + block := UploadBlock{ + Hash: hash, + Seq: seq, + Size: int64(n), + Checksum: checksum, + IsUploaded: true, + } + blocks = append(blocks, block) + blockMeta[seq] = block + } + + needed, err := d.uploadBlocks(ctx, uploadPrep.UploadID, blocks, "explorer") + if err != nil { + return nil, err + } + + if len(needed.NeededUploadBlocks) > 0 { + sort.Slice(needed.NeededUploadBlocks, func(i, j int) bool { + return needed.NeededUploadBlocks[i].Seq < needed.NeededUploadBlocks[j].Seq + }) + const maxMergeBlockCount = 20 + var ( + groupSeqs []int + groupChecksums []string + groupSizes []int64 + groupRealSize int64 + groupExpectSum int64 + groupBuf bytes.Buffer + uploadedBytes int64 + ) + + flushGroup := func() error { + if len(groupSeqs) == 0 { + return nil + } + data := groupBuf.Bytes() + expectLen := groupExpectSum + if len(data) > 0 { + headLen := 32 + if len(data) < headLen { + headLen = len(data) + } + tailLen := 32 + if len(data) < tailLen { + tailLen = len(data) + } + fmt.Printf("head32 = %x\n", data[:headLen]) + fmt.Printf("tail32 = %x\n", data[len(data)-tailLen:]) + } + if int64(len(data)) != expectLen { + return fmt.Errorf("[doubao_new] merge blocks invalid body len: got=%d expect=%d seqs=%v", len(data), expectLen, groupSeqs) + } + mergeResp, err := d.mergeUploadBlocks(ctx, uploadPrep.UploadID, groupSeqs, groupChecksums, groupSizes, blockSize, data) + if err != nil { + return err + } + if len(mergeResp.SuccessSeqList) != len(groupSeqs) { + return fmt.Errorf("[doubao_new] merge blocks incomplete: %v", mergeResp.SuccessSeqList) + } + success := make(map[int]bool, len(mergeResp.SuccessSeqList)) + for _, seq := range mergeResp.SuccessSeqList { + success[seq] = true + } + for _, seq := range groupSeqs { + if !success[seq] { + return fmt.Errorf("[doubao_new] merge blocks missing seq %d", seq) + } + } + + uploadedBytes += groupRealSize + groupSeqs = groupSeqs[:0] + groupChecksums = groupChecksums[:0] + groupSizes = groupSizes[:0] + groupRealSize = 0 + groupExpectSum = 0 + groupBuf.Reset() + if up != nil { + percent := float64(uploadedBytes) / float64(totalSize) * 100 + up(percent) + } + return nil + } + + for _, item := range needed.NeededUploadBlocks { + if _, ok := blockMeta[item.Seq]; !ok { + return nil, fmt.Errorf("[doubao_new] missing block meta for seq %d", item.Seq) + } + if item.Size <= 0 { + return nil, fmt.Errorf("[doubao_new] invalid block size from needed list: seq=%d size=%d", item.Seq, item.Size) + } + offset := int64(item.Seq) * blockSize + buf := make([]byte, int(item.Size)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + if n != len(buf) { + return nil, fmt.Errorf("[doubao_new] short read: seq=%d want=%d got=%d", item.Seq, len(buf), n) + } + buf = buf[:n] + realAdler := adler32String(buf) + if realAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] block checksum mismatch: seq=%d offset=%d adler32=%s step2=%s", item.Seq, offset, realAdler, item.Checksum) + } + payloadStart := groupBuf.Len() + groupBuf.Write(buf) + payloadEnd := groupBuf.Len() + payloadAdler := adler32String(groupBuf.Bytes()[payloadStart:payloadEnd]) + if payloadAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] payload checksum mismatch: seq=%d start=%d end=%d adler32=%s step2=%s", item.Seq, payloadStart, payloadEnd, payloadAdler, item.Checksum) + } + groupSeqs = append(groupSeqs, item.Seq) + groupChecksums = append(groupChecksums, item.Checksum) + groupSizes = append(groupSizes, item.Size) + groupRealSize += int64(n) + groupExpectSum += item.Size + if len(groupSeqs) >= maxMergeBlockCount { + if err := flushGroup(); err != nil { + return nil, err + } + } + } + + if err := flushGroup(); err != nil { + return nil, err + } + if up != nil { + up(100) + } + } else if up != nil { + up(100) + } + + numBlocksFinish := uploadPrep.NumBlocks + if numBlocksFinish <= 0 { + numBlocksFinish = numBlocks + } + finish, err := d.finishUpload(ctx, uploadPrep.UploadID, numBlocksFinish, "explorer") + if err != nil { + return nil, err + } + + nodeToken := finish.Extra.NodeToken + if nodeToken == "" { + nodeToken = finish.FileToken + } + now := time.Now() + return &Object{ + Object: model.Object{ + ID: nodeToken, + Path: dstDir.GetID(), + Name: file.GetName(), + Size: file.GetSize(), + Modified: now, + Ctime: now, + IsFolder: false, + }, + ObjToken: finish.FileToken, + }, nil +} + +func (d *DoubaoNew) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch args.Method { + case "doubao_preview", "preview": + obj, ok := args.Obj.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errs.NotSupport + } + + imgExt := ".webp" + pageNums := 1 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + if extra.PageNums > 0 { + pageNums = extra.PageNums + } + } + } + + return base.Json{ + "version": info.Version, + "img_ext": imgExt, + "page_nums": pageNums, + }, nil + default: + return nil, errs.NotSupport + } +} + +func (d *DoubaoNew) listAllChildren(ctx context.Context, parentToken string) ([]Node, error) { + nodes := make([]Node, 0, 50) + lastLabel := "" + for page := 0; page < 100; page++ { + data, err := d.listChildren(ctx, parentToken, lastLabel) + if err != nil { + return nil, err + } + + if len(data.NodeList) > 0 { + for _, token := range data.NodeList { + node, ok := data.Entities.Nodes[token] + if !ok { + continue + } + nodes = append(nodes, node) + } + } else { + for _, node := range data.Entities.Nodes { + nodes = append(nodes, node) + } + } + + if !data.HasMore || data.LastLabel == "" || data.LastLabel == lastLabel { + break + } + lastLabel = data.LastLabel + } + + if len(nodes) == 0 { + return nil, nil + } + return nodes, nil +} + +func (d *DoubaoNew) previewLink(ctx context.Context, obj *Object, args model.LinkArgs) (*model.Link, error) { + auth := d.resolveAuthorization() + dpop := d.resolveDpop() + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errors.New("preview not available") + } + + subID := "" + pageIndex := 0 + if args.HttpReq != nil { + query := args.HttpReq.URL.Query() + if v := query.Get("sub_id"); v != "" { + subID = v + } else if v := query.Get("page"); v != "" { + if p, err := strconv.Atoi(v); err == nil && p >= 0 { + pageIndex = p + } + } + } + if subID == "" { + imgExt := ".webp" + pageNums := 0 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + pageNums = extra.PageNums + } + } + if pageNums > 0 && pageIndex >= pageNums { + pageIndex = pageNums - 1 + } + subID = fmt.Sprintf("img_%d%s", pageIndex, imgExt) + } + + query := url.Values{} + query.Set("preview_type", "22") + query.Set("sub_id", subID) + if info.Version != "" { + query.Set("version", info.Version) + } + previewURL := fmt.Sprintf("%s/space/api/box/stream/download/preview_sub/%s?%s", BaseURL, obj.ObjToken, query.Encode()) + + headers := http.Header{ + "Referer": []string{"https://www.doubao.com/"}, + "User-Agent": []string{base.UserAgent}, + "Authorization": []string{auth}, + "Dpop": []string{dpop}, + } + + return &model.Link{ + URL: previewURL, + Header: headers, + }, nil +} + +func parseSize(size string) int64 { + if size == "" { + return 0 + } + val, err := strconv.ParseInt(size, 10, 64) + if err != nil { + return 0 + } + return val +} + +var _ driver.Driver = (*DoubaoNew)(nil) diff --git a/drivers/doubao_new/meta.go b/drivers/doubao_new/meta.go new file mode 100644 index 00000000000..5a8050a2653 --- /dev/null +++ b/drivers/doubao_new/meta.go @@ -0,0 +1,36 @@ +package doubao_new + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + Authorization string `json:"authorization" help:"DPoP access token (Authorization header value); optional if present in cookie"` + Dpop string `json:"dpop" help:"DPoP header value; optional if present in cookie"` + Cookie string `json:"cookie" help:"Optional cookie; only used to extract authorization/dpop tokens"` + Debug bool `json:"debug" help:"Enable debug logs for upload"` +} + +var config = driver.Config{ + Name: "DoubaoNew", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &DoubaoNew{} + }) +} diff --git a/drivers/doubao_new/types.go b/drivers/doubao_new/types.go new file mode 100644 index 00000000000..ea8acfc5b18 --- /dev/null +++ b/drivers/doubao_new/types.go @@ -0,0 +1,182 @@ +package doubao_new + +import "github.com/alist-org/alist/v3/internal/model" + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` +} + +type ListResp struct { + BaseResp + Data ListData `json:"data"` +} + +type ListData struct { + HasMore bool `json:"has_more"` + LastLabel string `json:"last_label"` + NodeList []string `json:"node_list"` + Entities struct { + Nodes map[string]Node `json:"nodes"` + Users map[string]User `json:"users"` + } `json:"entities"` +} + +type Node struct { + Token string `json:"token"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` + Name string `json:"name"` + Type int `json:"type"` + NodeType int `json:"node_type"` + OwnerID string `json:"owner_id"` + EditUID string `json:"edit_uid"` + CreateTime int64 `json:"create_time"` + EditTime int64 `json:"edit_time"` + URL string `json:"url"` + Extra struct { + Size string `json:"size"` + } `json:"extra"` +} + +type User struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type Object struct { + model.Object + ObjToken string + NodeType int + ObjType int + URL string +} + +type CreateFolderResp struct { + BaseResp + Data struct { + Entities struct { + Nodes map[string]Node `json:"nodes"` + } `json:"entities"` + NodeList []string `json:"node_list"` + } `json:"data"` +} + +type FileInfoResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileInfo `json:"data"` +} + +type FileInfo struct { + Name string `json:"name"` + NumBlocks int `json:"num_blocks"` + Version string `json:"version"` + MimeType string `json:"mime_type"` + MountPoint string `json:"mount_point"` + PreviewMeta PreviewMeta `json:"preview_meta"` +} + +type PreviewMeta struct { + Data map[string]PreviewMetaEntry `json:"data"` +} + +type PreviewMetaEntry struct { + Status int `json:"status"` + Extra string `json:"extra"` + PreviewFileSize int64 `json:"preview_file_size"` +} + +type PreviewImageExtra struct { + ImgExt string `json:"img_ext"` + PageNums int `json:"page_nums"` +} + +type UserStorageResp struct { + BaseResp + Data UserStorageData `json:"data"` +} + +type UserStorageData struct { + ShowSizeLimit bool `json:"show_size_limit"` + TotalSizeLimitBytes int64 `json:"total_size_limit_bytes"` + UsedSizeBytes int64 `json:"used_size_bytes"` +} + +type UploadPrepareResp struct { + BaseResp + Data UploadPrepareData `json:"data"` +} + +type UploadPrepareData struct { + BlockSize int64 `json:"block_size"` + NumBlocks int `json:"num_blocks"` + OptionBlockSize int64 `json:"option_block_size"` + DedupeSupport bool `json:"dedupe_support"` + UploadID string `json:"upload_id"` +} + +type UploadBlock struct { + Hash string `json:"hash"` + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + IsUploaded bool `json:"isUploaded"` +} + +type UploadBlocksResp struct { + BaseResp + Data UploadBlocksData `json:"data"` +} + +type UploadBlocksData struct { + NeededUploadBlocks []UploadBlockNeed `json:"needed_upload_blocks"` +} + +type UploadBlockNeed struct { + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + Hash string `json:"hash"` +} + +type UploadMergeResp struct { + BaseResp + Data UploadMergeData `json:"data"` +} + +type UploadMergeData struct { + SuccessSeqList []int `json:"success_seq_list"` +} + +type UploadFinishResp struct { + BaseResp + Data UploadFinishData `json:"data"` +} + +type UploadFinishData struct { + Version string `json:"version"` + DataVersion string `json:"data_version"` + Extra struct { + NodeToken string `json:"node_token"` + } `json:"extra"` + FileToken string `json:"file_token"` +} + +type RemoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + } `json:"data"` +} + +type TaskStatusResp struct { + BaseResp + Data TaskStatusData `json:"data"` +} + +type TaskStatusData struct { + IsFinish bool `json:"is_finish"` + IsFail bool `json:"is_fail"` +} diff --git a/drivers/doubao_new/util.go b/drivers/doubao_new/util.go new file mode 100644 index 00000000000..5c21ca090db --- /dev/null +++ b/drivers/doubao_new/util.go @@ -0,0 +1,909 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "hash/adler32" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" +) + +const ( + BaseURL = "https://my.feishu.cn" + DownloadBaseURL = "https://internal-api-drive-stream.feishu.cn" +) + +var defaultObjTypes = []string{"124", "0", "12", "30", "123", "22"} + +func (d *DoubaoNew) request(ctx context.Context, path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, BaseURL+path) + if err != nil { + return nil, err + } + if res != nil { + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + } + + body := res.Body() + var common BaseResp + if err = json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return body, fmt.Errorf(msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return body, fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func getCookieValue(cookie, name string) string { + parts := strings.Split(cookie, ";") + prefix := name + "=" + for _, part := range parts { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, prefix) { + return strings.TrimPrefix(part, prefix) + } + } + return "" +} + +func adler32String(data []byte) string { + sum := adler32.Checksum(data) + return strconv.FormatUint(uint64(sum), 10) +} + +func buildCommaHeader(items []string) string { + return strings.Join(items, ",") +} + +func joinIntComma(items []int) string { + if len(items) == 0 { + return "" + } + var sb strings.Builder + for i, v := range items { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(strconv.Itoa(v)) + } + return sb.String() +} + +func previewList(items []string, n int) string { + if n <= 0 || len(items) == 0 { + return "" + } + if len(items) < n { + n = len(items) + } + return strings.Join(items[:n], ",") +} + +func (d *DoubaoNew) resolveAuthorization() string { + auth := strings.TrimSpace(d.Authorization) + if auth == "" && d.Cookie != "" { + if token := getCookieValue(d.Cookie, "LARK_SUITE_ACCESS_TOKEN"); token != "" { + auth = token + } + } + if auth == "" { + return "" + } + if !strings.HasPrefix(auth, "DPoP ") && !strings.HasPrefix(auth, "dpop ") { + auth = "DPoP " + auth + } + return auth +} + +func (d *DoubaoNew) resolveDpop() string { + dpop := strings.TrimSpace(d.Dpop) + if dpop == "" && d.Cookie != "" { + dpop = getCookieValue(d.Cookie, "LARK_SUITE_DPOP") + } + return dpop +} + +func (d *DoubaoNew) listChildren(ctx context.Context, parentToken string, lastLabel string) (ListData, error) { + var resp ListResp + _, err := d.request(ctx, "/space/api/explorer/doubao/children/list/", http.MethodGet, func(req *resty.Request) { + values := url.Values{} + for _, t := range defaultObjTypes { + values.Add("obj_type", t) + } + values.Set("length", "50") + values.Set("rank", "0") + values.Set("asc", "0") + values.Set("min_length", "40") + values.Set("thumbnail_width", "1028") + values.Set("thumbnail_height", "1028") + values.Set("thumbnail_policy", "4") + if parentToken != "" { + values.Set("token", parentToken) + } + if lastLabel != "" { + values.Set("last_label", lastLabel) + } + req.SetQueryParamsFromValues(values) + }, &resp) + if err != nil { + return ListData{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) getFileInfo(ctx context.Context, fileToken string) (FileInfo, error) { + var resp FileInfoResp + _, err := d.request(ctx, "/space/api/box/file/info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "caller": "explorer", + "file_token": fileToken, + "mount_point": "explorer", + "option_params": []string{"preview_meta", "check_cipher"}, + }) + }, &resp) + if err != nil { + return FileInfo{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) createFolder(ctx context.Context, parentToken, name string) (Node, error) { + data := url.Values{} + data.Set("name", name) + data.Set("source", "0") + if parentToken != "" { + data.Set("parent_token", parentToken) + } + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/create/folder/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return Node{}, err + } + if err := decodeBaseResp(body, res); err != nil { + return Node{}, err + } + + var resp CreateFolderResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return Node{}, fmt.Errorf(msg) + } + + var node Node + if len(resp.Data.NodeList) > 0 { + if n, ok := resp.Data.Entities.Nodes[resp.Data.NodeList[0]]; ok { + node = n + } + } + if node.Token == "" { + for _, n := range resp.Data.Entities.Nodes { + node = n + break + } + } + if node.Token == "" && node.ObjToken == "" && node.NodeToken == "" { + return Node{}, fmt.Errorf("[doubao_new] create folder failed: empty response") + } + if node.NodeToken == "" { + if node.Token != "" { + node.NodeToken = node.Token + } else if node.ObjToken != "" { + node.NodeToken = node.ObjToken + } + } + if node.ObjToken == "" && node.Token != "" { + node.ObjToken = node.Token + } + return node, nil +} + +func (d *DoubaoNew) renameFolder(ctx context.Context, token, name string) error { + if token == "" { + return fmt.Errorf("[doubao_new] rename folder missing token") + } + data := url.Values{} + data.Set("token", token) + data.Set("name", name) + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/rename/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func isCsrfTokenError(body []byte, res *resty.Response) bool { + if len(body) == 0 { + return false + } + if strings.Contains(strings.ToLower(string(body)), "csrf token error") { + return true + } + if res != nil && res.StatusCode() == http.StatusForbidden { + return true + } + return false +} + +func doRequestWithCsrf(doRequest func(csrfToken string) (*resty.Response, []byte, error)) (*resty.Response, []byte, error) { + res, body, err := doRequest("") + if err != nil { + return res, body, err + } + if isCsrfTokenError(body, res) { + csrfToken := extractCsrfTokenFromResponse(res) + if csrfToken != "" { + return doRequest(csrfToken) + } + } + return res, body, err +} + +func extractCsrfTokenFromResponse(res *resty.Response) string { + if res == nil || res.Request == nil { + return "" + } + if res.Request.RawRequest != nil { + if csrf := getCookieValue(res.Request.RawRequest.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + } + if csrf := getCookieValue(res.Request.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + for _, c := range res.Cookies() { + if c.Name == "_csrf_token" { + return c.Value + } + } + return "" +} + +func decodeBaseResp(body []byte, res *resty.Response) error { + var common BaseResp + if err := json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf(msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + return nil +} + +func (d *DoubaoNew) renameFile(ctx context.Context, fileToken, name string) error { + if fileToken == "" { + return fmt.Errorf("[doubao_new] rename file missing file token") + } + _, err := d.request(ctx, "/space/api/box/file/update_info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "file_token": fileToken, + "name": name, + }) + }, nil) + return err +} + +func (d *DoubaoNew) moveObj(ctx context.Context, srcToken, destToken string) error { + if srcToken == "" { + return fmt.Errorf("[doubao_new] move missing src token") + } + data := url.Values{} + data.Set("src_token", srcToken) + if destToken != "" { + data.Set("dest_token", destToken) + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/move/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func (d *DoubaoNew) removeObj(ctx context.Context, tokens []string) error { + if len(tokens) == 0 { + return fmt.Errorf("[doubao_new] remove missing tokens") + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "tokens": tokens, + "apply": 1, + }) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v3/remove/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + var resp RemoveResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + if resp.Data.TaskID == "" { + return nil + } + return d.waitTask(ctx, resp.Data.TaskID) +} + +func (d *DoubaoNew) getUserStorage(ctx context.Context) (UserStorageData, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("agw-js-conv", "str") + req.SetHeader("content-type", "application/json") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + } + req.SetBody(base.Json{}) + + res, err := req.Execute(http.MethodPost, "https://www.doubao.com/alice/aispace/facade/get_user_storage") + if err != nil { + return UserStorageData{}, err + } + + body := res.Body() + var resp UserStorageResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UserStorageData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UserStorageData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) waitTask(ctx context.Context, taskID string) error { + const ( + taskPollInterval = time.Second + taskPollMaxAttempts = 120 + ) + var lastErr error + for attempt := 0; attempt < taskPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, taskPollInterval); err != nil { + return err + } + } + status, err := d.getTaskStatus(ctx, taskID) + if err != nil { + lastErr = err + continue + } + if status.IsFail { + return fmt.Errorf("[doubao_new] remove task failed: %s", taskID) + } + if status.IsFinish { + return nil + } + } + if lastErr != nil { + return lastErr + } + return fmt.Errorf("[doubao_new] remove task timed out: %s", taskID) +} + +func (d *DoubaoNew) getTaskStatus(ctx context.Context, taskID string) (TaskStatusData, error) { + if taskID == "" { + return TaskStatusData{}, fmt.Errorf("[doubao_new] task status missing task_id") + } + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + req.SetQueryParam("task_id", taskID) + res, err := req.Execute(http.MethodGet, BaseURL+"/space/api/explorer/v2/task/") + if err != nil { + return TaskStatusData{}, err + } + body := res.Body() + var resp TaskStatusResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return TaskStatusData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return TaskStatusData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + return resp.Data, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (d *DoubaoNew) prepareUpload(ctx context.Context, name string, size int64, mountNodeToken string) (UploadPrepareData, error) { + var resp UploadPrepareResp + _, err := d.request(ctx, "/space/api/box/upload/prepare/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.prepare") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + body := base.Json{ + "mount_point": "explorer", + "mount_node_token": "", + "name": name, + "size": size, + "size_checker": true, + } + if mountNodeToken != "" { + body["mount_node_token"] = mountNodeToken + } + req.SetBody(body) + }, &resp) + if err != nil { + return UploadPrepareData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlocks(ctx context.Context, uploadID string, blocks []UploadBlock, mountPoint string) (UploadBlocksData, error) { + if uploadID == "" { + return UploadBlocksData{}, fmt.Errorf("[doubao_new] upload blocks missing upload_id") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadBlocksResp + _, err := d.request(ctx, "/space/api/box/upload/blocks/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.blocks") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetBody(base.Json{ + "blocks": blocks, + "upload_id": uploadID, + "mount_point": mountPoint, + }) + }, &resp) + if err != nil { + return UploadBlocksData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) mergeUploadBlocks(ctx context.Context, uploadID string, seqList []int, checksumList []string, sizeList []int64, blockOriginSize int64, data []byte) (UploadMergeData, error) { + if uploadID == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing upload_id") + } + if len(seqList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty seq list") + } + if len(checksumList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty checksum list") + } + if len(sizeList) != len(seqList) { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks size list mismatch") + } + if blockOriginSize <= 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks invalid block origin size") + } + if len(data) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty data") + } + + seqHeader := joinIntComma(seqList) + checksumHeader := buildCommaHeader(checksumList) + + client := base.NewRestyClient() + client.SetCookieJar(nil) + req := client.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("content-type", "application/octet-stream") + req.Header.Set("x-block-list-checksum", checksumHeader) + req.Header.Set("x-seq-list", seqHeader) + req.SetHeader("x-block-origin-size", strconv.FormatInt(blockOriginSize, 10)) + req.SetHeader("x-command", "space.api.box.stream.upload.merge_block") + req.SetHeader("x-csrftoken", "") + reqID := "" + if buf := make([]byte, 16); true { + if _, err := rand.Read(buf); err == nil { + reqID = hex.EncodeToString(buf) + } + } + if reqID != "" { + req.SetHeader("x-request-id", reqID) + } + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + req.Header.Del("Cookie") + req.Header.Del("cookie") + if req.Header.Get("x-command") == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing x-command header") + } + req.SetBody(data) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + urlStr := "https://internal-api-drive-stream.feishu.cn/space/api/box/stream/upload/merge_block/?" + values.Encode() + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return UploadMergeData{}, err + } + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + body := res.Body() + var resp UploadMergeResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UploadMergeData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + if res != nil && res.StatusCode() == http.StatusBadRequest && resp.Code == 2 { + success := make([]int, 0, len(seqList)) + offset := 0 + for i, seq := range seqList { + size := sizeList[i] + if size <= 0 { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback invalid size: seq=%d size=%d", seq, size) + } + if offset+int(size) > len(data) { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback payload out of range: seq=%d offset=%d size=%d total=%d", seq, offset, size, len(data)) + } + payload := data[offset : offset+int(size)] + block := UploadBlockNeed{ + Seq: seq, + Size: size, + Checksum: checksumList[i], + } + if err := d.uploadBlockV3(ctx, uploadID, block, payload); err != nil { + return UploadMergeData{SuccessSeqList: success}, err + } + success = append(success, seq) + offset += int(size) + } + return UploadMergeData{SuccessSeqList: success}, nil + } + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UploadMergeData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlockV3(ctx context.Context, uploadID string, block UploadBlockNeed, data []byte) error { + if uploadID == "" { + return fmt.Errorf("[doubao_new] upload v3 block missing upload_id") + } + if block.Seq < 0 { + return fmt.Errorf("[doubao_new] upload v3 block invalid seq") + } + if len(data) == 0 { + return fmt.Errorf("[doubao_new] upload v3 block empty data") + } + + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("x-block-seq", strconv.Itoa(block.Seq)) + req.SetHeader("x-block-checksum", block.Checksum) + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + + req.SetMultipartFormData(map[string]string{ + "upload_id": uploadID, + "size": strconv.FormatInt(int64(len(data)), 10), + }) + req.SetMultipartField("file", "blob", "application/octet-stream", bytes.NewReader(data)) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("seq", strconv.Itoa(block.Seq)) + values.Set("size", strconv.FormatInt(int64(len(data)), 10)) + values.Set("checksum", block.Checksum) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + urlStr := "https://internal-api-drive-stream.feishu.cn/space/api/box/stream/upload/v3/block/?" + values.Encode() + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return err + } + body := res.Body() + if err := decodeBaseResp(body, res); err != nil { + return err + } + return nil +} + +func (d *DoubaoNew) finishUpload(ctx context.Context, uploadID string, numBlocks int, mountPoint string) (UploadFinishData, error) { + if uploadID == "" { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload missing upload_id") + } + if numBlocks <= 0 { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload invalid num_blocks") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadFinishResp + _, err := d.request(ctx, "/space/api/box/upload/finish/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.finish") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetHeader("biz-scene", "file_upload") + req.SetHeader("biz-ua-type", "Web") + req.SetBody(base.Json{ + "upload_id": uploadID, + "num_blocks": numBlocks, + "mount_point": mountPoint, + "push_open_history_record": 1, + }) + }, &resp) + if err != nil { + return UploadFinishData{}, err + } + return resp.Data, nil +} diff --git a/server/handles/down.go b/server/handles/down.go index 59c75530d3b..e93ed1eb103 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -55,6 +55,20 @@ func Proxy(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if c.Query("type") == "preview" && storage.GetStorage().Driver == "DoubaoNew" { + // Force proxy for DoubaoNew preview so headers are preserved. + link, file, err := fs.Link(c, rawPath, model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + localProxy(c, link, file, storage.GetStorage().ProxyRange) + return + } if canProxy(storage, filename) { downProxyUrl := storage.GetStorage().DownProxyUrl if downProxyUrl != "" { From 7323583af8c1f32b76cfc5270b7fd9b275ab6969 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 4 Feb 2026 13:20:11 +0800 Subject: [PATCH 586/659] chore(driver, archive): Remove debug logs and add missing `os` import --- drivers/doubao_new/driver.go | 2 -- internal/archive/tool/securepath.go | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go index 7846551790a..af0f9dd2ac0 100644 --- a/drivers/doubao_new/driver.go +++ b/drivers/doubao_new/driver.go @@ -304,8 +304,6 @@ func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileSt if len(data) < tailLen { tailLen = len(data) } - fmt.Printf("head32 = %x\n", data[:headLen]) - fmt.Printf("tail32 = %x\n", data[len(data)-tailLen:]) } if int64(len(data)) != expectLen { return fmt.Errorf("[doubao_new] merge blocks invalid body len: got=%d expect=%d seqs=%v", len(data), expectLen, groupSeqs) diff --git a/internal/archive/tool/securepath.go b/internal/archive/tool/securepath.go index 45f5e749d53..f9bd89a914a 100644 --- a/internal/archive/tool/securepath.go +++ b/internal/archive/tool/securepath.go @@ -3,6 +3,7 @@ package tool import ( "errors" "fmt" + "os" "path" "path/filepath" "strings" From a0389e2777644e339716ff462c76179b698da199 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 4 Feb 2026 16:39:15 +0800 Subject: [PATCH 587/659] feat(iso9660): Refactor path handling for secure file extraction - Introduced `SecureJoin` to prevent path traversal vulnerabilities - Refactored `decompress` logic into `decompressEntry` for improved modularity - Ensured secure directory creation with `MkdirAll` in all operations --- internal/archive/iso9660/iso9660.go | 11 +++++++---- internal/archive/iso9660/utils.go | 26 +++++++++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/internal/archive/iso9660/iso9660.go b/internal/archive/iso9660/iso9660.go index be107d7b4c4..7de8da6f393 100644 --- a/internal/archive/iso9660/iso9660.go +++ b/internal/archive/iso9660/iso9660.go @@ -1,14 +1,14 @@ package iso9660 import ( + "io" + "os" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" "github.com/kdomanski/iso9660" - "io" - "os" - stdpath "path" ) type ISO9660 struct { @@ -78,7 +78,10 @@ func (ISO9660) Decompress(ss []*stream.SeekableStream, outputPath string, args m } if obj.IsDir() { if args.InnerPath != "/" { - outputPath = stdpath.Join(outputPath, obj.Name()) + outputPath, err = tool.SecureJoin(outputPath, obj.Name()) + if err != nil { + return err + } if err = os.MkdirAll(outputPath, 0700); err != nil { return err } diff --git a/internal/archive/iso9660/utils.go b/internal/archive/iso9660/utils.go index 0e4cfb1caf3..dd2e00354f8 100644 --- a/internal/archive/iso9660/utils.go +++ b/internal/archive/iso9660/utils.go @@ -1,10 +1,12 @@ package iso9660 import ( + "io" "os" - stdpath "path" + "path/filepath" "strings" + "github.com/alist-org/alist/v3/internal/archive/tool" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/stream" @@ -62,15 +64,26 @@ func toModelObj(file *iso9660.File) model.Obj { } func decompress(f *iso9660.File, path string, up model.UpdateProgress) error { - file, err := os.OpenFile(stdpath.Join(path, f.Name()), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + return decompressEntry(f.Reader(), f.Size(), path, f.Name(), up) +} + +func decompressEntry(reader io.Reader, size int64, path, entryName string, up model.UpdateProgress) error { + dstPath, err := tool.SecureJoin(path, entryName) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + file, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) if err != nil { return err } defer file.Close() _, err = utils.CopyWithBuffer(file, &stream.ReaderUpdatingProgress{ Reader: &stream.SimpleReaderWithSize{ - Reader: f.Reader(), - Size: f.Size(), + Reader: reader, + Size: size, }, UpdateProgress: up, }) @@ -84,7 +97,10 @@ func decompressAll(children []*iso9660.File, path string) error { if err != nil { return err } - nextPath := stdpath.Join(path, child.Name()) + nextPath, err := tool.SecureJoin(path, child.Name()) + if err != nil { + return err + } if err = os.MkdirAll(nextPath, 0700); err != nil { return err } From 1880b5afb8f8d47dc37f2b07b9366f533d4ae562 Mon Sep 17 00:00:00 2001 From: RedwindA Date: Sat, 7 Feb 2026 21:15:15 +0800 Subject: [PATCH 588/659] feat(drivers): add FTPS driver with TLS support - Implement FTPS (FTP over SSL/TLS) driver for secure file transfers - Support both Explicit (STARTTLS on port 21) and Implicit (direct TLS on port 990) TLS modes - Configurable TLS certificate verification skip for self-signed certificates - Character encoding support via mahonia library - Full CRUD operations: list, create directories, upload, download, move, rename, delete --- drivers/all.go | 1 + drivers/ftps/driver.go | 127 +++++++++++++++++++++++++++++++++++++++ drivers/ftps/meta.go | 46 ++++++++++++++ drivers/ftps/types.go | 1 + drivers/ftps/util.go | 132 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 307 insertions(+) create mode 100644 drivers/ftps/driver.go create mode 100644 drivers/ftps/meta.go create mode 100644 drivers/ftps/types.go create mode 100644 drivers/ftps/util.go diff --git a/drivers/all.go b/drivers/all.go index 3b0435595d2..c53ba6caddc 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -32,6 +32,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" + _ "github.com/alist-org/alist/v3/drivers/ftps" _ "github.com/alist-org/alist/v3/drivers/gitee" _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/github_releases" diff --git a/drivers/ftps/driver.go b/drivers/ftps/driver.go new file mode 100644 index 00000000000..4467380f09c --- /dev/null +++ b/drivers/ftps/driver.go @@ -0,0 +1,127 @@ +package ftps + +import ( + "context" + stdpath "path" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/jlaffaye/ftp" +) + +type FTPS struct { + model.Storage + Addition + conn *ftp.ServerConn +} + +func (d *FTPS) Config() driver.Config { + return config +} + +func (d *FTPS) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *FTPS) Init(ctx context.Context) error { + return d.login() +} + +func (d *FTPS) Drop(ctx context.Context) error { + if d.conn != nil { + _ = d.conn.Logout() + } + return nil +} + +func (d *FTPS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.login(); err != nil { + return nil, err + } + entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0) + for _, entry := range entries { + if entry.Name == "." || entry.Name == ".." { + continue + } + f := model.Object{ + Name: decode(entry.Name, d.Encoding), + Size: int64(entry.Size), + Modified: entry.Time, + IsFolder: entry.Type == ftp.EntryTypeFolder, + } + res = append(res, &f) + } + return res, nil +} + +func (d *FTPS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.login(); err != nil { + return nil, err + } + r := NewFileReader(d.conn, encode(file.GetPath(), d.Encoding), file.GetSize()) + link := &model.Link{ + MFile: r, + } + return link, nil +} + +func (d *FTPS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.login(); err != nil { + return err + } + return d.conn.MakeDir(encode(stdpath.Join(parentDir.GetPath(), dirName), d.Encoding)) +} + +func (d *FTPS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.login(); err != nil { + return err + } + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(dstDir.GetPath(), srcObj.GetName()), d.Encoding), + ) +} + +func (d *FTPS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.login(); err != nil { + return err + } + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), d.Encoding), + ) +} + +func (d *FTPS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *FTPS) Remove(ctx context.Context, obj model.Obj) error { + if err := d.login(); err != nil { + return err + } + path := encode(obj.GetPath(), d.Encoding) + if obj.IsDir() { + return d.conn.RemoveDirRecur(path) + } else { + return d.conn.Delete(path) + } +} + +func (d *FTPS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + if err := d.login(); err != nil { + return err + } + path := stdpath.Join(dstDir.GetPath(), s.GetName()) + return d.conn.Stor(encode(path, d.Encoding), driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + })) +} + +var _ driver.Driver = (*FTPS)(nil) diff --git a/drivers/ftps/meta.go b/drivers/ftps/meta.go new file mode 100644 index 00000000000..a752ec01aa6 --- /dev/null +++ b/drivers/ftps/meta.go @@ -0,0 +1,46 @@ +package ftps + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" + "github.com/axgle/mahonia" +) + +func encode(str string, encoding string) string { + if encoding == "" { + return str + } + encoder := mahonia.NewEncoder(encoding) + return encoder.ConvertString(str) +} + +func decode(str string, encoding string) string { + if encoding == "" { + return str + } + decoder := mahonia.NewDecoder(encoding) + return decoder.ConvertString(str) +} + +type Addition struct { + Address string `json:"address" required:"true"` + Encoding string `json:"encoding" required:"false"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + TLSMode string `json:"tls_mode" type:"select" options:"Explicit,Implicit" default:"Explicit" required:"true" help:"Explicit: STARTTLS on port 21; Implicit: direct TLS on port 990"` + TLSInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false" help:"Allow insecure TLS connections (e.g. self-signed certificates)"` + driver.RootPath +} + +var config = driver.Config{ + Name: "FTPS", + LocalSort: true, + OnlyLocal: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &FTPS{} + }) +} diff --git a/drivers/ftps/types.go b/drivers/ftps/types.go new file mode 100644 index 00000000000..d01dc417a16 --- /dev/null +++ b/drivers/ftps/types.go @@ -0,0 +1 @@ +package ftps diff --git a/drivers/ftps/util.go b/drivers/ftps/util.go new file mode 100644 index 00000000000..37591dcbb78 --- /dev/null +++ b/drivers/ftps/util.go @@ -0,0 +1,132 @@ +package ftps + +import ( + "crypto/tls" + "io" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/jlaffaye/ftp" +) + +func (d *FTPS) login() error { + if d.conn != nil { + _, err := d.conn.CurrentDir() + if err == nil { + return nil + } + } + + tlsConfig := &tls.Config{ + InsecureSkipVerify: d.TLSInsecureSkipVerify, + } + + opts := []ftp.DialOption{ + ftp.DialWithShutTimeout(10 * time.Second), + } + if d.TLSMode == "Implicit" { + opts = append(opts, ftp.DialWithTLS(tlsConfig)) + } else { + opts = append(opts, ftp.DialWithExplicitTLS(tlsConfig)) + } + + conn, err := ftp.Dial(d.Address, opts...) + if err != nil { + return err + } + err = conn.Login(d.Username, d.Password) + if err != nil { + _ = conn.Quit() + return err + } + d.conn = conn + return nil +} + +type FileReader struct { + conn *ftp.ServerConn + resp *ftp.Response + offset atomic.Int64 + readAtOffset int64 + mu sync.Mutex + path string + size int64 +} + +func NewFileReader(conn *ftp.ServerConn, path string, size int64) *FileReader { + return &FileReader{ + conn: conn, + path: path, + size: size, + } +} + +func (r *FileReader) Read(buf []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + off := r.offset.Load() + n, err = r.readAtLocked(buf, off) + r.offset.Add(int64(n)) + return +} + +func (r *FileReader) ReadAt(buf []byte, off int64) (n int, err error) { + if off < 0 { + return 0, os.ErrInvalid + } + r.mu.Lock() + defer r.mu.Unlock() + return r.readAtLocked(buf, off) +} + +func (r *FileReader) readAtLocked(buf []byte, off int64) (n int, err error) { + if r.resp != nil && off != r.readAtOffset { + _ = r.resp.Close() + r.resp = nil + } + + if r.resp == nil { + r.resp, err = r.conn.RetrFrom(r.path, uint64(off)) + r.readAtOffset = off + if err != nil { + return 0, err + } + } + + n, err = r.resp.Read(buf) + r.readAtOffset += int64(n) + return +} + +func (r *FileReader) Seek(offset int64, whence int) (int64, error) { + oldOffset := r.offset.Load() + var newOffset int64 + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = oldOffset + offset + case io.SeekEnd: + newOffset = r.size + offset + default: + return -1, os.ErrInvalid + } + + if newOffset < 0 { + return oldOffset, os.ErrInvalid + } + if newOffset == oldOffset { + return oldOffset, nil + } + r.offset.Store(newOffset) + return newOffset, nil +} + +func (r *FileReader) Close() error { + if r.resp != nil { + return r.resp.Close() + } + return nil +} From 426d594c5178e02cfa5ab5dd73dc76a13210094e Mon Sep 17 00:00:00 2001 From: RedwindA Date: Sat, 7 Feb 2026 21:29:20 +0800 Subject: [PATCH 589/659] fix(ftps): add host parsing for TLS configuration in login function --- drivers/ftps/util.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/drivers/ftps/util.go b/drivers/ftps/util.go index 37591dcbb78..10680fa8b80 100644 --- a/drivers/ftps/util.go +++ b/drivers/ftps/util.go @@ -3,6 +3,7 @@ package ftps import ( "crypto/tls" "io" + "net" "os" "sync" "sync/atomic" @@ -19,7 +20,13 @@ func (d *FTPS) login() error { } } + host, _, err := net.SplitHostPort(d.Address) + if err != nil { + host = d.Address + } + tlsConfig := &tls.Config{ + ServerName: host, InsecureSkipVerify: d.TLSInsecureSkipVerify, } From 39962d67f64847a647a7855f9783ea5513123c90 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Mon, 9 Feb 2026 14:08:56 +0800 Subject: [PATCH 590/659] feat(driver): Upgrade dependencies and enhance 115 cloud functionality - Updated `github.com/SheltonZhu/115driver` to v1.2.3-1 and `golang.org/x/oauth2` to v0.30.0 - Added support for `ThumbURL` handling in file objects, enabling thumbnail retrieval - Refactored 115 share operations with new methods for enhanced User-Agent configuration - Improved file fetching logic to include thumbnails and support paginated results - Consolidated download and snapshot logic with User-Agent handling for better compatibility --- drivers/115/driver.go | 8 +- drivers/115/types.go | 5 ++ drivers/115/util.go | 161 +++++++++++++++++++++--------------- drivers/115_share/driver.go | 20 +++-- drivers/115_share/utils.go | 101 +++++++++++++++++++++- go.mod | 6 +- go.sum | 8 +- 7 files changed, 222 insertions(+), 87 deletions(-) diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 0dcb64d8284..60fe60e68e7 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -66,9 +66,11 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err := d.WaitLimit(ctx); err != nil { return nil, err } - userAgent := args.Header.Get("User-Agent") - downloadInfo, err := d. - DownloadWithUA(file.(*FileObj).PickCode, userAgent) + userAgent := "" + if args.Header != nil { + userAgent = args.Header.Get("User-Agent") + } + downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } diff --git a/drivers/115/types.go b/drivers/115/types.go index 40b951d80ce..7a80e3ef047 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -12,6 +12,7 @@ var _ model.Obj = (*FileObj)(nil) type FileObj struct { driver.File + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -22,6 +23,10 @@ func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + type UploadResult struct { driver.BasicResp Data struct { diff --git a/drivers/115/util.go b/drivers/115/util.go index fc17fe3cebf..79d869178cb 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -9,7 +9,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "strconv" "strings" @@ -25,11 +24,28 @@ import ( "github.com/aliyun/aliyun-oss-go-sdk/oss" cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" - crypto "github.com/SheltonZhu/115driver/pkg/crypto/m115" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/pkg/errors" ) +type fileInfoWithThumb struct { + driver115.FileInfo + ThumbURL string `json:"u"` +} + +type fileListRespWithThumb struct { + driver115.BasicResp + CategoryID driver115.IntString `json:"cid"` + Count int `json:"count"` + Offset int `json:"offset"` + Files []fileInfoWithThumb `json:"data"` +} + +type getFileInfoResponseWithThumb struct { + driver115.BasicResp + Files []*fileInfoWithThumb `json:"data"` +} + // var UserAgent = driver115.UA115Browser func (d *Pan115) login() error { var err error @@ -66,100 +82,113 @@ func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { if d.PageSize <= 0 { d.PageSize = driver115.FileListLimit } - files, err := d.client.ListWithLimit(fileId, d.PageSize, driver115.WithMultiUrls()) - if err != nil { - return nil, err + limit := d.PageSize + if limit > driver115.MaxDirPageLimit { + limit = driver115.MaxDirPageLimit } - for _, file := range *files { - res = append(res, FileObj{file}) + + opts := driver115.DefaultListOptions() + driver115.WithMultiUrls()(opts) + if len(opts.ApiURLs) == 0 { + opts.ApiURLs = []string{driver115.ApiFileList} } + + offset := int64(0) + for i := 0; ; i++ { + result, err := d.getFilesPageWithThumb(fileId, opts.ApiURLs[i%len(opts.ApiURLs)], limit, offset) + if err != nil { + return nil, err + } + for _, fileInfo := range result.Files { + res = append(res, fileObjFromInfo(&fileInfo)) + } + offset = int64(result.Offset) + limit + if offset >= int64(result.Count) { + break + } + } + return res, nil } func (d *Pan115) getNewFile(fileId string) (*FileObj, error) { - file, err := d.client.GetFile(fileId) + fileInfo, err := d.getFileInfoWithThumb("file_id", fileId) if err != nil { return nil, err } - return &FileObj{*file}, nil + file := fileObjFromInfo(fileInfo) + return &file, nil } func (d *Pan115) getNewFileByPickCode(pickCode string) (*FileObj, error) { - result := driver115.GetFileInfoResponse{} - req := d.client.NewRequest(). - SetQueryParam("pick_code", pickCode). - ForceContentType("application/json;charset=UTF-8"). - SetResult(&result) - resp, err := req.Get(driver115.ApiFileInfo) - if err := driver115.CheckErr(err, &result, resp); err != nil { + fileInfo, err := d.getFileInfoWithThumb("pick_code", pickCode) + if err != nil { return nil, err } - if len(result.Files) == 0 { - return nil, errors.New("not get file info") - } - fileInfo := result.Files[0] - - f := &FileObj{} - f.From(fileInfo) - return f, nil + file := fileObjFromInfo(fileInfo) + return &file, nil } func (d *Pan115) getUA() string { return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) } -func (d *Pan115) DownloadWithUA(pickCode, ua string) (*driver115.DownloadInfo, error) { - key := crypto.GenerateKey() - result := driver115.DownloadResp{} - params, err := utils.Json.Marshal(map[string]string{"pick_code": pickCode}) - if err != nil { - return nil, err - } - - data := crypto.Encode(params, key) - - bodyReader := strings.NewReader(url.Values{"data": []string{data}}.Encode()) - reqUrl := fmt.Sprintf("%s?t=%s", driver115.AndroidApiDownloadGetUrl, driver115.Now().String()) - req, _ := http.NewRequest(http.MethodPost, reqUrl, bodyReader) - req.Header.Set("Content-Type", "application/x-www-form-urlencoded") - req.Header.Set("Cookie", d.Cookie) - req.Header.Set("User-Agent", ua) - - resp, err := d.client.Client.GetClient().Do(req) - if err != nil { - return nil, err +func fileObjFromInfo(fileInfo *fileInfoWithThumb) FileObj { + file := &driver115.File{} + file.From(&fileInfo.FileInfo) + return FileObj{ + File: *file, + ThumbURL: fileInfo.ThumbURL, } - defer resp.Body.Close() +} - body, err := io.ReadAll(resp.Body) - if err != nil { +func (d *Pan115) getFileInfoWithThumb(queryKey, queryVal string) (*fileInfoWithThumb, error) { + result := getFileInfoResponseWithThumb{} + req := d.client.NewRequest(). + SetQueryParam(queryKey, queryVal). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + resp, err := req.Get(driver115.ApiFileInfo) + if err := driver115.CheckErr(err, &result, resp); err != nil { return nil, err } - if err := utils.Json.Unmarshal(body, &result); err != nil { - return nil, err + if len(result.Files) == 0 { + return nil, errors.New("not get file info") } + return result.Files[0], nil +} - if err = result.Err(string(body)); err != nil { - return nil, err +func (d *Pan115) getFilesPageWithThumb(dirID, apiURL string, limit, offset int64) (*fileListRespWithThumb, error) { + if dirID == "" { + dirID = "0" + } + result := fileListRespWithThumb{} + params := map[string]string{ + "aid": "1", + "cid": dirID, + "o": driver115.FileOrderByTime, + "asc": "1", + "offset": strconv.FormatInt(offset, 10), + "show_dir": "1", + "limit": strconv.FormatInt(limit, 10), + "snap": "0", + "natsort": "0", + "record_open_time": "1", + "format": "json", + "fc_mix": "0", } - - b, err := crypto.Decode(string(result.EncodedData), key) - if err != nil { + req := d.client.NewRequest(). + ForceContentType("application/json;charset=UTF-8"). + SetQueryParams(params). + SetResult(&result) + resp, err := req.Get(apiURL) + if err := driver115.CheckErr(err, &result, resp); err != nil { return nil, err } - - downloadInfo := struct { - Url string `json:"url"` - }{} - if err := utils.Json.Unmarshal(b, &downloadInfo); err != nil { - return nil, err + if dirID != string(result.CategoryID) { + return nil, driver115.ErrUnexpected } - - info := &driver115.DownloadInfo{} - info.PickCode = pickCode - info.Header = resp.Request.Header - info.Url.Url = downloadInfo.Url - return info, nil + return &result, nil } func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go index 886a369c1b8..322b64afd45 100644 --- a/drivers/115_share/driver.go +++ b/drivers/115_share/driver.go @@ -4,6 +4,7 @@ import ( "context" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -50,8 +51,9 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr return nil, err } - files := make([]driver115.ShareFile, 0) - fileResp, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + ua := base.UserAgent + files := make([]shareFile, 0) + fileResp, err := d.getShareSnapWithUA(ua, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) if err != nil { return nil, err } @@ -59,10 +61,7 @@ func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListAr total := fileResp.Data.Count count := len(fileResp.Data.List) for total > count { - fileResp, err := d.client.GetShareSnap( - d.ShareCode, d.ReceiveCode, dir.GetID(), - driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count), - ) + fileResp, err := d.getShareSnapWithUA(ua, dir.GetID(), driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count)) if err != nil { return nil, err } @@ -77,7 +76,14 @@ func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkA if err := d.WaitLimit(ctx); err != nil { return nil, err } - downloadInfo, err := d.client.DownloadByShareCode(d.ShareCode, d.ReceiveCode, file.GetID()) + ua := "" + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + downloadInfo, err := d.downloadByShareCodeWithUA(ua, file.GetID()) if err != nil { return nil, err } diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go index 1f9e112deef..e36a5ef8ccb 100644 --- a/drivers/115_share/utils.go +++ b/drivers/115_share/utils.go @@ -6,6 +6,7 @@ import ( "time" driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" @@ -20,6 +21,7 @@ type FileObj struct { FileName string isDir bool FileID string + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -54,7 +56,39 @@ func (f *FileObj) GetPath() string { return "" } -func transFunc(sf driver115.ShareFile) (model.Obj, error) { +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + +type shareFile struct { + FileID string `json:"fid"` + UID int `json:"uid"` + CategoryID driver115.IntString `json:"cid"` + FileName string `json:"n"` + Type string `json:"ico"` + Sha1 string `json:"sha"` + Size driver115.StringInt64 `json:"s"` + Labels []*driver115.LabelInfo `json:"fl"` + UpdateTime string `json:"t"` + IsFile int `json:"fc"` + ParentID string `json:"pid"` + ThumbURL string `json:"u"` +} + +type shareSnapResp struct { + driver115.BasicResp + Data struct { + Count int `json:"count"` + List []shareFile `json:"list"` + } `json:"data"` +} + +type downloadShareResp struct { + driver115.BasicResp + Data driver115.SharedDownloadInfo `json:"data"` +} + +func transFunc(sf shareFile) (model.Obj, error) { timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) if err != nil { return nil, err @@ -74,18 +108,77 @@ func transFunc(sf driver115.ShareFile) (model.Obj, error) { FileName: string(sf.FileName), isDir: isDir, FileID: fileID, + ThumbURL: sf.ThumbURL, }, nil } -var UserAgent = driver115.UA115Browser +func buildShareReferer(shareCode, receiveCode string) string { + return fmt.Sprintf("https://115cdn.com/s/%s?password=%s&", shareCode, receiveCode) +} + +func (d *Pan115Share) getShareSnapWithUA(ua, dirID string, queries ...driver115.Query) (*shareSnapResp, error) { + result := shareSnapResp{} + query := map[string]string{ + "share_code": d.ShareCode, + "receive_code": d.ReceiveCode, + "cid": dirID, + "limit": "20", + "asc": "0", + "offset": "0", + "format": "json", + } + for _, q := range queries { + q(&query) + } + + req := d.client.NewRequest(). + SetQueryParams(query). + SetHeader("referer", buildShareReferer(d.ShareCode, d.ReceiveCode)). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + if ua != "" { + req = req.SetHeader("User-Agent", ua) + } + + resp, err := req.Get(driver115.ApiShareSnap) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + return &result, nil +} + +func (d *Pan115Share) downloadByShareCodeWithUA(ua, fileID string) (*driver115.SharedDownloadInfo, error) { + result := downloadShareResp{} + params := map[string]string{ + "share_code": d.ShareCode, + "receive_code": d.ReceiveCode, + "file_id": fileID, + "dl": "1", + } + + req := d.client.NewRequest(). + SetQueryParams(params). + ForceContentType("application/json"). + SetHeader("referer", buildShareReferer(d.ShareCode, d.ReceiveCode)). + SetResult(&result) + if ua != "" { + req = req.SetHeader("User-Agent", ua) + } + + resp, err := req.Get(driver115.ApiDownloadGetShareUrl) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + return &result.Data, nil +} func (d *Pan115Share) login() error { var err error opts := []driver115.Option{ - driver115.UA(UserAgent), + driver115.UA(base.UserAgent), } d.client = driver115.New(opts...) - if _, err := d.client.GetShareSnap(d.ShareCode, d.ReceiveCode, ""); err != nil { + if _, err = d.getShareSnapWithUA(base.UserAgent, ""); err != nil { return errors.Wrap(err, "failed to get share snap") } cr := &driver115.Credential{} diff --git a/go.mod b/go.mod index 5772806a7b4..2e202bc9de0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/KirCute/sftpd-alist v0.0.12 github.com/ProtonMail/go-crypto v1.0.0 github.com/ProtonMail/gopenpgp/v2 v2.7.4 - github.com/SheltonZhu/115driver v1.1.2 + github.com/SheltonZhu/115driver v1.2.3-1 github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 github.com/alist-org/gofakes3 v0.0.7 @@ -74,7 +74,7 @@ require ( golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 golang.org/x/net v0.38.0 - golang.org/x/oauth2 v0.22.0 + golang.org/x/oauth2 v0.30.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 @@ -286,4 +286,4 @@ replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed -replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.1.2 +replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.2.3-1 diff --git a/go.sum b/go.sum index 4b21f881331..af3a6b3f96c 100644 --- a/go.sum +++ b/go.sum @@ -524,8 +524,8 @@ github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= -github.com/okatu-loli/115driver v1.1.2 h1:XZT3r/51SZRQGzre2IeA+0/k4T1FneqArdhE4Wd600Q= -github.com/okatu-loli/115driver v1.1.2/go.mod h1:rKvNd4Y4OkXv1TMbr/SKjGdcvMQxh6AW5Tw9w0CJb7E= +github.com/okatu-loli/115driver v1.2.3-1 h1:UoBEREqh6RD6WlxiJ2Z29JxNZ/UcoChvdHn9r9Tx7nI= +github.com/okatu-loli/115driver v1.2.3-1/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= @@ -797,8 +797,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= -golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 3bc7de34ee53153eade8532ed212d70bc1273860 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 10 Feb 2026 16:46:57 +0800 Subject: [PATCH 591/659] feat(auth): Add consistent error messaging for invalid login credentials - Introduced `invalidLoginCredentialsMsg` constant for reusability - Updated error responses in login flow to provide a consistent message --- server/handles/auth.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/server/handles/auth.go b/server/handles/auth.go index 3520a459c3d..f59b4fb1ed7 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -26,8 +26,9 @@ import ( var loginCache = cache.NewMemCache[int]() var ( - defaultDuration = time.Minute * 5 - defaultTimes = 5 + defaultDuration = time.Minute * 5 + defaultTimes = 5 + invalidLoginCredentialsMsg = "username or password is incorrect" ) type LoginReq struct { @@ -69,13 +70,13 @@ func loginHash(c *gin.Context, req *LoginReq) { // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, invalidLoginCredentialsMsg, 400) loginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, invalidLoginCredentialsMsg, 400) loginCache.Set(ip, count+1) return } From 338569fdc253979cb87fb31be27c68ff399fd755 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 20 Feb 2026 22:01:17 +0800 Subject: [PATCH 592/659] feat(fs): Add pagination support and enhance response structure - Introduced new pagination fields: `page`, `per_page`, `has_more`, and `pages_total` in `FsListResp` - Added `normalizeListPage` and `calcPagesTotal` for effective pagination handling - Updated default and max page size values; implemented validations for page parameters - Refactored `FsList` handler to include paginated responses and filtered totals - Adjusted default page size in site settings from 30 to 50 --- internal/bootstrap/data/setting.go | 2 +- server/handles/fsread.go | 81 +++++++++++++++++++++++------- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index bbb633e33a6..c5541fa38d4 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -100,7 +100,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.SiteTitle, Value: "AList", Type: conf.TypeString, Group: model.SITE}, {Key: conf.Announcement, Value: "### repo\nhttps://github.com/alist-org/alist", Type: conf.TypeText, Group: model.SITE}, {Key: "pagination_type", Value: "all", Type: conf.TypeSelect, Options: "all,pagination,load_more,auto_load_more", Group: model.SITE}, - {Key: "default_page_size", Value: "30", Type: conf.TypeNumber, Group: model.SITE}, + {Key: "default_page_size", Value: "50", Type: conf.TypeNumber, Group: model.SITE}, {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 15dd9f1ce7e..eb984698b16 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -49,12 +49,17 @@ type ObjResp struct { } type FsListResp struct { - Content []ObjLabelResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjLabelResp `json:"content"` + Total int64 `json:"total"` + FilteredTotal int64 `json:"filtered_total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + HasMore bool `json:"has_more"` + PagesTotal int `json:"pages_total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` } type ObjLabelResp struct { @@ -74,13 +79,20 @@ type ObjLabelResp struct { StorageClass string `json:"storage_class,omitempty"` } +const ( + DefaultPerPage = 200 + MaxPerPage = 500 +) + func FsList(c *gin.Context) { var req ListReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } - req.Validate() + effPage, effPerPage := normalizeListPage(req.Page, req.PerPage) + req.Page = effPage + req.PerPage = effPerPage user := c.MustGet("user").(*model.User) reqPath, err := user.JoinPath(req.Path) if err != nil { @@ -104,6 +116,11 @@ func FsList(c *gin.Context) { common.ErrorStrResp(c, "Refresh without permission", 403) return } + provider := "unknown" + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if storageErr == nil { + provider = storage.GetStorage().Driver + } objs, err := fs.List(c, reqPath, &fs.ListArgs{Refresh: req.Refresh}) if err != nil { common.ErrorResp(c, err, 500) @@ -116,19 +133,23 @@ func FsList(c *gin.Context) { filtered = append(filtered, obj) } } - total, objs := pagination(filtered, &req.PageReq) - provider := "unknown" - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) - if err == nil { - provider = storage.GetStorage().Driver - } + total, pageObjs := pagination(filtered, &req.PageReq) + respContent := toObjsResp(pageObjs, reqPath, isEncrypt(meta, reqPath)) + pagesTotal := calcPagesTotal(total, req.PerPage) + hasMore := req.Page*req.PerPage < total + common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, reqPath), - Provider: provider, + Content: respContent, + Total: int64(total), + FilteredTotal: int64(total), + Page: req.Page, + PerPage: req.PerPage, + HasMore: hasMore, + PagesTotal: pagesTotal, + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, reqPath), + Provider: provider, }) } @@ -226,6 +247,28 @@ func isEncrypt(meta *model.Meta, path string) bool { return true } +func normalizeListPage(page, perPage int) (int, int) { + effPage := page + if effPage <= 0 { + effPage = 1 + } + effPerPage := perPage + if effPerPage <= 0 { + effPerPage = DefaultPerPage + } + if effPerPage > MaxPerPage { + effPerPage = MaxPerPage + } + return effPage, effPerPage +} + +func calcPagesTotal(total, perPage int) int { + if total <= 0 || perPage <= 0 { + return 0 + } + return (total + perPage - 1) / perPage +} + func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { pageIndex, pageSize := req.Page, req.PerPage total := len(objs) From 3f9933a0c275956dca2a25e8730b77a1b2bb465b Mon Sep 17 00:00:00 2001 From: Stanislav Chernov Date: Fri, 20 Feb 2026 23:23:12 +0300 Subject: [PATCH 593/659] fix: unescape URL path to handle '#' in filenames Fixes AlistGo/alist#9361 --- server/middlewares/down.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/middlewares/down.go b/server/middlewares/down.go index d015672de80..7e67663b74b 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -1,6 +1,7 @@ package middlewares import ( + "net/url" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -41,9 +42,8 @@ func Down(verifyFunc func(string, string) error) func(c *gin.Context) { } } -// TODO: implement -// path maybe contains # ? etc. func parsePath(path string) string { + path, _ = url.PathUnescape(path) return utils.FixAndCleanPath(path) } From cf01ff6f4e40253a5bbccb082a0f1b982c630849 Mon Sep 17 00:00:00 2001 From: Sky_slience Date: Sun, 1 Mar 2026 00:38:25 +0800 Subject: [PATCH 594/659] feat(thumbnail): add configurable thumbnail size setting and update thumbnail generation --- drivers/local/driver.go | 13 +++++++++++++ drivers/local/util.go | 4 ++-- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 7ff72d11cdf..5ba82786622 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -18,6 +18,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -31,6 +32,7 @@ type Local struct { model.Storage Addition mkdirPerm int32 + thumbSize int // zero means no limit thumbConcurrency int @@ -71,6 +73,17 @@ func (d *Local) Init(ctx context.Context) error { return err } } + d.thumbSize = 144 + if item, err := op.GetSettingItemByKey(conf.ThumbnailSize); err == nil && item != nil && strings.TrimSpace(item.Value) != "" { + v, err := strconv.ParseUint(item.Value, 10, 32) + if err != nil { + return fmt.Errorf("invalid setting %s value: %s, err: %s", conf.ThumbnailSize, item.Value, err) + } + if v == 0 { + return fmt.Errorf("invalid setting %s value: %s, the value must be a positive integer", conf.ThumbnailSize, item.Value) + } + d.thumbSize = int(v) + } if d.ThumbConcurrency != "" { v, err := strconv.ParseUint(d.ThumbConcurrency, 10, 32) if err != nil { diff --git a/drivers/local/util.go b/drivers/local/util.go index b9df717fb40..5a4a2be9107 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -111,7 +111,7 @@ func readDir(dirname string) ([]fs.FileInfo, error) { func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() thumbPrefix := "alist_thumb_" - thumbName := thumbPrefix + utils.GetMD5EncodeStr(fullPath) + ".png" + thumbName := thumbPrefix + utils.GetMD5EncodeStr(fmt.Sprintf("%s:%d", fullPath, d.thumbSize)) + ".png" if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail if strings.HasPrefix(file.GetName(), thumbPrefix) { @@ -142,7 +142,7 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { if err != nil { return nil, nil, err } - thumbImg := imaging.Resize(image, 144, 0, imaging.Lanczos) + thumbImg := imaging.Resize(image, d.thumbSize, 0, imaging.Lanczos) var buf bytes.Buffer err = imaging.Encode(&buf, thumbImg, imaging.PNG) if err != nil { diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index bbb633e33a6..006580888c6 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -146,6 +146,7 @@ func InitialSettings() []model.SettingItem { {Key: "audio_cover", Value: "https://jsd.nn.ci/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW}, {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.ThumbnailSize, Value: "144", Type: conf.TypeNumber, Group: model.PREVIEW, Help: "Thumbnail width in pixels. Height is scaled proportionally."}, {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 1a5581633a0..db2c84dc4ae 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -33,6 +33,7 @@ const ( ProxyIgnoreHeaders = "proxy_ignore_headers" AudioAutoplay = "audio_autoplay" VideoAutoplay = "video_autoplay" + ThumbnailSize = "thumbnail_size" PreviewArchivesByDefault = "preview_archives_by_default" ReadMeAutoRender = "readme_autorender" FilterReadMeScripts = "filter_readme_scripts" From 7dc8231e177bb1e602e18de5f4d18a113c4b6f30 Mon Sep 17 00:00:00 2001 From: Sky_slience Date: Sun, 1 Mar 2026 17:10:51 +0800 Subject: [PATCH 595/659] fix(rename): block rename for password-protected paths --- server/handles/fsbatch.go | 6 ++++++ server/handles/fsmanage.go | 22 +++++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index 35cec645e1b..bccbee72d29 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -195,6 +195,9 @@ func FsBatchRename(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if !canRenamePath(c, filePath) { + return + } if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { common.ErrorResp(c, err, 500) return @@ -261,6 +264,9 @@ func FsRegexRename(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if !canRenamePath(c, filePath) { + return + } newFileName := srcRegexp.ReplaceAllString(file.GetName(), req.NewNameRegex) if err := utils.ValidateNameComponent(newFileName); err != nil { common.ErrorResp(c, err, 400) diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index dcb5a7b9a09..31976edd9ad 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -2,10 +2,11 @@ package handles import ( "fmt" - "github.com/alist-org/alist/v3/internal/task" "io" stdpath "path" + "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" @@ -213,6 +214,22 @@ type RenameReq struct { Overwrite bool `json:"overwrite"` } +func canRenamePath(c *gin.Context, reqPath string) bool { + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return false + } + return true + } + if meta != nil && meta.Password != "" && common.IsApply(meta.Path, reqPath, meta.PSub) { + common.ErrorStrResp(c, "Path is password-protected and cannot be renamed.", 403) + return false + } + return true +} + func FsRename(c *gin.Context) { var req RenameReq if err := c.ShouldBind(&req); err != nil { @@ -229,6 +246,9 @@ func FsRename(c *gin.Context) { common.ErrorResp(c, errs.PermissionDenied, 403) return } + if !canRenamePath(c, reqPath) { + return + } perm := common.MergeRolePermissions(user, reqPath) if !common.HasPermission(perm, common.PermRename) { common.ErrorResp(c, errs.PermissionDenied, 403) From d4cc6efd4ef6dce7e832a43c13d1de280aca6f43 Mon Sep 17 00:00:00 2001 From: hgkdzbf6 Date: Fri, 6 Mar 2026 16:14:23 +0800 Subject: [PATCH 596/659] fix: reduce WebDAV logging for NotFoundError Skip logging full error stack trace for NotFoundError when a file doesn't exist, as this is a normal case and not a program error. Use DEBUG level instead of ERROR to reduce log noise. Fixes #9141 --- server/webdav.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/webdav.go b/server/webdav.go index ac520070686..d188cb8010d 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -9,6 +9,7 @@ import ( "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/server/middlewares" @@ -31,6 +32,12 @@ func WebDav(dav *gin.RouterGroup) { Prefix: path.Join(conf.URL.Path, "/dav"), LockSystem: webdav.NewMemLS(), Logger: func(request *http.Request, err error) { + // Skip logging for NotFoundError as it's not a program error + // but a normal case when a file doesn't exist + if errs.IsNotFoundError(err) { + log.Debugf("%s %s %v", request.Method, request.URL.Path, err) + return + } log.Errorf("%s %s %+v", request.Method, request.URL.Path, err) }, } From 1175f8ac3e2896ef714b0d3c0fdf1e3c1748e381 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 6 Mar 2026 21:44:36 +0800 Subject: [PATCH 597/659] feat: add streamtape driver --- drivers/all.go | 1 + drivers/streamtape/driver.go | 400 +++++++++++++++++++++++++++++++++++ drivers/streamtape/meta.go | 38 ++++ drivers/streamtape/types.go | 54 +++++ drivers/streamtape/util.go | 146 +++++++++++++ 5 files changed, 639 insertions(+) create mode 100644 drivers/streamtape/driver.go create mode 100644 drivers/streamtape/meta.go create mode 100644 drivers/streamtape/types.go create mode 100644 drivers/streamtape/util.go diff --git a/drivers/all.go b/drivers/all.go index c53ba6caddc..b40a7fd677a 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -66,6 +66,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/smb" + _ "github.com/alist-org/alist/v3/drivers/streamtape" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go new file mode 100644 index 00000000000..042df42cbe6 --- /dev/null +++ b/drivers/streamtape/driver.go @@ -0,0 +1,400 @@ +package streamtape + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + log "github.com/sirupsen/logrus" +) + +type Streamtape struct { + model.Storage + Addition +} + +var waitMoreSecondsRe = regexp.MustCompile(`wait\s+(\d+)\s+more\s+seconds?`) + +func (d *Streamtape) Config() driver.Config { + return config +} + +func (d *Streamtape) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Streamtape) Init(ctx context.Context) error { + if strings.TrimSpace(d.APILogin) == "" || strings.TrimSpace(d.APIKey) == "" { + return errors.New("api_login and api_key are required") + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + + var account accountInfo + if err := d.callAPI(ctx, "/account/info", nil, &account); err != nil { + return err + } + + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Streamtape) Drop(ctx context.Context) error { + return nil +} + +func (d *Streamtape) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = folderIDFromObjID(dir.GetID()) + } + + params := map[string]string{} + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + + var result listFolderResult + if err := d.callAPI(ctx, "/file/listfolder", params, &result); err != nil { + return nil, err + } + + objects := make([]model.Obj, 0, len(result.Folders)+len(result.Files)) + for _, f := range result.Folders { + objects = append(objects, &model.Object{ + ID: encodeFolderID(f.ID), + Name: f.Name, + IsFolder: true, + }) + } + for _, f := range result.Files { + objects = append(objects, buildFileObj(f)) + } + return objects, nil +} + +func (d *Streamtape) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := fileIDFromObjID(file.GetID()) + if fileID == "" { + return nil, errors.New("empty file id") + } + + var ticket dlTicketResult + if err := d.callAPI(ctx, "/file/dlticket", map[string]string{"file": fileID}, &ticket); err != nil { + return nil, err + } + + var dl dlResult + waitSeconds := ticket.WaitTime + if waitSeconds > 0 { + timer := time.NewTimer(time.Duration(waitSeconds+1) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + var err error + for i := 0; i < 3; i++ { + err = d.callAPI(ctx, "/file/dl", map[string]string{ + "file": fileID, + "ticket": ticket.Ticket, + }, &dl) + if err == nil { + break + } + waitSeconds = extractWaitSecondsFromErr(err) + if waitSeconds <= 0 { + return nil, err + } + timer := time.NewTimer(time.Duration(waitSeconds+1) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + if err != nil { + return nil, err + } + + finalURL := ensureStreamQuery(dl.URL) + log.Infof("streamtape direct link file=%s url=%s", fileID, finalURL) + link := &model.Link{ + URL: finalURL, + Header: http.Header{ + "Referer": []string{"https://streamtape.com/"}, + "Origin": []string{"https://streamtape.com"}, + }, + } + d.applyRangeStrategy(link, file.GetSize()) + return link, nil +} + +func extractWaitSecondsFromErr(err error) int { + if err == nil { + return 0 + } + matches := waitMoreSecondsRe.FindStringSubmatch(strings.ToLower(err.Error())) + if len(matches) < 2 { + return 0 + } + seconds, convErr := strconv.Atoi(matches[1]) + if convErr != nil || seconds < 0 { + return 0 + } + return seconds +} + +func ensureStreamQuery(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + q := u.Query() + if q.Get("stream") == "" { + q.Set("stream", "1") + u.RawQuery = q.Encode() + } + return u.String() +} + +func (d *Streamtape) applyRangeStrategy(link *model.Link, size int64) { + if !d.EnableRangeControl || size <= 0 { + return + } + + mode := strings.ToLower(strings.TrimSpace(d.RangeMode)) + if mode == "" { + mode = "chunk" + } + + switch mode { + case "full": + // Keep single full-tail behavior while still using ranged requests. + link.Concurrency = 1 + link.PartSize = int(size) + case "percent": + percent := d.RangePercent + if percent <= 0 { + percent = 15 + } + if percent > 100 { + percent = 100 + } + partSize := size * int64(percent) / 100 + if partSize < 1*1024*1024 { + partSize = 1 * 1024 * 1024 + } + if partSize > size { + partSize = size + } + link.Concurrency = 1 + link.PartSize = int(partSize) + default: + chunkMB := d.RangeChunkMB + if chunkMB <= 0 { + chunkMB = 8 + } + partSize := int64(chunkMB) * 1024 * 1024 + if partSize > size { + partSize = size + } + concurrency := d.RangeConcurrency + if concurrency <= 0 { + concurrency = 4 + } + link.Concurrency = concurrency + link.PartSize = int(partSize) + } +} + +func (d *Streamtape) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + pid := d.RootFolderID + if parentDir.GetID() != "" { + pid = folderIDFromObjID(parentDir.GetID()) + } + + params := map[string]string{"name": dirName} + if pid != "" && pid != "0" { + params["pid"] = pid + } + + var result createFolderResult + if err := d.callAPI(ctx, "/file/createfolder", params, &result); err != nil { + return nil, err + } + + return &model.Object{ + ID: encodeFolderID(result.FolderID), + Name: dirName, + IsFolder: true, + }, nil +} + +func (d *Streamtape) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, errs.NotImplement + } + fileID := fileIDFromObjID(srcObj.GetID()) + if fileID == "" { + return nil, errors.New("empty file id") + } + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + if folderID == "" || folderID == "0" { + return nil, fmt.Errorf("streamtape move to root is not supported by API") + } + + if err := d.callAPI(ctx, "/file/move", map[string]string{ + "file": fileID, + "folder": folderID, + }, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: false, + }, nil +} + +func (d *Streamtape) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + endpoint := "/file/rename" + params := map[string]string{"name": newName} + if srcObj.IsDir() { + endpoint = "/file/renamefolder" + params["folder"] = folderIDFromObjID(srcObj.GetID()) + } else { + params["file"] = fileIDFromObjID(srcObj.GetID()) + } + + if err := d.callAPI(ctx, endpoint, params, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Streamtape) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Remove(ctx context.Context, obj model.Obj) error { + endpoint := "/file/delete" + params := map[string]string{} + if obj.IsDir() { + endpoint = "/file/deletefolder" + params["folder"] = folderIDFromObjID(obj.GetID()) + } else { + params["file"] = fileIDFromObjID(obj.GetID()) + } + return d.callAPI(ctx, endpoint, params, nil) +} + +func (d *Streamtape) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + params := map[string]string{} + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + + var uploadURL uploadURLResult + if err := d.callAPI(ctx, "/file/ul", params, &uploadURL); err != nil { + return nil, err + } + + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) + + res, err := base.RestyClient.R(). + SetContext(ctx). + SetFileReader("file1", file.GetName(), reader). + Post(uploadURL.URL) + if err != nil { + return nil, err + } + if res.StatusCode() >= http.StatusBadRequest { + return nil, fmt.Errorf("streamtape upload failed: http %d", res.StatusCode()) + } + + uploadedID := extractFileIDFromUploadBody(res.Body()) + if uploadedID == "" { + list, listErr := d.List(ctx, &model.Object{ID: encodeFolderID(folderID), IsFolder: true}, model.ListArgs{}) + if listErr == nil { + for _, obj := range list { + if obj.IsDir() { + continue + } + if obj.GetName() == file.GetName() && (file.GetSize() <= 0 || obj.GetSize() == file.GetSize()) { + return obj, nil + } + } + } + return &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil + } + + return &model.Object{ + ID: encodeFileID(uploadedID), + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Streamtape) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Streamtape)(nil) diff --git a/drivers/streamtape/meta.go b/drivers/streamtape/meta.go new file mode 100644 index 00000000000..50b09f14d41 --- /dev/null +++ b/drivers/streamtape/meta.go @@ -0,0 +1,38 @@ +package streamtape + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APILogin string `json:"api_login" required:"true" help:"API Login from Streamtape account settings"` + APIKey string `json:"api_key" required:"true" help:"API Key from Streamtape account settings"` + RangeMode string `json:"range_mode" type:"select" options:"chunk,full,percent" default:"chunk" help:"Range strategy for preview: chunk=bounded ranges, full=single full-tail range, percent=part size by file percentage"` + RangeChunkMB int `json:"range_chunk_mb" type:"number" default:"8" help:"Chunk mode part size in MB"` + RangeConcurrency int `json:"range_concurrency" type:"number" default:"4" help:"Chunk mode concurrent upstream requests"` + RangePercent int `json:"range_percent" type:"number" default:"15" help:"Percent mode part size percentage (1-100)"` + EnableRangeControl bool `json:"enable_range_control" default:"true" help:"Enable driver-level range shaping for smoother streaming"` +} + +var config = driver.Config{ + Name: "Streamtape", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, + ProxyRangeOption: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Streamtape{} + }) +} diff --git a/drivers/streamtape/types.go b/drivers/streamtape/types.go new file mode 100644 index 00000000000..4e470ff927e --- /dev/null +++ b/drivers/streamtape/types.go @@ -0,0 +1,54 @@ +package streamtape + +import "encoding/json" + +type apiResponse struct { + Status int `json:"status"` + Msg string `json:"msg"` + Result json.RawMessage `json:"result"` +} + +type accountInfo struct { + APIID string `json:"apiid"` + Email string `json:"email"` + SignupAt string `json:"signup_at"` +} + +type listFolderResult struct { + Folders []folderItem `json:"folders"` + Files []fileItem `json:"files"` +} + +type folderItem struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type fileItem struct { + Name string `json:"name"` + Size int64 `json:"size"` + Link string `json:"link"` + CreatedAt int64 `json:"created_at"` + Downloads int64 `json:"downloads"` + LinkID string `json:"linkid"` + Convert string `json:"convert"` +} + +type dlTicketResult struct { + Ticket string `json:"ticket"` + WaitTime int `json:"wait_time"` +} + +type dlResult struct { + Name string `json:"name"` + Size int64 `json:"size"` + URL string `json:"url"` +} + +type createFolderResult struct { + FolderID string `json:"folderid"` +} + +type uploadURLResult struct { + URL string `json:"url"` +} diff --git a/drivers/streamtape/util.go b/drivers/streamtape/util.go new file mode 100644 index 00000000000..f6d8373763e --- /dev/null +++ b/drivers/streamtape/util.go @@ -0,0 +1,146 @@ +package streamtape + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" +) + +const apiBase = "https://api.streamtape.com" + +func (d *Streamtape) callAPI(ctx context.Context, endpoint string, params map[string]string, out any) error { + query := map[string]string{ + "login": d.APILogin, + "key": d.APIKey, + } + for k, v := range params { + if strings.TrimSpace(v) == "" { + continue + } + query[k] = v + } + + var resp apiResponse + r, err := base.RestyClient.R(). + SetContext(ctx). + SetQueryParams(query). + SetResult(&resp). + Get(apiBase + endpoint) + if err != nil { + return err + } + if r.StatusCode() != http.StatusOK { + return fmt.Errorf("streamtape http error: %d", r.StatusCode()) + } + if resp.Status != 200 { + return fmt.Errorf("streamtape api error: status=%d msg=%s", resp.Status, resp.Msg) + } + if out == nil || len(resp.Result) == 0 || string(resp.Result) == "null" { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("decode streamtape result failed: %w", err) + } + return nil +} + +func folderIDFromObjID(id string) string { + if id == "" || id == "0" || id == "/" { + return "0" + } + if strings.HasPrefix(id, "d:") { + return strings.TrimPrefix(id, "d:") + } + return id +} + +func fileIDFromObjID(id string) string { + if strings.HasPrefix(id, "f:") { + return strings.TrimPrefix(id, "f:") + } + return id +} + +func encodeFolderID(id string) string { + if id == "" || id == "0" || id == "/" { + return "d:0" + } + return "d:" + id +} + +func encodeFileID(id string) string { + if strings.HasPrefix(id, "f:") { + return id + } + return "f:" + id +} + +func extractFileIDFromLink(link string) string { + if link == "" { + return "" + } + u, err := url.Parse(link) + if err != nil { + return "" + } + parts := strings.Split(strings.Trim(path.Clean(u.Path), "/"), "/") + for i := 0; i < len(parts)-1; i++ { + if parts[i] == "v" { + return parts[i+1] + } + } + return "" +} + +func buildFileObj(f fileItem) model.Obj { + id := f.LinkID + if id == "" { + id = extractFileIDFromLink(f.Link) + } + mod := time.Now() + if f.CreatedAt > 0 { + mod = time.Unix(f.CreatedAt, 0) + } + return &model.Object{ + ID: encodeFileID(id), + Name: f.Name, + Size: f.Size, + Modified: mod, + IsFolder: false, + } +} + +func extractFileIDFromUploadBody(body []byte) string { + if len(body) == 0 { + return "" + } + + var resp apiResponse + if err := json.Unmarshal(body, &resp); err != nil { + return "" + } + if resp.Status != 200 || len(resp.Result) == 0 { + return "" + } + + var result map[string]any + if err := json.Unmarshal(resp.Result, &result); err != nil { + return "" + } + for _, key := range []string{"file", "fileid", "id", "linkid"} { + if v, ok := result[key]; ok { + if s, ok := v.(string); ok && s != "" { + return s + } + } + } + return "" +} From 5773c637e2a5331596cf1311165f3f01fd73888a Mon Sep 17 00:00:00 2001 From: Zhongwen Luo <75424880+Muione@users.noreply.github.com> Date: Sat, 7 Mar 2026 10:32:29 +0000 Subject: [PATCH 598/659] fix: handle nil buffer case in resizeImageToBufferWithFFmpegGo --- drivers/local/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/local/util.go b/drivers/local/util.go index 6c59911b4c8..fa95ddd24a0 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -80,7 +80,7 @@ func resizeImageToBufferWithFFmpegGo(inputFile string, width int, outputFormat s if err != nil { return nil, fmt.Errorf("ffmpeg-go failed to resize image %s to buffer: %w", inputFile, err) } - if outBuffer.Len() == 0 { + if outBuffer == nil || outBuffer.Len() == 0 { return nil, fmt.Errorf("ffmpeg-go produced empty buffer for %s", inputFile) } From 5852a81773e506323710d4a41cf3d8a2c70942b2 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Mon, 9 Mar 2026 19:45:16 +0800 Subject: [PATCH 599/659] feat(strm): add STRM driver with independent sync flow - implement STRM mount and generate behavior with alias mapping - support local save modes: insert, update, sync - keep config compatibility and remove previous intermediate history --- drivers/all.go | 1 + drivers/strm/driver.go | 237 ++++++++++++++++++++++++++++++++ drivers/strm/hook.go | 3 + drivers/strm/meta.go | 42 ++++++ drivers/strm/util.go | 305 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 588 insertions(+) create mode 100644 drivers/strm/driver.go create mode 100644 drivers/strm/hook.go create mode 100644 drivers/strm/meta.go create mode 100644 drivers/strm/util.go diff --git a/drivers/all.go b/drivers/all.go index c53ba6caddc..a9130555fd6 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -66,6 +66,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/smb" + _ "github.com/alist-org/alist/v3/drivers/strm" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go new file mode 100644 index 00000000000..c0f3a27d0ac --- /dev/null +++ b/drivers/strm/driver.go @@ -0,0 +1,237 @@ +package strm + +import ( + "context" + "errors" + stdpath "path" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" +) + +type Strm struct { + model.Storage + Addition + + aliases map[string][]string + autoFlatten bool + singleRootKey string + + mediaExtSet map[string]struct{} + downloadExtSet map[string]struct{} + normalizedMode string + normalizedPrefix string +} + +func (d *Strm) Config() driver.Config { + return config +} + +func (d *Strm) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Strm) Init(ctx context.Context) error { + if strings.TrimSpace(d.Paths) == "" { + return errors.New("paths is required") + } + if d.SaveStrmToLocal && strings.TrimSpace(d.SaveStrmLocalPath) == "" { + return errors.New("SaveStrmLocalPath is required") + } + + d.aliases = parseAliases(d.Paths) + if len(d.aliases) == 0 { + return errors.New("no valid path mapping found") + } + + d.autoFlatten = len(d.aliases) == 1 + d.singleRootKey = "" + if d.autoFlatten { + for k := range d.aliases { + d.singleRootKey = k + } + } + + d.mediaExtSet = parseExtSet(defaultIfEmpty(d.FilterFileTypes, defaultMediaExt)) + d.downloadExtSet = parseExtSet(defaultIfEmpty(d.DownloadFileTypes, defaultDownloadExt)) + d.normalizedPrefix = normalizePrefix(defaultIfEmpty(d.PathPrefix, "/d")) + d.normalizedMode = normalizeSaveMode(d.SaveLocalMode) + + if d.Version != 5 { + d.FilterFileTypes = mergeDefaultExtCSV(d.FilterFileTypes, defaultMediaExt) + d.DownloadFileTypes = mergeDefaultExtCSV(d.DownloadFileTypes, defaultDownloadExt) + d.PathPrefix = "/d" + d.Version = 5 + } + if d.SaveLocalMode == "" { + d.SaveLocalMode = SaveLocalInsertMode + } + return nil +} + +func (d *Strm) Drop(ctx context.Context) error { + d.aliases = nil + d.mediaExtSet = nil + d.downloadExtSet = nil + return nil +} + +func (Addition) GetRootPath() string { + return "/" +} + +func (d *Strm) Get(ctx context.Context, path string) (model.Obj, error) { + path = cleanPath(path) + root, sub := d.splitVirtualPath(path) + targets, ok := d.aliases[root] + if !ok { + return nil, errs.ObjectNotFound + } + + for _, targetRoot := range targets { + realPath := stdpath.Join(targetRoot, sub) + obj, err := fs.Get(ctx, realPath, &fs.GetArgs{NoLog: true}) + if err != nil { + continue + } + if obj.IsDir() { + return wrapObj(path, obj, 0), nil + } + return wrapObj(realPath, obj, obj.GetSize()), nil + } + + if strings.HasSuffix(strings.ToLower(path), ".strm") { + return nil, errs.NotSupport + } + return nil, errs.ObjectNotFound +} + +func (d *Strm) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + virtualDir := cleanPath(dir.GetPath()) + if virtualDir == "/" && !d.autoFlatten { + objs := d.listVirtualRoots() + d.syncLocalDir(ctx, virtualDir, objs) + return objs, nil + } + + root, sub := d.splitVirtualPath(virtualDir) + targets, ok := d.aliases[root] + if !ok { + return nil, errs.ObjectNotFound + } + + out := make([]model.Obj, 0) + for _, targetRoot := range targets { + realDir := stdpath.Join(targetRoot, sub) + objs, err := fs.List(ctx, realDir, &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) + if err != nil { + continue + } + out = append(out, d.mapListedObjects(ctx, realDir, objs)...) + } + + d.syncLocalDir(ctx, virtualDir, out) + return out, nil +} + +func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.GetID() == "strm" { + line := d.buildStrmLine(ctx, file.GetPath()) + return &model.Link{MFile: model.NewNopMFile(strings.NewReader(line + "\n"))}, nil + } + return d.linkRealFile(ctx, file.GetPath(), args) +} + +func (d *Strm) listVirtualRoots() []model.Obj { + objs := make([]model.Obj, 0, len(d.aliases)) + for k := range d.aliases { + objs = append(objs, &model.Object{ + Path: "/" + k, + Name: k, + IsFolder: true, + Modified: d.Modified, + }) + } + return objs +} + +func (d *Strm) mapListedObjects(ctx context.Context, realDir string, listed []model.Obj) []model.Obj { + ret := make([]model.Obj, 0, len(listed)) + for _, obj := range listed { + if obj.IsDir() { + ret = append(ret, &model.Object{ + Name: obj.GetName(), + Path: "", + IsFolder: true, + Modified: obj.ModTime(), + }) + continue + } + + realPath := stdpath.Join(realDir, obj.GetName()) + ext := fileExt(obj.GetName()) + + if _, ok := d.downloadExtSet[ext]; ok { + ret = append(ret, d.cloneWithPath(obj, realPath, obj.GetName(), "", obj.GetSize())) + continue + } + if _, ok := d.mediaExtSet[ext]; ok { + strmName := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) + ".strm" + size := int64(len(d.buildStrmLine(ctx, realPath)) + 1) + ret = append(ret, d.cloneWithPath(obj, realPath, strmName, "strm", size)) + } + } + return ret +} + +func (d *Strm) cloneWithPath(src model.Obj, realPath, name, id string, size int64) model.Obj { + baseObj := model.Object{ + ID: id, + Path: realPath, + Name: name, + Size: size, + Modified: src.ModTime(), + IsFolder: src.IsDir(), + } + thumb, ok := model.GetThumb(src) + if !ok { + return &baseObj + } + return &model.ObjThumb{Object: baseObj, Thumbnail: model.Thumbnail{Thumbnail: thumb}} +} + +func (d *Strm) splitVirtualPath(path string) (string, string) { + if d.autoFlatten { + return d.singleRootKey, path + } + trimmed := strings.TrimPrefix(path, "/") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) == 1 { + return parts[0], "" + } + return parts[0], parts[1] +} + +func cleanPath(path string) string { + if path == "" { + return "/" + } + return filepath.ToSlash(stdpath.Clean("/" + strings.TrimPrefix(path, "/"))) +} + +func wrapObj(path string, src model.Obj, size int64) model.Obj { + return &model.Object{ + Path: path, + Name: src.GetName(), + Size: size, + Modified: src.ModTime(), + IsFolder: src.IsDir(), + HashInfo: src.GetHash(), + } +} + +var _ driver.Driver = (*Strm)(nil) diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go new file mode 100644 index 00000000000..6fad6993c67 --- /dev/null +++ b/drivers/strm/hook.go @@ -0,0 +1,3 @@ +package strm + +// Local sync is triggered during STRM directory listing. diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go new file mode 100644 index 00000000000..ac757e9ebad --- /dev/null +++ b/drivers/strm/meta.go @@ -0,0 +1,42 @@ +package strm + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + SaveLocalInsertMode = "insert" + SaveLocalUpdateMode = "update" + SaveLocalSyncMode = "sync" +) + +type Addition struct { + Paths string `json:"paths" required:"true" type:"text"` + SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of generated strm file"` + PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix in strm content"` + DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Extensions to download as local files"` + FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Extensions to expose as .strm"` + EncodePath bool `json:"encodePath" default:"true" required:"true" help:"Encode path in strm content"` + WithoutUrl bool `json:"withoutUrl" default:"false" help:"Generate path-only strm content"` + WithSign bool `json:"withSign" default:"false" help:"Append sign query to generated URL"` + SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"Save generated files to local disk"` + SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"Local path for generated files"` + SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"Local save mode" options:"insert,update,sync" default:"insert"` + Version int +} + +var config = driver.Config{ + Name: "Strm", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + NoUpload: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Strm{Addition: Addition{EncodePath: true}} + }) +} diff --git a/drivers/strm/util.go b/drivers/strm/util.go new file mode 100644 index 00000000000..4cba1769d27 --- /dev/null +++ b/drivers/strm/util.go @@ -0,0 +1,305 @@ +package strm + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + stdpath "path" + "path/filepath" + "sort" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + pkgerr "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +const ( + defaultMediaExt = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" + defaultDownloadExt = "ass,srt,vtt,sub,strm" +) + +func parseAliases(raw string) map[string][]string { + aliases := map[string][]string{} + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + name, target := parseAliasLine(line) + aliases[name] = append(aliases[name], cleanPath(target)) + } + return aliases +} + +func parseAliasLine(line string) (string, string) { + if strings.Contains(line, ":") { + parts := strings.SplitN(line, ":", 2) + if !strings.Contains(parts[0], "/") { + return parts[0], parts[1] + } + } + return stdpath.Base(line), line +} + +func parseExtSet(csv string) map[string]struct{} { + ret := map[string]struct{}{} + for _, part := range strings.Split(csv, ",") { + ext := normalizeExt(part) + if ext != "" { + ret[ext] = struct{}{} + } + } + return ret +} + +func mergeDefaultExtCSV(csv, defaults string) string { + base := parseExtSet(csv) + for ext := range parseExtSet(defaults) { + base[ext] = struct{}{} + } + keys := make([]string, 0, len(base)) + for k := range base { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +func normalizeExt(ext string) string { + ext = strings.ToLower(strings.TrimSpace(ext)) + ext = strings.TrimPrefix(ext, ".") + return ext +} + +func fileExt(name string) string { + return normalizeExt(stdpath.Ext(name)) +} + +func defaultIfEmpty(v, fallback string) string { + if strings.TrimSpace(v) == "" { + return fallback + } + return v +} + +func normalizeSaveMode(mode string) string { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "sync": + return SaveLocalSyncMode + case "update": + return SaveLocalUpdateMode + case "insert", "missing": + return SaveLocalInsertMode + default: + return SaveLocalInsertMode + } +} + +func normalizePrefix(prefix string) string { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return "/d" + } + if !strings.HasPrefix(prefix, "/") { + prefix = "/" + prefix + } + return prefix +} + +func (d *Strm) buildStrmLine(ctx context.Context, realPath string) string { + pathPart := realPath + if d.EncodePath { + pathPart = utils.EncodePath(pathPart, true) + } + if d.WithSign { + sep := "?" + if strings.Contains(pathPart, "?") { + sep = "&" + } + pathPart += sep + "sign=" + sign.Sign(realPath) + } + joined := stdpath.Join(d.normalizedPrefix, pathPart) + if !strings.HasPrefix(joined, "/") { + joined = "/" + joined + } + if d.WithoutUrl { + return joined + } + baseURL := strings.TrimSpace(d.SiteUrl) + if baseURL == "" { + if c, ok := ctx.(*gin.Context); ok { + baseURL = common.GetApiUrl(c.Request) + } else { + baseURL = common.GetApiUrl(nil) + } + } + baseURL = strings.TrimSuffix(baseURL, "/") + return baseURL + joined +} + +func (d *Strm) linkRealFile(ctx context.Context, realPath string, args model.LinkArgs) (*model.Link, error) { + storage, actualPath, err := op.GetStorageAndActualPath(realPath) + if err != nil { + return nil, err + } + if !args.Redirect { + link, _, linkErr := op.Link(ctx, storage, actualPath, args) + return link, linkErr + } + obj, err := fs.Get(ctx, realPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err + } + if common.ShouldProxy(storage, obj.GetName()) { + api := common.GetApiUrl(args.HttpReq) + if api == "" { + api = strings.TrimSuffix(strings.TrimSpace(d.SiteUrl), "/") + } + if api == "" { + api = common.GetApiUrl(nil) + } + return &model.Link{URL: fmt.Sprintf("%s/p%s?sign=%s", api, utils.EncodePath(realPath, true), sign.Sign(realPath))}, nil + } + link, _, linkErr := op.Link(ctx, storage, actualPath, args) + return link, linkErr +} + +func (d *Strm) syncLocalDir(ctx context.Context, virtualDir string, objs []model.Obj) { + if !d.SaveStrmToLocal || strings.TrimSpace(d.SaveStrmLocalPath) == "" { + return + } + baseDir := filepath.Clean(d.SaveStrmLocalPath) + localDir := baseDir + if virtualDir != "/" { + localDir = filepath.Join(baseDir, filepath.FromSlash(strings.TrimPrefix(virtualDir, "/"))) + } + if err := os.MkdirAll(localDir, 0o755); err != nil { + log.Warnf("strm: mkdir failed %s: %v", localDir, err) + return + } + + expected := map[string]bool{} + for _, obj := range objs { + name := obj.GetName() + expected[name] = obj.IsDir() + localPath := filepath.Join(localDir, name) + if obj.IsDir() { + _ = os.MkdirAll(localPath, 0o755) + continue + } + payload, err := d.localPayload(ctx, obj) + if err != nil { + log.Warnf("strm: build local payload failed %s: %v", localPath, err) + continue + } + if err = d.writeLocal(localPath, payload); err != nil { + log.Warnf("strm: write local failed %s: %v", localPath, err) + } + } + + if d.normalizedMode == SaveLocalSyncMode { + d.syncDeleteExtras(localDir, expected) + } +} + +func (d *Strm) localPayload(ctx context.Context, obj model.Obj) ([]byte, error) { + if obj.GetID() == "strm" { + return []byte(d.buildStrmLine(ctx, obj.GetPath()) + "\n"), nil + } + link, err := d.linkRealFile(ctx, obj.GetPath(), model.LinkArgs{Redirect: true}) + if err != nil { + return nil, err + } + return readLinkBytes(ctx, link) +} + +func readLinkBytes(ctx context.Context, link *model.Link) ([]byte, error) { + if link.MFile != nil { + defer link.MFile.Close() + return io.ReadAll(link.MFile) + } + if link.RangeReadCloser != nil { + rc, err := link.RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) + if err == nil && rc != nil { + defer rc.Close() + return io.ReadAll(rc) + } + } + if link.URL == "" { + return nil, fmt.Errorf("empty link") + } + url := link.URL + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + api := common.GetApiUrl(nil) + if api == "" { + return nil, fmt.Errorf("relative url without site url: %s", url) + } + url = strings.TrimSuffix(api, "/") + url + } + res, err := base.RestyClient.R().SetContext(ctx).SetDoNotParseResponse(true).Get(url) + if err != nil { + return nil, err + } + defer res.RawBody().Close() + if res.StatusCode() >= http.StatusBadRequest { + return nil, fmt.Errorf("read url failed: status=%d", res.StatusCode()) + } + return io.ReadAll(res.RawBody()) +} + +func (d *Strm) writeLocal(path string, payload []byte) error { + if d.normalizedMode == SaveLocalInsertMode && utils.Exists(path) { + return nil + } + if st, err := os.Stat(path); err == nil && st.IsDir() { + if d.normalizedMode != SaveLocalSyncMode { + return nil + } + if err = os.RemoveAll(path); err != nil { + return err + } + } + if d.normalizedMode != SaveLocalInsertMode { + if old, err := os.ReadFile(path); err == nil { + if bytes.Equal(old, payload) { + return nil + } + } + } + f, err := utils.CreateNestedFile(path) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(payload) + return err +} + +func (d *Strm) syncDeleteExtras(localDir string, expected map[string]bool) { + entries, err := os.ReadDir(localDir) + if err != nil { + if pkgerr.Cause(err) != os.ErrNotExist { + log.Warnf("strm: read local dir failed %s: %v", localDir, err) + } + return + } + for _, e := range entries { + expectDir, ok := expected[e.Name()] + full := filepath.Join(localDir, e.Name()) + if !ok || expectDir != e.IsDir() { + _ = os.RemoveAll(full) + } + } +} From 866d4f4c3551849c679b46bd13ac7d613519be34 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 10 Mar 2026 11:27:46 +0800 Subject: [PATCH 600/659] feat(strm): add driver-level sign expiry and rotate action - add SignExpireHours for STRM-specific sign duration - add RotateSignNow to trigger immediate local STRM rewrite - auto reset rotate flag after scheduling background rotation --- drivers/strm/driver.go | 46 ++++++++++++++++++++++++++++++++++++++++++ drivers/strm/meta.go | 2 ++ drivers/strm/util.go | 28 +++++++++++++++++-------- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go index c0f3a27d0ac..175a558dfcc 100644 --- a/drivers/strm/driver.go +++ b/drivers/strm/driver.go @@ -11,6 +11,8 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + log "github.com/sirupsen/logrus" ) type Strm struct { @@ -70,6 +72,20 @@ func (d *Strm) Init(ctx context.Context) error { if d.SaveLocalMode == "" { d.SaveLocalMode = SaveLocalInsertMode } + if d.SignExpireHours < 0 { + d.SignExpireHours = 0 + } + if d.RotateSignNow { + d.RotateSignNow = false + op.MustSaveDriverStorage(d) + if d.SaveStrmToLocal && strings.TrimSpace(d.SaveStrmLocalPath) != "" { + go func() { + log.Infof("strm: start rotating signs for [%s]", d.MountPath) + d.rotateAllLocal(context.Background()) + log.Infof("strm: finished rotating signs for [%s]", d.MountPath) + }() + } + } return nil } @@ -159,6 +175,36 @@ func (d *Strm) listVirtualRoots() []model.Obj { return objs } +func (d *Strm) rotateAllLocal(ctx context.Context) { + for alias, roots := range d.aliases { + virtualRoot := "/" + if !d.autoFlatten { + virtualRoot = "/" + alias + } + for _, realRoot := range roots { + d.walkAndSync(ctx, virtualRoot, realRoot) + } + } +} + +func (d *Strm) walkAndSync(ctx context.Context, virtualDir, realDir string) { + objs, err := fs.List(ctx, realDir, &fs.ListArgs{NoLog: true, Refresh: true}) + if err != nil { + log.Warnf("strm: rotate list failed %s: %v", realDir, err) + return + } + mapped := d.mapListedObjects(ctx, realDir, objs) + d.syncLocalDirWithMode(ctx, virtualDir, mapped, SaveLocalUpdateMode) + for _, obj := range objs { + if !obj.IsDir() { + continue + } + childVirtual := stdpath.Join(virtualDir, obj.GetName()) + childReal := stdpath.Join(realDir, obj.GetName()) + d.walkAndSync(ctx, childVirtual, childReal) + } +} + func (d *Strm) mapListedObjects(ctx context.Context, realDir string, listed []model.Obj) []model.Obj { ret := make([]model.Obj, 0, len(listed)) for _, obj := range listed { diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go index ac757e9ebad..803ed035762 100644 --- a/drivers/strm/meta.go +++ b/drivers/strm/meta.go @@ -20,6 +20,8 @@ type Addition struct { EncodePath bool `json:"encodePath" default:"true" required:"true" help:"Encode path in strm content"` WithoutUrl bool `json:"withoutUrl" default:"false" help:"Generate path-only strm content"` WithSign bool `json:"withSign" default:"false" help:"Append sign query to generated URL"` + SignExpireHours int `json:"SignExpireHours" type:"number" default:"0" help:"Driver-level sign expiration in hours. 0 uses global link_expiration"` + RotateSignNow bool `json:"RotateSignNow" type:"bool" default:"false" help:"Set true and save to rotate signs now (rewrite local STRM), then auto reset to false"` SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"Save generated files to local disk"` SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"Local path for generated files"` SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"Local save mode" options:"insert,update,sync" default:"insert"` diff --git a/drivers/strm/util.go b/drivers/strm/util.go index 4cba1769d27..2d640bc32fb 100644 --- a/drivers/strm/util.go +++ b/drivers/strm/util.go @@ -11,6 +11,7 @@ import ( "path/filepath" "sort" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/fs" @@ -128,7 +129,7 @@ func (d *Strm) buildStrmLine(ctx context.Context, realPath string) string { if strings.Contains(pathPart, "?") { sep = "&" } - pathPart += sep + "sign=" + sign.Sign(realPath) + pathPart += sep + "sign=" + d.generateSign(realPath) } joined := stdpath.Join(d.normalizedPrefix, pathPart) if !strings.HasPrefix(joined, "/") { @@ -170,13 +171,17 @@ func (d *Strm) linkRealFile(ctx context.Context, realPath string, args model.Lin if api == "" { api = common.GetApiUrl(nil) } - return &model.Link{URL: fmt.Sprintf("%s/p%s?sign=%s", api, utils.EncodePath(realPath, true), sign.Sign(realPath))}, nil + return &model.Link{URL: fmt.Sprintf("%s/p%s?sign=%s", api, utils.EncodePath(realPath, true), d.generateSign(realPath))}, nil } link, _, linkErr := op.Link(ctx, storage, actualPath, args) return link, linkErr } func (d *Strm) syncLocalDir(ctx context.Context, virtualDir string, objs []model.Obj) { + d.syncLocalDirWithMode(ctx, virtualDir, objs, d.normalizedMode) +} + +func (d *Strm) syncLocalDirWithMode(ctx context.Context, virtualDir string, objs []model.Obj, mode string) { if !d.SaveStrmToLocal || strings.TrimSpace(d.SaveStrmLocalPath) == "" { return } @@ -204,12 +209,12 @@ func (d *Strm) syncLocalDir(ctx context.Context, virtualDir string, objs []model log.Warnf("strm: build local payload failed %s: %v", localPath, err) continue } - if err = d.writeLocal(localPath, payload); err != nil { + if err = d.writeLocal(localPath, payload, mode); err != nil { log.Warnf("strm: write local failed %s: %v", localPath, err) } } - if d.normalizedMode == SaveLocalSyncMode { + if mode == SaveLocalSyncMode { d.syncDeleteExtras(localDir, expected) } } @@ -259,19 +264,19 @@ func readLinkBytes(ctx context.Context, link *model.Link) ([]byte, error) { return io.ReadAll(res.RawBody()) } -func (d *Strm) writeLocal(path string, payload []byte) error { - if d.normalizedMode == SaveLocalInsertMode && utils.Exists(path) { +func (d *Strm) writeLocal(path string, payload []byte, mode string) error { + if mode == SaveLocalInsertMode && utils.Exists(path) { return nil } if st, err := os.Stat(path); err == nil && st.IsDir() { - if d.normalizedMode != SaveLocalSyncMode { + if mode != SaveLocalSyncMode { return nil } if err = os.RemoveAll(path); err != nil { return err } } - if d.normalizedMode != SaveLocalInsertMode { + if mode != SaveLocalInsertMode { if old, err := os.ReadFile(path); err == nil { if bytes.Equal(old, payload) { return nil @@ -303,3 +308,10 @@ func (d *Strm) syncDeleteExtras(localDir string, expected map[string]bool) { } } } + +func (d *Strm) generateSign(path string) string { + if d.SignExpireHours > 0 { + return sign.WithDuration(path, time.Duration(d.SignExpireHours)*time.Hour) + } + return sign.Sign(path) +} From faa596c8d486fddef17a1b72c4fa1fb920bc95de Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 10 Mar 2026 19:41:02 +0800 Subject: [PATCH 601/659] feat(quark): support 302 with transcoding and stable video preview --- drivers/quark_uc/driver.go | 50 ++++++++++------------ drivers/quark_uc/meta.go | 10 +++-- drivers/quark_uc/types.go | 74 ++++++++++++++++++++++++++++++-- drivers/quark_uc/util.go | 87 +++++++++++++++++++++++++++++++++++--- server/handles/down.go | 3 ++ server/handles/fsread.go | 3 +- 6 files changed, 184 insertions(+), 43 deletions(-) diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 7f497494502..691874c321e 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -7,12 +7,14 @@ import ( "hash" "io" "net/http" + "strings" "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" streamPkg "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -36,6 +38,14 @@ func (d *QuarkOrUC) GetAddition() driver.Additional { func (d *QuarkOrUC) Init(ctx context.Context) error { _, err := d.request("/config", http.MethodGet, nil, nil) + if err == nil && d.AdditionVersion != 2 { + d.AdditionVersion = 2 + if !d.UseTransCodingAddress && len(d.DownProxyUrl) == 0 { + d.WebProxy = true + d.WebdavPolicy = "native_proxy" + } + op.MustSaveDriverStorage(d) + } return err } @@ -44,39 +54,23 @@ func (d *QuarkOrUC) Drop(ctx context.Context) error { } func (d *QuarkOrUC) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files, err := d.GetFiles(dir.GetID()) - if err != nil { - return nil, err - } - return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil - }) + return d.GetFiles(dir.GetID()) } func (d *QuarkOrUC) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - data := base.Json{ - "fids": []string{file.GetID()}, - } - var resp DownResp - ua := d.conf.ua - _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { - req.SetHeader("User-Agent", ua). - SetBody(data) - }, &resp) - if err != nil { + f := file.(*File) + if d.UseTransCodingAddress && d.config.Name == "Quark" && f.Category == 1 && f.Size > 0 { + link, err := d.getTranscodingLink(file) + if err == nil { + return link, nil + } + if strings.Contains(err.Error(), "plf_invalid") { + log.Warnf("quark transcoding link invalid for %s, fallback to download link: %v", file.GetName(), err) + return d.getDownloadLink(file) + } return nil, err } - - return &model.Link{ - URL: resp.Data[0].DownloadUrl, - Header: http.Header{ - "Cookie": []string{d.Cookie}, - "Referer": []string{d.conf.referer}, - "User-Agent": []string{ua}, - }, - Concurrency: 3, - PartSize: 10 * utils.MB, - }, nil + return d.getDownloadLink(file) } func (d *QuarkOrUC) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { diff --git a/drivers/quark_uc/meta.go b/drivers/quark_uc/meta.go index f3acfe88562..6940c44b9d6 100644 --- a/drivers/quark_uc/meta.go +++ b/drivers/quark_uc/meta.go @@ -8,8 +8,11 @@ import ( type Addition struct { Cookie string `json:"cookie" required:"true"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + UseTransCodingAddress bool `json:"use_transcoding_address" help:"You can watch the transcoded video and support 302 redirection" required:"true" default:"false"` + OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` + AdditionVersion int } type Conf struct { @@ -24,7 +27,6 @@ func init() { return &QuarkOrUC{ config: driver.Config{ Name: "Quark", - OnlyLocal: true, DefaultRoot: "0", NoOverwriteUpload: true, }, @@ -40,7 +42,7 @@ func init() { return &QuarkOrUC{ config: driver.Config{ Name: "UC", - OnlyLocal: true, + OnlyProxy: true, DefaultRoot: "0", NoOverwriteUpload: true, }, diff --git a/drivers/quark_uc/types.go b/drivers/quark_uc/types.go index afbdb3eff89..13bfac2f7d5 100644 --- a/drivers/quark_uc/types.go +++ b/drivers/quark_uc/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" ) type Resp struct { @@ -18,13 +19,13 @@ type File struct { Fid string `json:"fid"` FileName string `json:"file_name"` //PdirFid string `json:"pdir_fid"` - //Category int `json:"category"` + Category int `json:"category"` //FileType int `json:"file_type"` Size int64 `json:"size"` //FormatType string `json:"format_type"` //Status int `json:"status"` //Tags string `json:"tags,omitempty"` - //LCreatedAt int64 `json:"l_created_at"` + LCreatedAt int64 `json:"l_created_at"` LUpdatedAt int64 `json:"l_updated_at"` //NameSpace int `json:"name_space"` //IncludeItems int `json:"include_items,omitempty"` @@ -32,8 +33,8 @@ type File struct { //BackupSign int `json:"backup_sign"` //Duration int `json:"duration"` //FileSource string `json:"file_source"` - File bool `json:"file"` - //CreatedAt int64 `json:"created_at"` + File bool `json:"file"` + CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` //PrivateExtra struct {} `json:"_private_extra"` //ObjCategory string `json:"obj_category,omitempty"` @@ -50,6 +51,38 @@ func fileToObj(f File) *model.Object { } } +func (f *File) GetSize() int64 { + return f.Size +} + +func (f *File) GetName() string { + return f.FileName +} + +func (f *File) ModTime() time.Time { + return time.UnixMilli(f.UpdatedAt) +} + +func (f *File) CreateTime() time.Time { + return time.UnixMilli(f.CreatedAt) +} + +func (f *File) IsDir() bool { + return !f.File +} + +func (f *File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f *File) GetID() string { + return f.Fid +} + +func (f *File) GetPath() string { + return "" +} + type SortResp struct { Resp Data struct { @@ -100,6 +133,39 @@ type DownResp struct { //} `json:"metadata"` } +type TranscodingResp struct { + Resp + Data struct { + DefaultResolution string `json:"default_resolution"` + OriginDefaultResolution string `json:"origin_default_resolution"` + VideoList []struct { + Resolution string `json:"resolution"` + VideoInfo struct { + Duration int `json:"duration"` + Size int64 `json:"size"` + Format string `json:"format"` + Width int `json:"width"` + Height int `json:"height"` + Bitrate float64 `json:"bitrate"` + Codec string `json:"codec"` + Fps float64 `json:"fps"` + Rotate int `json:"rotate"` + UpdateTime int64 `json:"update_time"` + URL string `json:"url"` + Resolution string `json:"resolution"` + HlsType string `json:"hls_type"` + Finish bool `json:"finish"` + Resoultion string `json:"resoultion"` + Success bool `json:"success"` + } `json:"video_info,omitempty"` + } `json:"video_list"` + FileName string `json:"file_name"` + NameSpace int `json:"name_space"` + Size int64 `json:"size"` + Thumbnail string `json:"thumbnail"` + } `json:"data"` +} + type UpPreResp struct { Resp Data struct { diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index c5845cc6823..2f99c308f33 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "html" "io" "net/http" "strconv" @@ -50,20 +51,29 @@ func (d *QuarkOrUC) request(pathname string, method string, callback base.ReqCal d.Cookie = cookie.SetStr(d.Cookie, "__puus", __puus.Value) op.MustSaveDriverStorage(d) } + if d.UseTransCodingAddress && d.config.Name == "Quark" { + __pus := cookie.GetCookie(res.Cookies(), "__pus") + if __pus != nil { + d.Cookie = cookie.SetStr(d.Cookie, "__pus", __pus.Value) + op.MustSaveDriverStorage(d) + } + } if e.Status >= 400 || e.Code != 0 { return nil, errors.New(e.Message) } return res.Body(), nil } -func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { - files := make([]File, 0) +func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { + files := make([]model.Obj, 0) page := 1 size := 100 query := map[string]string{ - "pdir_fid": parent, - "_size": strconv.Itoa(size), - "_fetch_total": "1", + "pdir_fid": parent, + "_size": strconv.Itoa(size), + "_fetch_total": "1", + "fetch_all_file": "1", + "fetch_risk_file_name": "1", } if d.OrderBy != "none" { query["_sort"] = "file_type:asc," + d.OrderBy + ":" + d.OrderDirection @@ -77,7 +87,16 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { if err != nil { return nil, err } - files = append(files, resp.Data.List...) + for _, file := range resp.Data.List { + file.FileName = html.UnescapeString(file.FileName) + if d.OnlyListVideoFile { + if file.IsDir() || file.Category == 1 { + files = append(files, &file) + } + } else { + files = append(files, &file) + } + } if page*size >= resp.Metadata.Total { break } @@ -86,6 +105,62 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { return files, nil } +func (d *QuarkOrUC) getDownloadLink(file model.Obj) (*model.Link, error) { + data := base.Json{ + "fids": []string{file.GetID()}, + } + var resp DownResp + ua := d.conf.ua + _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua). + SetBody(data) + }, &resp) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: resp.Data[0].DownloadUrl, + Header: http.Header{ + "Cookie": []string{d.Cookie}, + "Referer": []string{d.conf.referer}, + "User-Agent": []string{ua}, + }, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil +} + +func (d *QuarkOrUC) getTranscodingLink(file model.Obj) (*model.Link, error) { + data := base.Json{ + "fid": file.GetID(), + "resolutions": "low,normal,high,super,2k,4k", + "supports": "fmp4_av,m3u8,dolby_vision", + } + var resp TranscodingResp + ua := d.conf.ua + + _, err := d.request("/file/v2/play/project", http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua). + SetBody(data) + }, &resp) + if err != nil { + return nil, err + } + + for _, info := range resp.Data.VideoList { + if info.VideoInfo.URL != "" { + return &model.Link{ + URL: info.VideoInfo.URL, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil + } + } + + return nil, errors.New("no link found") +} + func (d *QuarkOrUC) upPre(file model.FileStreamer, parentId string) (UpPreResp, error) { now := time.Now() data := base.Json{ diff --git a/server/handles/down.go b/server/handles/down.go index e93ed1eb103..37439f00bb3 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -187,6 +187,9 @@ func canProxy(storage driver.Driver, filename string) bool { if storage.Config().MustProxy() || storage.GetStorage().WebProxy || storage.GetStorage().WebdavProxy() { return true } + if storage.GetStorage().Driver == "Quark" && utils.GetFileType(filename) == conf.VIDEO { + return true + } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { return true } diff --git a/server/handles/fsread.go b/server/handles/fsread.go index eb984698b16..c370f631fc0 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -377,7 +377,8 @@ func FsGet(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + forceProxyRawURL := storage.GetStorage().Driver == "Quark" && utils.GetFileType(obj.GetName()) == conf.VIDEO + if storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL { query := "" if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { query = "?sign=" + sign.Sign(reqPath) From 82ab5768ceb8fe62a24f98c904927f4ac56a20b1 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sat, 14 Mar 2026 14:19:47 +0800 Subject: [PATCH 602/659] fix(auth): prevent and heal dirty admin permissions --- internal/db/role.go | 3 +++ internal/op/role.go | 30 ++++++++++++++++++++++++++++++ internal/op/user.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) diff --git a/internal/db/role.go b/internal/db/role.go index 808a6f5f06b..d0b776b391d 100644 --- a/internal/db/role.go +++ b/internal/db/role.go @@ -79,6 +79,9 @@ func UpdateRolePermissionsPathPrefix(oldPath, newPath string) ([]uint, error) { } for _, role := range roles { + if role.Name == "admin" || role.Name == "guest" { + continue + } updated := false for i, entry := range role.PermissionScopes { entryPath := path.Clean(entry.Path) diff --git a/internal/op/role.go b/internal/op/role.go index 5c9aad06f41..bd874eeed19 100644 --- a/internal/op/role.go +++ b/internal/op/role.go @@ -21,6 +21,24 @@ func init() { model.FetchRole = GetRole } +func enforceAdminRoleDefaults(r *model.Role) error { + if r == nil || r.Name != "admin" { + return nil + } + if len(r.PermissionScopes) == 1 { + scopePath := utils.FixAndCleanPath(r.PermissionScopes[0].Path) + if scopePath == "/" && r.PermissionScopes[0].Permission == 0xFFFF { + r.PermissionScopes[0].Path = "/" + return nil + } + } + + r.PermissionScopes = []model.PermissionEntry{ + {Path: "/", Permission: 0xFFFF}, + } + return db.UpdateRole(r) +} + func GetRole(id uint) (*model.Role, error) { key := fmt.Sprint(id) if r, ok := roleCache.Get(key); ok { @@ -31,7 +49,11 @@ func GetRole(id uint) (*model.Role, error) { if err != nil { return nil, err } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(_r.Name, _r, cache.WithEx[*model.Role](time.Hour)) return _r, nil }) return r, err @@ -46,7 +68,11 @@ func GetRoleByName(name string) (*model.Role, error) { if err != nil { return nil, err } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } roleCache.Set(name, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(fmt.Sprint(_r.ID), _r, cache.WithEx[*model.Role](time.Hour)) return _r, nil }) return r, err @@ -89,7 +115,11 @@ func GetRolesByUserID(userID uint) ([]model.Role, error) { if err != nil { return nil, err } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(_r.Name, _r, cache.WithEx[*model.Role](time.Hour)) return _r, nil }) if err != nil { diff --git a/internal/op/user.go b/internal/op/user.go index 44b19db3508..b58a87ed338 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -16,6 +16,25 @@ var userG singleflight.Group[*model.User] var guestUser *model.User var adminUser *model.User +func enforceAdminUserDefaults(u *model.User) error { + if u == nil || !u.IsAdmin() { + return nil + } + changed := false + if utils.FixAndCleanPath(u.BasePath) != "/" { + u.BasePath = "/" + changed = true + } + if u.Permission != 0xFFFF { + u.Permission = 0xFFFF + changed = true + } + if !changed { + return nil + } + return db.UpdateUser(u) +} + func GetAdmin() (*model.User, error) { if adminUser == nil { role, err := GetRoleByName("admin") @@ -26,7 +45,12 @@ func GetAdmin() (*model.User, error) { if err != nil { return nil, err } + if err := enforceAdminUserDefaults(user); err != nil { + return nil, err + } adminUser = user + } else if err := enforceAdminUserDefaults(adminUser); err != nil { + return nil, err } return adminUser, nil } @@ -59,6 +83,9 @@ func GetUserByName(username string) (*model.User, error) { return nil, errs.EmptyUsername } if user, ok := userCache.Get(username); ok { + if err := enforceAdminUserDefaults(user); err != nil { + return nil, err + } return user, nil } user, err, _ := userG.Do(username, func() (*model.User, error) { @@ -66,6 +93,9 @@ func GetUserByName(username string) (*model.User, error) { if err != nil { return nil, err } + if err := enforceAdminUserDefaults(_user); err != nil { + return nil, err + } userCache.Set(username, _user, cache.WithEx[*model.User](time.Hour)) return _user, nil }) From 2280505f2a6c02e0123018e5ac1de389236cd0d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 16 Mar 2026 19:13:57 +0800 Subject: [PATCH 603/659] chore(deps): refresh indirect module checksums (#9446) --- go.mod | 2 +- go.sum | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2e202bc9de0..bc2475c548d 100644 --- a/go.mod +++ b/go.mod @@ -250,7 +250,7 @@ require ( github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/shirou/gopsutil/v3 v3.24.4 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/shoenig/go-m1cpu v0.2.0 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index af3a6b3f96c..e6b04779ede 100644 --- a/go.sum +++ b/go.sum @@ -589,8 +589,11 @@ github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRB github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/go-m1cpu v0.2.0 h1:t4GNqvPZ84Vjtpboo/kT3pIkbaK3vc+JIlD/Wz1zSFY= +github.com/shoenig/go-m1cpu v0.2.0/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= From 6bde813ef14c6032af39bd0820fbf8901f8a37a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 16 Mar 2026 19:15:06 +0800 Subject: [PATCH 604/659] feat(wukong): add WuKongNetdisk driver with read-write support (#9449) --- drivers/all.go | 3 +- drivers/wukong/driver.go | 1116 ++++++++++++++++++++++++++++++++++++++ drivers/wukong/meta.go | 34 ++ drivers/wukong/types.go | 113 ++++ 4 files changed, 1265 insertions(+), 1 deletion(-) create mode 100644 drivers/wukong/driver.go create mode 100644 drivers/wukong/meta.go create mode 100644 drivers/wukong/types.go diff --git a/drivers/all.go b/drivers/all.go index 58cdac85f03..a4fce9d0cac 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -66,8 +66,8 @@ import ( _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/smb" - _ "github.com/alist-org/alist/v3/drivers/strm" _ "github.com/alist-org/alist/v3/drivers/streamtape" + _ "github.com/alist-org/alist/v3/drivers/strm" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" @@ -81,6 +81,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/webdav" _ "github.com/alist-org/alist/v3/drivers/weiyun" _ "github.com/alist-org/alist/v3/drivers/wopan" + _ "github.com/alist-org/alist/v3/drivers/wukong" _ "github.com/alist-org/alist/v3/drivers/yandex_disk" ) diff --git a/drivers/wukong/driver.go b/drivers/wukong/driver.go new file mode 100644 index 00000000000..cb5dee1ae40 --- /dev/null +++ b/drivers/wukong/driver.go @@ -0,0 +1,1116 @@ +package wukong + +import ( + "context" + "crypto/hmac" + "crypto/md5" + crand "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "io" + "net/http" + "net/url" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" +) + +const ( + wukongBaseURL = "https://api.wkbrowser.com" + webReferer = "https://pan.wkbrowser.com/" + vodBaseURL = "https://vod.bytedanceapi.com" + vodRegion = "cn-north-1" + vodService = "vod" + videoSpaceName = "wukong_netdisk_ugc" + minUploadSubmitSuccess = 2000 + multipartChunkSize = int64(5 * 1024 * 1024) +) + +type Wukong struct { + model.Storage + Addition + client *resty.Client +} + +func (d *Wukong) Config() driver.Config { + return config +} + +func (d *Wukong) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Wukong) Init(ctx context.Context) error { + d.client = base.NewRestyClient(). + SetBaseURL(wukongBaseURL). + SetHeader("accept", "application/json, text/plain, */*"). + SetHeader("content-type", "application/json"). + SetHeader("referer", webReferer). + SetHeader("origin", "https://pan.wkbrowser.com") + if d.Cookie != "" { + d.client.SetHeader("cookie", d.Cookie) + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + if strings.TrimSpace(d.Aid) == "" { + d.Aid = "590353" + } + if strings.TrimSpace(d.Language) == "" { + d.Language = "zh" + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + return nil +} + +func (d *Wukong) Drop(ctx context.Context) error { + return nil +} + +func (d *Wukong) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + fatherID := dir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + offset := 0 + limit := d.PageSize + objs := make([]model.Obj, 0) + for { + var resp filterFileResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "offset": strconv.Itoa(offset), + "limit": strconv.Itoa(limit), + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "father_id": asIDValue(fatherID), + "filter_type": 2, + "is_desc": 1, + "file_type": 0, + }). + SetResult(&resp). + Post("/netdisk/user_file/filter_file") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong list failed: code=%d message=%s", resp.Code, resp.Message) + } + + for _, item := range resp.Data.FileList { + objs = append(objs, &model.Object{ + ID: strconv.FormatInt(item.FileID, 10), + Path: strconv.FormatInt(item.FatherID, 10), + Name: item.FileName, + Size: item.Size, + Modified: parseUnix(item.UpdatedAt), + Ctime: parseUnix(item.CreatedAt), + IsFolder: item.IsDirectory == 1, + }) + } + + if !hasMore(resp.Data.HasMore) || len(resp.Data.FileList) == 0 { + break + } + offset += len(resp.Data.FileList) + } + return objs, nil +} + +func (d *Wukong) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := file.GetID() + if fileID == "" { + return nil, errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(fileID)}, + }). + SetResult(&resp). + Post("/netdisk/user_file/detail") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong detail failed: code=%d message=%s", resp.Code, resp.Message) + } + + url := extractDetailMainURL(resp.Data) + if url == "" { + url = extractURL(resp.Data) + } + if url == "" { + return nil, errs.NotImplement + } + + return &model.Link{ + URL: url, + Header: http.Header{ + "Referer": []string{webReferer}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Wukong) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + fatherID := parentDir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "father_id": asIDValue(fatherID), + "file_name": dirName, + }). + SetResult(&resp). + Post("/netdisk/user_file/create_directory") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong create directory failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + srcID := srcObj.GetID() + if srcID == "" { + return errors.New("missing source file id") + } + + dstID := dstDir.GetID() + if dstID == "" { + dstID = d.RootFolderID + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(srcID)}, + "new_father_id": asIDValue(dstID), + }). + SetResult(&resp). + Post("/netdisk/user_file/move_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong move failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + srcID := srcObj.GetID() + if srcID == "" { + return errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id": asIDValue(srcID), + "new_name": newName, + }). + SetResult(&resp). + Post("/netdisk/user_file/rename_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong rename failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Remove(ctx context.Context, obj model.Obj) error { + fileID := obj.GetID() + if fileID == "" { + return errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(fileID)}, + }). + SetResult(&resp). + Post("/netdisk/user_file/delete_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong delete failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + fatherID := dstDir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return err + } + defer tempFile.Close() + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + + md5Hex, crc32Hex, err := calcFileMD5AndCRC32(tempFile) + if err != nil { + return err + } + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(file.GetName())), ".") + fileType := detectWukongFileType(file.GetMimetype(), file.GetName()) + size := file.GetSize() + up(5) + + uploadType := detectUploadType(file.GetMimetype(), file.GetName()) + authToken, err := d.getUploadAuthToken(ctx, uploadType) + if err != nil { + return err + } + up(10) + + candidates, err := d.getUploadCandidates(ctx, authToken) + if err != nil { + return err + } + bestHosts := collectCandidateHosts(candidates) + if len(bestHosts) == 0 { + return errors.New("wukong upload candidates is empty") + } + up(20) + + applyResp, err := d.applyUploadInner(ctx, authToken, uploadType, size, strings.Join(bestHosts, ",")) + if err != nil { + return err + } + if len(applyResp.Result.InnerUploadAddress.UploadNodes) == 0 || + len(applyResp.Result.InnerUploadAddress.UploadNodes[0].StoreInfos) == 0 { + return errors.New("wukong apply upload inner returns empty upload node") + } + node := applyResp.Result.InnerUploadAddress.UploadNodes[0] + store := node.StoreInfos[0] + up(30) + + if size > multipartChunkSize { + if err = d.uploadToTOSMultipart(ctx, node.UploadHost, store.StoreURI, store.Auth, getStorageUserID(store.StorageHeader), tempFile, size, up); err != nil { + return err + } + } else { + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + reader := &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{Reader: tempFile, Size: size}, + UpdateProgress: func(percent float64) { + up(30 + percent*0.5) + }, + } + if err = d.uploadToTOS(ctx, node.UploadHost, store.StoreURI, store.Auth, getStorageUserID(store.StorageHeader), crc32Hex, file.GetName(), reader, size); err != nil { + return err + } + } + up(85) + + videoVid, err := d.commitUploadInner(ctx, authToken, chooseCommitSpace(uploadType, authToken.SpaceName), node.SessionKey) + if err != nil { + return err + } + up(92) + + if fileType == 3000 && videoVid == "" { + return errors.New("wukong video upload missing vid in commit response") + } + if err = d.uploadSubmit(ctx, fatherID, file.GetName(), ext, fileType, size, md5Hex, store.StoreURI, videoVid); err != nil { + return err + } + up(100) + return nil +} + +func (d *Wukong) getUploadAuthToken(ctx context.Context, uploadType string) (*uploadAuthTokenResp, error) { + var resp uploadAuthTokenResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "upload_source": uploadSourceByType(uploadType), + "type": uploadType, + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetResult(&resp). + Get("/toutiao/upload/auth_token/v1/") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong get upload auth token failed: code=%d message=%s", resp.Code, resp.Message) + } + return &resp, nil +} + +func (d *Wukong) getUploadCandidates(ctx context.Context, auth *uploadAuthTokenResp) (*getUploadCandidatesResp, error) { + q := map[string]string{ + "Action": "GetUploadCandidates", + "Version": "2020-11-19", + "SpaceName": videoSpaceName, + } + var resp getUploadCandidatesResp + if err := d.vodRequest(ctx, http.MethodGet, q, nil, auth, &resp); err != nil { + return nil, err + } + if resp.ResponseMetadata.Error.Code != "" { + return nil, fmt.Errorf("wukong get upload candidates failed: %s", resp.ResponseMetadata.Error.Message) + } + return &resp, nil +} + +func (d *Wukong) applyUploadInner(ctx context.Context, auth *uploadAuthTokenResp, uploadType string, fileSize int64, bestHosts string) (*applyUploadInnerResp, error) { + spaceName := auth.SpaceName + if uploadType == "video" { + spaceName = videoSpaceName + } + q := map[string]string{ + "Action": "ApplyUploadInner", + "Version": "2020-11-19", + "SpaceName": spaceName, + "FileType": uploadType, + "IsInner": "1", + "ClientBestHosts": bestHosts, + "NeedFallback": "true", + "FileSize": strconv.FormatInt(fileSize, 10), + "s": randomString(8), + } + var resp applyUploadInnerResp + if err := d.vodRequest(ctx, http.MethodGet, q, nil, auth, &resp); err != nil { + return nil, err + } + if resp.ResponseMetadata.Error.Code != "" { + return nil, fmt.Errorf("wukong apply upload inner failed: %s", resp.ResponseMetadata.Error.Message) + } + return &resp, nil +} + +func (d *Wukong) commitUploadInner(ctx context.Context, auth *uploadAuthTokenResp, spaceName, sessionKey string) (string, error) { + q := map[string]string{ + "Action": "CommitUploadInner", + "Version": "2020-11-19", + "SpaceName": spaceName, + } + body, _ := json.Marshal(map[string]any{ + "SessionKey": sessionKey, + "Functions": []any{}, + }) + var resp commitUploadInnerResp + if err := d.vodRequest(ctx, http.MethodPost, q, body, auth, &resp); err != nil { + return "", err + } + if resp.ResponseMetadata.Error.Code != "" { + return "", fmt.Errorf("wukong commit upload inner failed: %s", resp.ResponseMetadata.Error.Message) + } + if len(resp.Result.Results) > 0 { + status := resp.Result.Results[0].URIStatus + if status != 0 && status != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong commit upload inner failed: uri_status=%d", status) + } + } + return extractVideoVid(&resp), nil +} + +func (d *Wukong) uploadSubmit(ctx context.Context, fatherID, fileName, ext string, fileType int, size int64, md5Hex, storeURI, videoVid string) error { + var resp uploadSubmitResp + body := map[string]any{ + "base_info": map[string]any{ + "father_id": asIDValue(fatherID), + "file_type": fileType, + "size": size, + "extension": ext, + "file_name": fileName, + "is_directory": 0, + "md5": md5Hex, + "slice_md5": md5Hex, + }, + } + switch fileType { + case 3000: + if videoVid != "" { + body["video_info"] = map[string]any{"vid": videoVid} + } + case 2000: + body["image_info"] = map[string]any{"uri": storeURI} + default: + body["general_info"] = map[string]any{"key": storeURI} + } + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(body). + SetResult(&resp). + Post("/netdisk/upload_submit/") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong upload submit failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func extractVideoVid(resp *commitUploadInnerResp) string { + for _, item := range resp.Result.Results { + if item.Vid != "" { + return item.Vid + } + } + if resp.Result.PluginResult != nil { + if vid := findStringByKey(resp.Result.PluginResult, "Vid"); vid != "" { + return vid + } + if vid := findStringByKey(resp.Result.PluginResult, "vid"); vid != "" { + return vid + } + } + return "" +} + +func findStringByKey(v any, key string) string { + switch cur := v.(type) { + case map[string]any: + if val, ok := cur[key]; ok { + if s, ok := val.(string); ok && s != "" { + return s + } + } + for _, child := range cur { + if s := findStringByKey(child, key); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findStringByKey(child, key); s != "" { + return s + } + } + } + return "" +} + +func (d *Wukong) uploadToTOS(ctx context.Context, host, storeURI, auth, storageUser, crc32Hex, fileName string, body io.Reader, size int64) error { + var resp tosUploadResp + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Crc32", crc32Hex). + SetHeader("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, url.QueryEscape(fileName))). + SetHeader("Content-Length", strconv.FormatInt(size, 10)). + SetBody(body). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + _, err := req.Post(uploadURL) + if err != nil { + return err + } + if resp.Code != minUploadSubmitSuccess { + return fmt.Errorf("wukong upload to tos failed: code=%d message=%s", resp.Code, resp.Message) + } + if resp.Data.Crc32 != "" && !strings.EqualFold(resp.Data.Crc32, crc32Hex) { + return fmt.Errorf("wukong upload to tos crc32 mismatch: local=%s remote=%s", crc32Hex, resp.Data.Crc32) + } + return nil +} + +func (d *Wukong) uploadToTOSMultipart(ctx context.Context, host, storeURI, auth, storageUser string, tempFile model.File, size int64, up driver.UpdateProgress) error { + uploadID, err := d.initMultipartUpload(ctx, host, storeURI, auth, storageUser) + if err != nil { + return err + } + + totalParts := int((size + multipartChunkSize - 1) / multipartChunkSize) + if totalParts <= 0 { + return errors.New("invalid multipart parts") + } + parts := make([]string, 0, totalParts) + for i := 0; i < totalParts; i++ { + partNumber := i + 1 + offset := int64(i) * multipartChunkSize + partSize := multipartChunkSize + if remain := size - offset; remain < partSize { + partSize = remain + } + buf := make([]byte, partSize) + n, readErr := tempFile.ReadAt(buf, offset) + if readErr != nil && readErr != io.EOF { + return readErr + } + buf = buf[:n] + crc32Hex := fmt.Sprintf("%08x", crc32.ChecksumIEEE(buf)) + remoteCRC32, err := d.uploadMultipartPart(ctx, host, storeURI, auth, storageUser, uploadID, partNumber, buf, crc32Hex) + if err != nil { + return err + } + if remoteCRC32 != "" && !strings.EqualFold(remoteCRC32, crc32Hex) { + return fmt.Errorf("multipart part crc32 mismatch: part=%d local=%s remote=%s", partNumber, crc32Hex, remoteCRC32) + } + parts = append(parts, fmt.Sprintf("%d:%s", partNumber, crc32Hex)) + up(30 + float64(partNumber)/float64(totalParts)*50) + } + + return d.finishMultipartUpload(ctx, host, storeURI, auth, storageUser, uploadID, strings.Join(parts, ",")) +} + +func (d *Wukong) initMultipartUpload(ctx context.Context, host, storeURI, auth, storageUser string) (string, error) { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetQueryParams(map[string]string{ + "uploadmode": "part", + "phase": "init", + }). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return "", err + } + if resp.Code != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong init multipart upload failed: code=%d message=%s", resp.Code, resp.Message) + } + if resp.Data.UploadID == "" { + return "", errors.New("wukong init multipart upload returns empty uploadid") + } + return resp.Data.UploadID, nil +} + +func (d *Wukong) uploadMultipartPart(ctx context.Context, host, storeURI, auth, storageUser, uploadID string, partNumber int, data []byte, crc32Hex string) (string, error) { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Crc32", crc32Hex). + SetHeader("Content-Length", strconv.Itoa(len(data))). + SetQueryParams(map[string]string{ + "uploadid": uploadID, + "part_number": strconv.Itoa(partNumber), + "phase": "transfer", + }). + SetBody(data). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return "", err + } + if resp.Code != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong multipart transfer failed: code=%d message=%s part=%d", resp.Code, resp.Message, partNumber) + } + return resp.Data.Crc32, nil +} + +func (d *Wukong) finishMultipartUpload(ctx context.Context, host, storeURI, auth, storageUser, uploadID, body string) error { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetQueryParams(map[string]string{ + "uploadid": uploadID, + "phase": "finish", + "uploadmode": "part", + }). + SetBody(body). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return err + } + if resp.Code != minUploadSubmitSuccess && resp.Code != 4024 { + return fmt.Errorf("wukong multipart finish failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) vodRequest(ctx context.Context, method string, query map[string]string, body []byte, auth *uploadAuthTokenResp, resp any) error { + reqURL := vodBaseURL + "/" + amzDate := time.Now().UTC().Format("20060102T150405Z") + dateStamp := amzDate[:8] + headers := map[string]string{ + "x-amz-date": amzDate, + "x-amz-security-token": auth.SessionToken, + } + if method == http.MethodPost { + headers["x-amz-content-sha256"] = hashSHA256Bytes(body) + } + authorization := buildVodAuthorization(method, "/", query, headers, body, auth, dateStamp) + + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Authorization", authorization). + SetHeader("x-amz-date", amzDate). + SetHeader("x-amz-security-token", auth.SessionToken). + SetQueryParams(query). + SetResult(resp) + if method == http.MethodPost { + req.SetHeader("x-amz-content-sha256", headers["x-amz-content-sha256"]) + req.SetHeader("Content-Type", "text/plain;charset=UTF-8") + req.SetBody(body) + } + _, err := req.Execute(method, reqURL) + return err +} + +func buildVodAuthorization(method, canonicalURI string, query map[string]string, headers map[string]string, body []byte, auth *uploadAuthTokenResp, dateStamp string) string { + canonicalQueryString := getCanonicalQueryStringFromMap(query) + canonicalHeaders, signedHeaders := getCanonicalHeaders(headers) + payloadHash := hashSHA256Bytes(body) + canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash + credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", dateStamp, vodRegion, vodService) + stringToSign := "AWS4-HMAC-SHA256\n" + headers["x-amz-date"] + "\n" + credentialScope + "\n" + hashSHA256String(canonicalRequest) + signingKey := getSigningKey(auth.SecretAccessKey, dateStamp, vodRegion, vodService) + signature := hmacSHA256Hex(signingKey, stringToSign) + return fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", auth.AccessKeyID, credentialScope, signedHeaders, signature) +} + +func getCanonicalQueryStringFromMap(query map[string]string) string { + if len(query) == 0 { + return "" + } + keys := make([]string, 0, len(query)) + for k := range query { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, awsURLEncode(k)+"="+awsURLEncode(query[k])) + } + return strings.Join(parts, "&") +} + +func getCanonicalHeaders(headers map[string]string) (string, string) { + keys := make([]string, 0, len(headers)) + for k := range headers { + keys = append(keys, strings.ToLower(k)) + } + sort.Strings(keys) + var h strings.Builder + for _, k := range keys { + h.WriteString(k) + h.WriteString(":") + h.WriteString(strings.TrimSpace(headers[k])) + h.WriteString("\n") + } + return h.String(), strings.Join(keys, ";") +} + +func awsURLEncode(s string) string { + s = url.QueryEscape(s) + return strings.ReplaceAll(s, "+", "%20") +} + +func hashSHA256Bytes(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func hashSHA256String(s string) string { + return hashSHA256Bytes([]byte(s)) +} + +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + _, _ = h.Write([]byte(data)) + return h.Sum(nil) +} + +func hmacSHA256Hex(key []byte, data string) string { + return hex.EncodeToString(hmacSHA256(key, data)) +} + +func getSigningKey(secret, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secret), dateStamp) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + return hmacSHA256(kService, "aws4_request") +} + +func collectCandidateHosts(resp *getUploadCandidatesResp) []string { + seen := map[string]struct{}{} + hosts := make([]string, 0, len(resp.Result.Domains)) + add := func(domain vodDomain) { + if domain.Name == "" { + return + } + if _, ok := seen[domain.Name]; ok { + return + } + seen[domain.Name] = struct{}{} + hosts = append(hosts, domain.Name) + } + for _, candidate := range resp.Result.Candidates { + for _, domain := range candidate.Domains { + add(domain) + } + } + for _, domain := range resp.Result.Domains { + add(domain) + } + return hosts +} + +func getStorageUserID(header map[string]any) string { + if header == nil { + return "" + } + if s, ok := header["USER_ID"].(string); ok { + return s + } + if f, ok := header["USER_ID"].(float64); ok { + return strconv.FormatInt(int64(f), 10) + } + return "" +} + +func calcFileMD5AndCRC32(f model.File) (string, string, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", "", err + } + md5Hasher := md5.New() + crc := crc32.NewIEEE() + _, err := io.Copy(io.MultiWriter(md5Hasher, crc), f) + if err != nil { + return "", "", err + } + if _, err = f.Seek(0, io.SeekStart); err != nil { + return "", "", err + } + return hex.EncodeToString(md5Hasher.Sum(nil)), fmt.Sprintf("%08x", crc.Sum32()), nil +} + +func detectWukongFileType(mimetype, fileName string) int { + lowerName := strings.ToLower(fileName) + switch { + case strings.HasPrefix(mimetype, "image/"): + return 2000 + case strings.HasPrefix(mimetype, "video/"), strings.HasSuffix(lowerName, ".flv"), strings.HasSuffix(lowerName, ".mkv"): + return 3000 + case strings.HasPrefix(mimetype, "audio/"), strings.HasSuffix(lowerName, ".mp3"), strings.HasSuffix(lowerName, ".m4a"), strings.HasSuffix(lowerName, ".wav"): + return 4000 + case strings.HasSuffix(lowerName, ".zip"), strings.HasSuffix(lowerName, ".rar"), strings.HasSuffix(lowerName, ".7z"), strings.HasSuffix(lowerName, ".tar"), strings.HasSuffix(lowerName, ".gz"), strings.HasSuffix(lowerName, ".tgz"): + return 6000 + default: + return 5000 + } +} + +func randomString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + if n <= 0 { + return "" + } + buf := make([]byte, n) + if _, err := crand.Read(buf); err == nil { + for i := range buf { + buf[i] = letters[int(buf[i])%len(letters)] + } + return string(buf) + } + + now := uint64(time.Now().UnixNano()) + b := make([]byte, n) + for i := range b { + now = now*6364136223846793005 + 1 + b[i] = letters[int(now%uint64(len(letters)))] + } + return string(b) +} + +func uploadSourceByType(uploadType string) string { + switch uploadType { + case "video": + return "10150001" + case "image": + return "20150001" + default: + return "50150001" + } +} + +func detectUploadType(mimetype, fileName string) string { + lowerName := strings.ToLower(fileName) + if strings.HasPrefix(mimetype, "video/") || strings.HasPrefix(mimetype, "audio/") || + strings.HasSuffix(lowerName, ".flv") || strings.HasSuffix(lowerName, ".mkv") || + strings.HasSuffix(lowerName, ".mp3") || strings.HasSuffix(lowerName, ".m4a") || strings.HasSuffix(lowerName, ".wav") { + return "video" + } + if strings.HasPrefix(mimetype, "image/") { + return "image" + } + return "object" +} + +func chooseCommitSpace(uploadType, authSpace string) string { + if uploadType == "video" { + return videoSpaceName + } + return authSpace +} + +func asIDValue(id string) any { + if n, err := strconv.ParseInt(id, 10, 64); err == nil { + return n + } + return id +} + +func parseUnix(ts int64) time.Time { + if ts <= 0 { + return time.Time{} + } + if ts > 1e12 { + return time.UnixMilli(ts) + } + return time.Unix(ts, 0) +} + +func hasMore(v any) bool { + switch val := v.(type) { + case bool: + return val + case float64: + return val != 0 + case int: + return val != 0 + case int64: + return val != 0 + case string: + return val == "1" || strings.EqualFold(val, "true") + default: + return false + } +} + +func extractURL(data map[string]any) string { + priority := []string{ + "download_url", + "main_url", + "MainUrl", + "MainHTTPUrl", + "url", + "source_url", + "play_url", + "backup_url", + "BackupUrl", + "BackupHTTPUrl", + } + for _, key := range priority { + if url := findURLByKey(data, key); url != "" { + return url + } + } + return findAnyHTTPURL(data) +} + +func extractDetailMainURL(data map[string]any) string { + rawList, ok := data["list"] + if !ok { + return "" + } + list, ok := rawList.([]any) + if !ok || len(list) == 0 { + return "" + } + first, ok := list[0].(map[string]any) + if !ok { + return "" + } + generalInfo, ok := first["general_info"].(map[string]any) + if !ok { + return "" + } + mainURL, ok := generalInfo["main_url"].(string) + if !ok || !isHTTPURL(mainURL) { + return "" + } + return mainURL +} + +func findURLByKey(v any, key string) string { + switch cur := v.(type) { + case map[string]any: + if val, ok := cur[key]; ok { + if s, ok := val.(string); ok && isHTTPURL(s) { + return s + } + if decoded := tryDecodeJSONAny(val); decoded != nil { + if s := findURLByKey(decoded, key); s != "" { + return s + } + } + } + for _, child := range cur { + if s := findURLByKey(child, key); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findURLByKey(child, key); s != "" { + return s + } + } + case string: + if decoded := tryDecodeJSONString(cur); decoded != nil { + return findURLByKey(decoded, key) + } + } + return "" +} + +func findAnyHTTPURL(v any) string { + switch cur := v.(type) { + case string: + if isHTTPURL(cur) { + return cur + } + if decoded := tryDecodeJSONString(cur); decoded != nil { + return findAnyHTTPURL(decoded) + } + case map[string]any: + for _, child := range cur { + if s := findAnyHTTPURL(child); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findAnyHTTPURL(child); s != "" { + return s + } + } + } + return "" +} + +func isHTTPURL(v string) bool { + return strings.HasPrefix(v, "http://") || strings.HasPrefix(v, "https://") +} + +func tryDecodeJSONAny(v any) any { + s, ok := v.(string) + if !ok { + return nil + } + return tryDecodeJSONString(s) +} + +func tryDecodeJSONString(s string) any { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + if !(strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[")) { + return nil + } + var out any + if err := json.Unmarshal([]byte(s), &out); err != nil { + return nil + } + return out +} + +var _ driver.Driver = (*Wukong)(nil) diff --git a/drivers/wukong/meta.go b/drivers/wukong/meta.go new file mode 100644 index 00000000000..451fd7942b4 --- /dev/null +++ b/drivers/wukong/meta.go @@ -0,0 +1,34 @@ +package wukong + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Cookie string `json:"cookie" type:"text" required:"true" help:"Cookie from https://pan.wkbrowser.com/"` + Aid string `json:"aid" default:"590353" help:"aid query param used by web requests"` + Language string `json:"language" default:"zh"` + PageSize int `json:"page_size" type:"number" default:"100"` +} + +var config = driver.Config{ + Name: "WuKongNetdisk", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Wukong{} + }) +} diff --git a/drivers/wukong/types.go b/drivers/wukong/types.go new file mode 100644 index 00000000000..86ada25e06d --- /dev/null +++ b/drivers/wukong/types.go @@ -0,0 +1,113 @@ +package wukong + +type filterFileResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + FileList []wukongFile `json:"file_list"` + HasMore any `json:"has_more"` + } `json:"data"` +} + +type wukongFile struct { + FileID int64 `json:"file_id"` + FatherID int64 `json:"father_id"` + IsDirectory int `json:"is_directory"` + FileType int `json:"file_type"` + Size int64 `json:"size"` + FileName string `json:"file_name"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +type rawResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data map[string]any `json:"data"` +} + +type uploadAuthTokenResp struct { + Code int `json:"code"` + Message string `json:"message"` + CurrentTime int64 `json:"current_time"` + ExpireTime int64 `json:"expire_time"` + SpaceName string `json:"space_name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` +} + +type vodResponseMetadata struct { + RequestID string `json:"RequestId"` + Action string `json:"Action"` + Version string `json:"Version"` + Service string `json:"Service"` + Region string `json:"Region"` + Error struct { + CodeN int `json:"CodeN,omitempty"` + Code string `json:"Code,omitempty"` + Message string `json:"Message,omitempty"` + } `json:"Error,omitempty"` +} + +type vodDomain struct { + Name string `json:"Name"` + Sign string `json:"Sign"` + StoreID string `json:"StoreID"` +} + +type getUploadCandidatesResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + Candidates []struct { + Domains []vodDomain `json:"Domains"` + } `json:"Candidates"` + Domains []vodDomain `json:"Domains"` + } `json:"Result"` +} + +type applyUploadInnerResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + InnerUploadAddress struct { + UploadNodes []struct { + StoreInfos []struct { + StoreURI string `json:"StoreUri"` + Auth string `json:"Auth"` + UploadID string `json:"UploadID"` + StorageHeader map[string]any `json:"StorageHeader"` + } `json:"StoreInfos"` + UploadHost string `json:"UploadHost"` + SessionKey string `json:"SessionKey"` + } `json:"UploadNodes"` + } `json:"InnerUploadAddress"` + } `json:"Result"` +} + +type tosUploadResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Crc32 string `json:"crc32"` + UploadID string `json:"uploadid"` + PartNumber string `json:"part_number"` + Etag string `json:"etag"` + } `json:"data"` +} + +type commitUploadInnerResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + Results []struct { + URI string `json:"Uri"` + URIStatus int `json:"UriStatus"` + Vid string `json:"Vid"` + } `json:"Results"` + PluginResult any `json:"PluginResult"` + } `json:"Result"` +} + +type uploadSubmitResp struct { + Code int `json:"code"` + Message string `json:"message"` +} From a673503fe2afd5af6a8bef512d86a8c30bcfa490 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 18 Mar 2026 16:28:06 +0800 Subject: [PATCH 605/659] feat(s3): add placeholder toggle and fix empty-folder delete behavior --- drivers/s3/driver.go | 47 ++++++++++++++++++++++++++++++++++---------- drivers/s3/meta.go | 1 + drivers/s3/util.go | 23 +++++++++++++++++++--- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index 7825ca6f935..896f69b3028 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -69,6 +69,9 @@ func (d *S3) GetAddition() driver.Additional { } func (d *S3) Init(ctx context.Context) error { + if !strings.Contains(d.Storage.Addition, `"use_placeholder"`) { + d.UsePlaceholder = true + } if d.Region == "" { d.Region = "alist" } @@ -151,16 +154,20 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo } func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - return d.Put(ctx, &model.Object{ - Path: stdpath.Join(parentDir.GetPath(), dirName), - }, &stream.FileStream{ - Obj: &model.Object{ - Name: getPlaceholderName(d.Placeholder), - Modified: time.Now(), - }, - Reader: io.NopCloser(bytes.NewReader([]byte{})), - Mimetype: "application/octet-stream", - }, func(float64) {}) + dirPath := stdpath.Join(parentDir.GetPath(), dirName) + if d.UsePlaceholder { + return d.Put(ctx, &model.Object{ + Path: dirPath, + }, &stream.FileStream{ + Obj: &model.Object{ + Name: getPlaceholderName(d.Placeholder), + Modified: time.Now(), + }, + Reader: io.NopCloser(bytes.NewReader([]byte{})), + Mimetype: "application/octet-stream", + }, func(float64) {}) + } + return d.createDirMarker(ctx, dirPath) } func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -214,6 +221,26 @@ func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up return err } +func (d *S3) putEmptyObject(ctx context.Context, key string) error { + uploader := s3manager.NewUploader(d.Session) + contentType := "application/octet-stream" + input := &s3manager.UploadInput{ + Bucket: &d.Bucket, + Key: &key, + Body: driver.NewLimitedUploadStream(ctx, bytes.NewReader([]byte{})), + ContentType: &contentType, + } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } + _, err := uploader.UploadWithContext(ctx, input) + return err +} + +func (d *S3) createDirMarker(ctx context.Context, dirPath string) error { + return d.putEmptyObject(ctx, getKey(dirPath, true)) +} + var ( _ driver.Driver = (*S3)(nil) _ driver.Other = (*S3)(nil) diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 89d723b60b5..0c675e85646 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -19,6 +19,7 @@ type Addition struct { Placeholder string `json:"placeholder"` ForcePathStyle bool `json:"force_path_style"` ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` + UsePlaceholder bool `json:"use_placeholder" default:"true" help:"Create hidden placeholder file (for example .alist) to keep empty folders."` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` StorageClass string `json:"storage_class" type:"select" options:",standard,standard_ia,onezone_ia,intelligent_tiering,glacier,glacier_ir,deep_archive,archive" help:"Storage class for new objects. AWS and Tencent COS support different subsets (COS uses ARCHIVE/DEEP_ARCHIVE)."` diff --git a/drivers/s3/util.go b/drivers/s3/util.go index 9d2b285ce1d..863b88bcab8 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -8,6 +8,7 @@ import ( "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -105,6 +106,9 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { files = append(files, &file) } for _, object := range listObjectsResult.Contents { + if strings.HasSuffix(*object.Key, "/") { + continue + } name := path.Base(*object.Key) if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue @@ -210,10 +214,13 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { } func (d *S3) copyDir(ctx context.Context, src string, dst string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) + objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true, Refresh: true}) if err != nil { return err } + if len(objs) == 0 && !d.UsePlaceholder { + return d.createDirMarker(ctx, dst) + } for _, obj := range objs { cSrc := path.Join(src, obj.GetName()) cDst := path.Join(dst, obj.GetName()) @@ -230,8 +237,13 @@ func (d *S3) copyDir(ctx context.Context, src string, dst string) error { } func (d *S3) removeDir(ctx context.Context, src string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{}) + d.cleanupDirArtifacts(src) + + objs, err := op.List(ctx, d, src, model.ListArgs{Refresh: true}) if err != nil { + if errs.IsObjectNotFound(err) { + return nil + } return err } for _, obj := range objs { @@ -245,9 +257,14 @@ func (d *S3) removeDir(ctx context.Context, src string) error { return err } } + d.cleanupDirArtifacts(src) + return nil +} + +func (d *S3) cleanupDirArtifacts(src string) { _ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder))) _ = d.removeFile(path.Join(src, d.Placeholder)) - return nil + _ = d.removeFile(getKey(src, true)) } func (d *S3) removeFile(src string) error { From a8ff7c99950573092bb24a26b6b9b4601d4af08f Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 19 Mar 2026 11:37:25 +0800 Subject: [PATCH 606/659] feat: add public share support --- internal/db/db.go | 2 +- internal/db/share.go | 66 ++++++ internal/model/share.go | 31 +++ internal/share/access.go | 31 +++ server/handles/share.go | 368 +++++++++++++++++++++++++++++++++ server/handles/share_page.go | 16 ++ server/handles/share_public.go | 259 +++++++++++++++++++++++ server/router.go | 18 ++ 8 files changed, 790 insertions(+), 1 deletion(-) create mode 100644 internal/db/share.go create mode 100644 internal/model/share.go create mode 100644 internal/share/access.go create mode 100644 server/handles/share.go create mode 100644 server/handles/share_page.go create mode 100644 server/handles/share_public.go diff --git a/internal/db/db.go b/internal/db/db.go index 4577059d4f8..f33927be0ff 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile), new(model.Session)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile), new(model.Session), new(model.Share)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/share.go b/internal/db/share.go new file mode 100644 index 00000000000..54eb3d5f2f9 --- /dev/null +++ b/internal/db/share.go @@ -0,0 +1,66 @@ +package db + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" + "gorm.io/gorm" +) + +func GetShareByShareID(shareID string) (*model.Share, error) { + var share model.Share + if err := db.Where("share_id = ?", shareID).Take(&share).Error; err != nil { + return nil, err + } + return &share, nil +} + +func ShareIDExists(shareID string) bool { + var count int64 + if err := db.Model(&model.Share{}).Where("share_id = ?", shareID).Count(&count).Error; err != nil { + return false + } + return count > 0 +} + +func CreateShare(share *model.Share) error { + return db.Create(share).Error +} + +func UpdateShare(share *model.Share) error { + return db.Save(share).Error +} + +func GetSharesByCreator(creatorID uint, pageIndex, pageSize int) (shares []model.Share, count int64, err error) { + tx := db.Model(&model.Share{}).Where("creator_id = ?", creatorID) + err = tx.Count(&count).Error + if err != nil { + return nil, 0, err + } + err = tx.Order("created_at desc").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&shares).Error + return +} + +func DeleteShareByShareID(creatorID uint, shareID string) error { + return db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Delete(&model.Share{}).Error +} + +func TouchShareView(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ?", shareID). + UpdateColumns(map[string]interface{}{ + "last_access_at": now, + "view_count": gorm.Expr("view_count + ?", 1), + }).Error +} + +func TouchShareDownload(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ?", shareID). + UpdateColumns(map[string]interface{}{ + "last_access_at": now, + "download_count": gorm.Expr("download_count + ?", 1), + }).Error +} diff --git a/internal/model/share.go b/internal/model/share.go new file mode 100644 index 00000000000..a3fc90af008 --- /dev/null +++ b/internal/model/share.go @@ -0,0 +1,31 @@ +package model + +import "time" + +type Share struct { + ID uint `json:"id" gorm:"primaryKey"` + ShareID string `json:"share_id" gorm:"uniqueIndex;size:32;not null"` + CreatorID uint `json:"creator_id" gorm:"index;not null"` + Name string `json:"name" gorm:"size:255;not null"` + RootPath string `json:"root_path" gorm:"size:4096;not null"` + IsDir bool `json:"is_dir"` + PasswordHash string `json:"-" gorm:"size:64"` + PasswordSalt string `json:"-" gorm:"size:32"` + AllowPreview bool `json:"allow_preview" gorm:"default:true"` + AllowDownload bool `json:"allow_download" gorm:"default:true"` + Enabled bool `json:"enabled" gorm:"default:true;index"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (s Share) HasPassword() bool { + return s.PasswordHash != "" +} + +func (s Share) IsExpired(now time.Time) bool { + return s.ExpiresAt != nil && s.ExpiresAt.Before(now) +} diff --git a/internal/share/access.go b/internal/share/access.go new file mode 100644 index 00000000000..7baa34e3ef9 --- /dev/null +++ b/internal/share/access.go @@ -0,0 +1,31 @@ +package share + +import ( + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + signPkg "github.com/alist-org/alist/v3/pkg/sign" +) + +func tokenPayload(share *model.Share) string { + updatedAt := int64(0) + if !share.UpdatedAt.IsZero() { + updatedAt = share.UpdatedAt.Unix() + } + return fmt.Sprintf("%s:%s:%d", share.ShareID, share.PasswordHash, updatedAt) +} + +func signer() signPkg.Sign { + return signPkg.NewHMACSign([]byte(setting.GetStr(conf.Token) + "-share-access")) +} + +func SignAccess(share *model.Share, d time.Duration) string { + return signer().Sign(tokenPayload(share), time.Now().Add(d).Unix()) +} + +func VerifyAccess(share *model.Share, token string) error { + return signer().Verify(tokenPayload(share), token) +} diff --git a/server/handles/share.go b/server/handles/share.go new file mode 100644 index 00000000000..4626da8f0a2 --- /dev/null +++ b/server/handles/share.go @@ -0,0 +1,368 @@ +package handles + +import ( + "crypto/subtle" + "fmt" + "net/url" + stdpath "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/db" + shareauth "github.com/alist-org/alist/v3/internal/share" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +const shareAccessTokenLifetime = 24 * time.Hour + +type CreateShareReq struct { + Path string `json:"path" binding:"required"` + Name string `json:"name"` + Password string `json:"password"` + ExpireHours int64 `json:"expire_hours"` + AllowPreview *bool `json:"allow_preview"` + AllowDownload *bool `json:"allow_download"` +} + +type ShareDeleteReq struct { + ShareID string `json:"share_id" binding:"required"` +} + +type ShareAuthReq struct { + ShareID string `json:"share_id" binding:"required"` + Password string `json:"password"` +} + +type PublicShareReq struct { + ShareID string `json:"share_id" binding:"required"` + Path string `json:"path"` + Token string `json:"token"` +} + +type PublicShareListReq struct { + model.PageReq + ShareID string `json:"share_id" binding:"required"` + Path string `json:"path"` + Token string `json:"token"` +} + +type ShareResp struct { + ID uint `json:"id"` + ShareID string `json:"share_id"` + Name string `json:"name"` + RootPath string `json:"root_path"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Enabled bool `json:"enabled"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` +} + +type PublicShareInfoResp struct { + ShareID string `json:"share_id"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Authed bool `json:"authed"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` +} + +type PublicShareObjResp struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Thumb string `json:"thumb"` + Type int `json:"type"` + Path string `json:"path"` + StorageClass string `json:"storage_class,omitempty"` + DownloadURL string `json:"download_url,omitempty"` + PreviewURL string `json:"preview_url,omitempty"` +} + +type PublicShareListResp struct { + Content []PublicShareObjResp `json:"content"` + Total int64 `json:"total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + HasMore bool `json:"has_more"` + PagesTotal int `json:"pages_total"` +} + +type PublicShareGetResp struct { + Item PublicShareObjResp `json:"item"` + Provider string `json:"provider"` +} + +func shareURL(c *gin.Context, shareID string) string { + return fmt.Sprintf("%s/s/%s", common.GetApiUrl(c.Request), shareID) +} + +func toShareResp(c *gin.Context, share *model.Share) ShareResp { + return ShareResp{ + ID: share.ID, + ShareID: share.ShareID, + Name: share.Name, + RootPath: share.RootPath, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Enabled: share.Enabled, + ViewCount: share.ViewCount, + DownloadCount: share.DownloadCount, + LastAccessAt: share.LastAccessAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + UpdatedAt: share.UpdatedAt, + URL: shareURL(c, share.ShareID), + } +} + +func normalizeShareName(obj model.Obj, name string) string { + if strings.TrimSpace(name) != "" { + return strings.TrimSpace(name) + } + return obj.GetName() +} + +func generateShareID() (string, error) { + for range 10 { + shareID := random.String(8) + if !db.ShareIDExists(shareID) { + return shareID, nil + } + } + return "", fmt.Errorf("failed to generate unique share id") +} + +func sharePasswordHash(password, salt string) string { + return model.HashPwd(model.StaticHash(password), salt) +} + +func sharePasswordMatched(share *model.Share, password string) bool { + if !share.HasPassword() { + return true + } + hash := sharePasswordHash(password, share.PasswordSalt) + return subtle.ConstantTimeCompare([]byte(hash), []byte(share.PasswordHash)) == 1 +} + +func getShareAccessToken(c *gin.Context, fallback string) string { + if fallback != "" { + return fallback + } + if token := c.Query("auth"); token != "" { + return token + } + return c.GetHeader("X-Share-Token") +} + +func ensureShareAvailable(c *gin.Context, share *model.Share) bool { + now := time.Now() + if !share.Enabled { + common.ErrorStrResp(c, "share is disabled", 404) + return false + } + if share.IsExpired(now) { + common.ErrorStrResp(c, "share is expired", 410) + return false + } + return true +} + +func ensureShareAccess(c *gin.Context, share *model.Share, token string) bool { + if !share.HasPassword() { + return true + } + if token == "" { + common.ErrorStrResp(c, "share password required", 401) + return false + } + if err := shareauth.VerifyAccess(share, token); err != nil { + common.ErrorResp(c, err, 401) + return false + } + return true +} + +func resolveShareTarget(share *model.Share, rawRelPath string) (string, string, error) { + cleanRelPath := utils.FixAndCleanPath(rawRelPath) + if !share.IsDir && cleanRelPath != "/" { + return "", "", fmt.Errorf("file share does not support nested path") + } + if cleanRelPath == "/" { + return share.RootPath, "/", nil + } + target := utils.FixAndCleanPath(stdpath.Join(share.RootPath, cleanRelPath)) + if !utils.IsSubPath(share.RootPath, target) { + return "", "", fmt.Errorf("share path out of range") + } + return target, cleanRelPath, nil +} + +func buildPublicShareAssetURL(c *gin.Context, prefix, shareID, relPath, token string, preview bool) string { + base := common.GetApiUrl(c.Request) + prefix + shareID + cleanPath := utils.FixAndCleanPath(relPath) + if cleanPath != "/" { + base += utils.EncodePath(cleanPath, true) + } + query := url.Values{} + if token != "" { + query.Set("auth", token) + } + if preview { + query.Set("type", "preview") + } + if encoded := query.Encode(); encoded != "" { + base += "?" + encoded + } + return base +} + +func buildPublicSharePreviewURL(c *gin.Context, obj model.Obj, targetPath, shareID, relPath, token string) string { + prefix := "/sd/" + storage, err := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if err == nil && canProxy(storage, obj.GetName()) { + prefix = "/sp/" + } + return buildPublicShareAssetURL(c, prefix, shareID, relPath, token, true) +} + +func toPublicShareObjResp(c *gin.Context, share *model.Share, obj model.Obj, targetPath, relPath, token string) PublicShareObjResp { + thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) + resp := PublicShareObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + Path: relPath, + StorageClass: storageClass, + } + if !obj.IsDir() && share.AllowDownload { + resp.DownloadURL = buildPublicShareAssetURL(c, "/sd/", share.ShareID, relPath, token, false) + } + if !obj.IsDir() && share.AllowPreview { + resp.PreviewURL = buildPublicSharePreviewURL(c, obj, targetPath, share.ShareID, relPath, token) + } + return resp +} + +func CreateShare(c *gin.Context) { + var req CreateShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CanReadPathByRole(user, reqPath) { + common.ErrorStrResp(c, "you have no permission", 403) + return + } + obj, err := fs.Get(c, reqPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + shareID, err := generateShareID() + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + allowPreview := true + if req.AllowPreview != nil { + allowPreview = *req.AllowPreview + } + allowDownload := true + if req.AllowDownload != nil { + allowDownload = *req.AllowDownload + } + var expiresAt *time.Time + if req.ExpireHours > 0 { + expires := time.Now().Add(time.Duration(req.ExpireHours) * time.Hour) + expiresAt = &expires + } + share := &model.Share{ + ShareID: shareID, + CreatorID: user.ID, + Name: normalizeShareName(obj, req.Name), + RootPath: reqPath, + IsDir: obj.IsDir(), + AllowPreview: allowPreview, + AllowDownload: allowDownload, + Enabled: true, + ExpiresAt: expiresAt, + } + if req.Password != "" { + share.PasswordSalt = random.String(16) + share.PasswordHash = sharePasswordHash(req.Password, share.PasswordSalt) + } + if err := db.CreateShare(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, toShareResp(c, share)) +} + +func ListShares(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + user := c.MustGet("user").(*model.User) + shares, total, err := db.GetSharesByCreator(user.ID, req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + resp := make([]ShareResp, 0, len(shares)) + for i := range shares { + resp = append(resp, toShareResp(c, &shares[i])) + } + common.SuccessResp(c, common.PageResp{ + Content: resp, + Total: total, + }) +} + +func DeleteShare(c *gin.Context) { + var req ShareDeleteReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if err := db.DeleteShareByShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/share_page.go b/server/handles/share_page.go new file mode 100644 index 00000000000..98cc62d8daf --- /dev/null +++ b/server/handles/share_page.go @@ -0,0 +1,16 @@ +package handles + +import ( + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/gin-gonic/gin" +) + +func GetSharePage(c *gin.Context) { + c.Header("Content-Type", "text/html") + c.Status(200) + _, _ = c.Writer.WriteString(strings.Replace(conf.IndexHtml, "", "", 1)) + c.Writer.Flush() + c.Writer.WriteHeaderNow() +} diff --git a/server/handles/share_public.go b/server/handles/share_public.go new file mode 100644 index 00000000000..36476bae466 --- /dev/null +++ b/server/handles/share_public.go @@ -0,0 +1,259 @@ +package handles + +import ( + stdpath "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/db" + shareauth "github.com/alist-org/alist/v3/internal/share" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +func GetPublicShareInfo(c *gin.Context) { + shareID := c.Query("share_id") + if shareID == "" { + common.ErrorStrResp(c, "share_id is required", 400) + return + } + share, err := db.GetShareByShareID(shareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, "") + authed := !share.HasPassword() + if share.HasPassword() && token != "" { + authed = shareauth.VerifyAccess(share, token) == nil + } + if authed { + _ = db.TouchShareView(share.ShareID) + } + common.SuccessResp(c, PublicShareInfoResp{ + ShareID: share.ShareID, + Name: share.Name, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Authed: authed, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + }) +} + +func AuthPublicShare(c *gin.Context) { + var req ShareAuthReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !sharePasswordMatched(share, req.Password) { + common.ErrorStrResp(c, "password is incorrect", 403) + return + } + token := "" + if share.HasPassword() { + ttl := shareAccessTokenLifetime + if share.ExpiresAt != nil { + remaining := time.Until(*share.ExpiresAt) + if remaining <= 0 { + common.ErrorStrResp(c, "share is expired", 410) + return + } + if remaining < ttl { + ttl = remaining + } + } + token = shareauth.SignAccess(share, ttl) + } + _ = db.TouchShareView(share.ShareID) + common.SuccessResp(c, gin.H{"token": token}) +} + +func ListPublicShare(c *gin.Context) { + var req PublicShareListReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Page, req.PerPage = normalizeListPage(req.Page, req.PerPage) + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, req.Token) + if !ensureShareAccess(c, share, token) { + return + } + targetPath, relPath, err := resolveShareTarget(share, req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !obj.IsDir() { + common.ErrorStrResp(c, "path is not a directory", 400) + return + } + objs, err := fs.List(c, targetPath, &fs.ListArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + total, pageObjs := pagination(objs, &req.PageReq) + content := make([]PublicShareObjResp, 0, len(pageObjs)) + for _, item := range pageObjs { + itemRelPath := relPath + if itemRelPath == "/" { + itemRelPath = stdpath.Join("/", item.GetName()) + } else { + itemRelPath = stdpath.Join(relPath, item.GetName()) + } + itemTargetPath, _, err := resolveShareTarget(share, itemRelPath) + if err != nil { + continue + } + content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) + } + common.SuccessResp(c, PublicShareListResp{ + Content: content, + Total: int64(total), + Page: req.Page, + PerPage: req.PerPage, + HasMore: req.Page*req.PerPage < total, + PagesTotal: calcPagesTotal(total, req.PerPage), + }) +} + +func GetPublicShare(c *gin.Context) { + var req PublicShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, req.Token) + if !ensureShareAccess(c, share, token) { + return + } + targetPath, relPath, err := resolveShareTarget(share, req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + provider := "unknown" + storage, storageErr := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if storageErr == nil { + provider = storage.GetStorage().Driver + } + common.SuccessResp(c, PublicShareGetResp{ + Item: toPublicShareObjResp(c, share, obj, targetPath, relPath, token), + Provider: provider, + }) +} + +func ShareDown(c *gin.Context) { + share, err := db.GetShareByShareID(c.Param("share_id")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !share.AllowDownload { + common.ErrorStrResp(c, "download is not allowed", 403) + return + } + token := getShareAccessToken(c, "") + if !ensureShareAccess(c, share, token) { + return + } + targetPath, _, err := resolveShareTarget(share, strings.TrimPrefix(c.Param("path"), "/")) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "directory download is not supported", 400) + return + } + _ = db.TouchShareDownload(share.ShareID) + c.Set("path", targetPath) + Down(c) +} + +func ShareProxy(c *gin.Context) { + share, err := db.GetShareByShareID(c.Param("share_id")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !share.AllowPreview { + common.ErrorStrResp(c, "preview is not allowed", 403) + return + } + token := getShareAccessToken(c, "") + if !ensureShareAccess(c, share, token) { + return + } + targetPath, _, err := resolveShareTarget(share, strings.TrimPrefix(c.Param("path"), "/")) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "directory preview is not supported", 400) + return + } + c.Set("path", targetPath) + Proxy(c) +} diff --git a/server/router.go b/server/router.go index 63503838af7..e63b667297e 100644 --- a/server/router.go +++ b/server/router.go @@ -47,6 +47,16 @@ func Init(e *gin.Engine) { g.GET("/p/*path", signCheck, downloadLimiter, handles.Proxy) g.HEAD("/d/*path", signCheck, handles.Down) g.HEAD("/p/*path", signCheck, handles.Proxy) + g.GET("/s/:share_id", handles.GetSharePage) + g.GET("/s/:share_id/*path", handles.GetSharePage) + g.GET("/sd/:share_id", downloadLimiter, handles.ShareDown) + g.GET("/sd/:share_id/*path", downloadLimiter, handles.ShareDown) + g.HEAD("/sd/:share_id", handles.ShareDown) + g.HEAD("/sd/:share_id/*path", handles.ShareDown) + g.GET("/sp/:share_id", downloadLimiter, handles.ShareProxy) + g.GET("/sp/:share_id/*path", downloadLimiter, handles.ShareProxy) + g.HEAD("/sp/:share_id", handles.ShareProxy) + g.HEAD("/sp/:share_id/*path", handles.ShareProxy) archiveSignCheck := middlewares.Down(sign.VerifyArchive) g.GET("/ad/*path", archiveSignCheck, downloadLimiter, handles.ArchiveDown) g.GET("/ap/*path", archiveSignCheck, downloadLimiter, handles.ArchiveProxy) @@ -93,8 +103,16 @@ func Init(e *gin.Engine) { public.Any("/settings", handles.PublicSettings) public.Any("/offline_download_tools", handles.OfflineDownloadTools) public.Any("/archive_extensions", handles.ArchiveExtensions) + public.GET("/share/info", handles.GetPublicShareInfo) + public.POST("/share/auth", handles.AuthPublicShare) + public.POST("/share/list", handles.ListPublicShare) + public.POST("/share/get", handles.GetPublicShare) _fs(auth.Group("/fs")) + share := auth.Group("/share", middlewares.AuthNotGuest) + share.POST("/create", handles.CreateShare) + share.GET("/list", handles.ListShares) + share.POST("/delete", handles.DeleteShare) _task(auth.Group("/task", middlewares.AuthNotGuest)) _label(auth.Group("/label")) _labelFileBinding(auth.Group("/label_file_binding")) From 35832d0431aa7bab16768abf863373c846956f58 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 24 Mar 2026 18:15:25 +0800 Subject: [PATCH 607/659] fix(139): recover personal host for personal_new requests --- drivers/139/driver.go | 24 ++++-------------------- drivers/139/util.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index a57609bc550..c182dd24e93 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -47,28 +47,12 @@ func (d *Yun139) Init(ctx context.Context) error { if err != nil { return err } - - // Query Route Policy - var resp QueryRoutePolicyResp - _, err = d.requestRoute(base.Json{ - "userInfo": base.Json{ - "userType": 1, - "accountType": 1, - "accountName": d.Account}, - "modAddrType": 1, - }, &resp) - if err != nil { - return err - } - for _, policyItem := range resp.Data.RoutePolicyList { - if policyItem.ModName == "personal" { - d.PersonalCloudHost = policyItem.HttpsUrl - break + if d.Addition.Type == MetaPersonalNew { + err = d.ensurePersonalCloudHost() + if err != nil { + return err } } - if len(d.PersonalCloudHost) == 0 { - return fmt.Errorf("PersonalCloudHost is empty") - } d.cron = cron.NewCron(time.Hour * 12) d.cron.Do(func() { diff --git a/drivers/139/util.go b/drivers/139/util.go index 5adc39b4116..79c78842470 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -215,6 +215,46 @@ func (d *Yun139) requestRoute(data interface{}, resp interface{}) ([]byte, error return res.Body(), nil } +func (d *Yun139) ensurePersonalCloudHost() error { + if d.ref != nil { + return d.ref.ensurePersonalCloudHost() + } + if d.PersonalCloudHost != "" { + return nil + } + if len(d.Authorization) == 0 { + return fmt.Errorf("authorization is empty") + } + if d.Account == "" { + if err := d.refreshToken(); err != nil { + return err + } + } + + var resp QueryRoutePolicyResp + _, err := d.requestRoute(base.Json{ + "userInfo": base.Json{ + "userType": 1, + "accountType": 1, + "accountName": d.Account, + }, + "modAddrType": 1, + }, &resp) + if err != nil { + return err + } + for _, policyItem := range resp.Data.RoutePolicyList { + if policyItem.ModName == "personal" && policyItem.HttpsUrl != "" { + d.PersonalCloudHost = strings.TrimRight(policyItem.HttpsUrl, "/") + break + } + } + if d.PersonalCloudHost == "" { + return fmt.Errorf("personal cloud host is empty") + } + return nil +} + func (d *Yun139) post(pathname string, data interface{}, resp interface{}) ([]byte, error) { return d.request(pathname, http.MethodPost, func(req *resty.Request) { req.SetBody(data) @@ -449,6 +489,9 @@ func unicode(str string) string { } func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if err := d.ensurePersonalCloudHost(); err != nil { + return nil, err + } url := d.getPersonalCloudHost() + pathname req := base.RestyClient.R() randStr := random.String(16) From 37a6b266be9b74697841db630ad41406753dfc64 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 25 Mar 2026 16:24:10 +0800 Subject: [PATCH 608/659] fix: handle aliyundrive open rate limits --- drivers/aliyundrive_open/driver.go | 47 ++++----- drivers/aliyundrive_open/limiter.go | 97 ++++++++++++++++++ drivers/aliyundrive_open/upload.go | 16 +-- drivers/aliyundrive_open/util.go | 150 +++++++++++++++++++++++++--- 4 files changed, 260 insertions(+), 50 deletions(-) create mode 100644 drivers/aliyundrive_open/limiter.go diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index 394eadb1b8c..5ef814184e3 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -3,12 +3,10 @@ package aliyundrive_open import ( "context" "errors" - "fmt" "net/http" "path/filepath" "time" - "github.com/Xhofe/rateg" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -24,9 +22,8 @@ type AliyundriveOpen struct { DriveId string - limitList func(ctx context.Context, data base.Json) (*Files, error) - limitLink func(ctx context.Context, file model.Obj) (*model.Link, error) - ref *AliyundriveOpen + limiter *limiter + ref *AliyundriveOpen } func (d *AliyundriveOpen) Config() driver.Config { @@ -38,25 +35,23 @@ func (d *AliyundriveOpen) GetAddition() driver.Additional { } func (d *AliyundriveOpen) Init(ctx context.Context) error { + d.limiter = getLimiterForUser(globalLimiterUserID) if d.LIVPDownloadFormat == "" { d.LIVPDownloadFormat = "jpeg" } if d.DriveType == "" { d.DriveType = "default" } - res, err := d.request("/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) + res, err := d.request(ctx, limiterOther, "/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) if err != nil { + d.limiter.free() + d.limiter = nil return err } d.DriveId = utils.Json.Get(res, d.DriveType+"_drive_id").ToString() - d.limitList = rateg.LimitFnCtx(d.list, rateg.LimitFnOption{ - Limit: 4, - Bucket: 1, - }) - d.limitLink = rateg.LimitFnCtx(d.link, rateg.LimitFnOption{ - Limit: 1, - Bucket: 1, - }) + userID := utils.Json.Get(res, "user_id").ToString() + d.limiter.free() + d.limiter = getLimiterForUser(userID) return nil } @@ -70,6 +65,8 @@ func (d *AliyundriveOpen) InitReference(storage driver.Driver) error { } func (d *AliyundriveOpen) Drop(ctx context.Context) error { + d.limiter.free() + d.limiter = nil d.ref = nil return nil } @@ -87,9 +84,6 @@ func (d *AliyundriveOpen) GetRoot(ctx context.Context) (model.Obj, error) { } func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if d.limitList == nil { - return nil, fmt.Errorf("driver not init") - } files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err @@ -108,7 +102,7 @@ func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.Li } func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link, error) { - res, err := d.request("/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { + res, err := d.request(ctx, limiterLink, "/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), @@ -133,16 +127,13 @@ func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link } func (d *AliyundriveOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - if d.limitLink == nil { - return nil, fmt.Errorf("driver not init") - } - return d.limitLink(ctx, file) + return d.link(ctx, file) } func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { nowTime, _ := getNowTime() newDir := File{CreatedAt: nowTime, UpdatedAt: nowTime} - _, err := d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "parent_file_id": parentDir.GetID(), @@ -168,7 +159,7 @@ func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirN func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var resp MoveOrCopyResp - _, err := d.request("/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -198,7 +189,7 @@ func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (m func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { var newFile File - _, err := d.request("/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -230,7 +221,7 @@ func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName func (d *AliyundriveOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { var resp MoveOrCopyResp - _, err := d.request("/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -256,7 +247,7 @@ func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { if d.RemoveWay == "delete" { uri = "/adrive/v1.0/openFile/delete" } - _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": obj.GetID(), @@ -295,7 +286,7 @@ func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (inte default: return nil, errs.NotSupport } - _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { diff --git a/drivers/aliyundrive_open/limiter.go b/drivers/aliyundrive_open/limiter.go new file mode 100644 index 00000000000..5138fbe2e50 --- /dev/null +++ b/drivers/aliyundrive_open/limiter.go @@ -0,0 +1,97 @@ +package aliyundrive_open + +import ( + "context" + "fmt" + "sync" + + "golang.org/x/time/rate" +) + +// Aliyun Open API rate limits are per user per app, so requests for the same +// user should share one limiter across all storage instances. +type limiterType int + +const ( + limiterList limiterType = iota + limiterLink + limiterOther +) + +const ( + listRateLimit = 3.9 + linkRateLimit = 0.9 + otherRateLimit = 14.9 + globalLimiterUserID = "" +) + +type limiter struct { + usedBy int + list *rate.Limiter + link *rate.Limiter + other *rate.Limiter +} + +var ( + limiters = make(map[string]*limiter) + limitersLock sync.Mutex +) + +func getLimiterForUser(userID string) *limiter { + limitersLock.Lock() + defer limitersLock.Unlock() + defer func() { + for id, lim := range limiters { + if lim.usedBy <= 0 && id != globalLimiterUserID { + delete(limiters, id) + } + } + }() + if lim, ok := limiters[userID]; ok { + lim.usedBy++ + return lim + } + lim := &limiter{ + usedBy: 1, + list: rate.NewLimiter(rate.Limit(listRateLimit), 1), + link: rate.NewLimiter(rate.Limit(linkRateLimit), 1), + other: rate.NewLimiter(rate.Limit(otherRateLimit), 1), + } + limiters[userID] = lim + return lim +} + +func (l *limiter) wait(ctx context.Context, typ limiterType) error { + if l == nil { + return fmt.Errorf("driver not init") + } + switch typ { + case limiterList: + return l.list.Wait(ctx) + case limiterLink: + return l.link.Wait(ctx) + case limiterOther: + return l.other.Wait(ctx) + default: + return fmt.Errorf("unknown limiter type") + } +} + +func (l *limiter) free() { + if l == nil { + return + } + limitersLock.Lock() + defer limitersLock.Unlock() + l.usedBy-- +} + +func (d *AliyundriveOpen) wait(ctx context.Context, typ limiterType) error { + if d == nil { + return fmt.Errorf("driver not init") + } + if d.ref != nil { + return d.ref.wait(ctx, typ) + } + return d.limiter.wait(ctx, typ) +} diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index 4114c195182..19a3c3d0f05 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -50,10 +50,10 @@ func calPartSize(fileSize int64) int64 { return partSize } -func (d *AliyundriveOpen) getUploadUrl(count int, fileId, uploadId string) ([]PartInfo, error) { +func (d *AliyundriveOpen) getUploadUrl(ctx context.Context, count int, fileId, uploadId string) ([]PartInfo, error) { partInfoList := makePartInfos(count) var resp CreateResp - _, err := d.request("/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, @@ -84,10 +84,10 @@ func (d *AliyundriveOpen) uploadPart(ctx context.Context, r io.Reader, partInfo return nil } -func (d *AliyundriveOpen) completeUpload(fileId, uploadId string) (model.Obj, error) { +func (d *AliyundriveOpen) completeUpload(ctx context.Context, fileId, uploadId string) (model.Obj, error) { // 3. complete var newFile File - _, err := d.request("/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, @@ -183,7 +183,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m createData["pre_hash"] = hash } var createResp CreateResp - _, err, e := d.requestReturnErrResp("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { @@ -208,7 +208,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m if err != nil { return nil, fmt.Errorf("cal proof code error: %s", err.Error()) } - _, err = d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err = d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { @@ -229,7 +229,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } // refresh upload url if 50 minutes passed if time.Since(preTime) > 50*time.Minute { - createResp.PartInfoList, err = d.getUploadUrl(count, createResp.FileId, createResp.UploadId) + createResp.PartInfoList, err = d.getUploadUrl(ctx, count, createResp.FileId, createResp.UploadId) if err != nil { return nil, err } @@ -266,5 +266,5 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m log.Debugf("[aliyundrive_open] create file success, resp: %+v", createResp) // 3. complete - return d.completeUpload(createResp.FileId, createResp.UploadId) + return d.completeUpload(ctx, createResp.FileId, createResp.UploadId) } diff --git a/drivers/aliyundrive_open/util.go b/drivers/aliyundrive_open/util.go index c3cda10aa88..544c8bdd10b 100644 --- a/drivers/aliyundrive_open/util.go +++ b/drivers/aliyundrive_open/util.go @@ -19,13 +19,94 @@ import ( // do others that not defined in Driver interface -func (d *AliyundriveOpen) _refreshToken() (string, string, error) { - url := API_URL + "/oauth/access_token" +const legacyOauthTokenURL = "https://api.alistgo.com/alist/ali_open/token" + +type refreshRateLimitError struct { + message string + retryAfter time.Duration +} + +func (e *refreshRateLimitError) Error() string { + if e.retryAfter > 0 { + return fmt.Sprintf("%s, retry after %s", e.message, e.retryAfter.Round(time.Second)) + } + return e.message +} + +func (d *AliyundriveOpen) _refreshToken(ctx context.Context) (string, string, error) { if d.OauthTokenURL != "" && d.ClientID == "" { - url = d.OauthTokenURL + return d.refreshTokenWithOnlineAPI(ctx) + } + + if d.ClientID == "" || d.ClientSecret == "" { + return "", "", fmt.Errorf("empty ClientID or ClientSecret") + } + return d.refreshTokenWithPost(ctx, API_URL+"/oauth/access_token") +} + +func (d *AliyundriveOpen) refreshTokenWithOnlineAPI(ctx context.Context) (string, string, error) { + // New hosted renew endpoint uses a GET API and returns {"refresh_token","access_token","text"}. + // Older hosted endpoints still expect the legacy POST payload, so we fall back when we detect that shape. + var resp struct { + RefreshToken string `json:"refresh_token"` + AccessToken string `json:"access_token"` + ErrorMessage string `json:"text"` + } + var e ErrResp + if err := d.wait(ctx, limiterOther); err != nil { + return "", "", err + } + res, err := base.RestyClient.R(). + SetResult(&resp). + SetError(&e). + SetQueryParams(map[string]string{ + "refresh_ui": d.RefreshToken, + "server_use": "true", + "driver_txt": "alicloud_qr", + }). + Get(d.OauthTokenURL) + if err != nil { + return "", "", err + } + if resp.RefreshToken != "" && resp.AccessToken != "" { + return resp.RefreshToken, resp.AccessToken, nil + } + if resp.ErrorMessage != "" { + if res != nil && res.StatusCode() == http.StatusTooManyRequests { + return d.refreshTokenWithLegacyFallback(ctx, resp.ErrorMessage, retryAfterFromResponse(res)) + } + return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + } + if res != nil && res.StatusCode() == http.StatusTooManyRequests { + return d.refreshTokenWithLegacyFallback(ctx, http.StatusText(http.StatusTooManyRequests), retryAfterFromResponse(res)) } + if e.Code != "" || e.Message != "" { + return d.refreshTokenWithPost(ctx, d.OauthTokenURL) + } + return "", "", fmt.Errorf("empty token returned from online API") +} + +func (d *AliyundriveOpen) refreshTokenWithLegacyFallback(ctx context.Context, message string, retryAfter time.Duration) (string, string, error) { + if d.OauthTokenURL != legacyOauthTokenURL { + log.Warnf("[ali_open] online refresh API is rate-limited, trying legacy fallback: %s", legacyOauthTokenURL) + if refresh, access, err := d.refreshTokenWithPost(ctx, legacyOauthTokenURL); err == nil { + return refresh, access, nil + } else if _, ok := err.(*refreshRateLimitError); !ok { + return "", "", err + } + } + return "", "", &refreshRateLimitError{ + message: fmt.Sprintf("failed to refresh token: %s", message), + retryAfter: retryAfter, + } +} + +func (d *AliyundriveOpen) refreshTokenWithPost(ctx context.Context, url string) (string, string, error) { //var resp base.TokenResp var e ErrResp + if err := d.wait(ctx, limiterOther); err != nil { + return "", "", err + } res, err := base.RestyClient.R(). //ForceContentType("application/json"). SetBody(base.Json{ @@ -41,6 +122,16 @@ func (d *AliyundriveOpen) _refreshToken() (string, string, error) { return "", "", err } log.Debugf("[ali_open] refresh token response: %s", res.String()) + if res.StatusCode() == http.StatusTooManyRequests { + message := e.Message + if message == "" { + message = http.StatusText(http.StatusTooManyRequests) + } + return "", "", &refreshRateLimitError{ + message: fmt.Sprintf("failed to refresh token: %s", message), + retryAfter: retryAfterFromResponse(res), + } + } if e.Code != "" { return "", "", fmt.Errorf("failed to refresh token: %s", e.Message) } @@ -74,18 +165,29 @@ func getSub(token string) (string, error) { return utils.Json.Get(bs, "sub").ToString(), nil } -func (d *AliyundriveOpen) refreshToken() error { +func (d *AliyundriveOpen) refreshToken(ctx context.Context) error { if d.ref != nil { - return d.ref.refreshToken() + return d.ref.refreshToken(ctx) } - refresh, access, err := d._refreshToken() + refresh, access, err := d._refreshToken(ctx) for i := 0; i < 3; i++ { if err == nil { break + } + if rateLimitErr, ok := err.(*refreshRateLimitError); ok { + wait := rateLimitErr.retryAfter + if wait <= 0 { + wait = time.Duration(i+1) * 2 * time.Second + } + if wait > 15*time.Second { + wait = 15 * time.Second + } + log.Warnf("[ali_open] %s", rateLimitErr.Error()) + time.Sleep(wait) } else { log.Errorf("[ali_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, err = d._refreshToken(ctx) } if err != nil { return err @@ -96,12 +198,29 @@ func (d *AliyundriveOpen) refreshToken() error { return nil } -func (d *AliyundriveOpen) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { - b, err, _ := d.requestReturnErrResp(uri, method, callback, retry...) +func retryAfterFromResponse(res *resty.Response) time.Duration { + if res == nil { + return 0 + } + retryAfter := strings.TrimSpace(res.Header().Get("Retry-After")) + if retryAfter == "" { + return 0 + } + if seconds, err := time.ParseDuration(retryAfter + "s"); err == nil { + return seconds + } + if t, err := http.ParseTime(retryAfter); err == nil { + return time.Until(t) + } + return 0 +} + +func (d *AliyundriveOpen) request(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { + b, err, _ := d.requestReturnErrResp(ctx, limitTy, uri, method, callback, retry...) return b, err } -func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { +func (d *AliyundriveOpen) requestReturnErrResp(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { req := base.RestyClient.R() // TODO check whether access_token is expired req.SetHeader("Authorization", "Bearer "+d.getAccessToken()) @@ -113,6 +232,9 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base } var e ErrResp req.SetError(&e) + if err := d.wait(ctx, limitTy); err != nil { + return nil, err, nil + } res, err := req.Execute(method, API_URL+uri) if err != nil { if res != nil { @@ -123,11 +245,11 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base isRetry := len(retry) > 0 && retry[0] if e.Code != "" { if !isRetry && (utils.SliceContains([]string{"AccessTokenInvalid", "AccessTokenExpired", "I400JD"}, e.Code) || d.getAccessToken() == "") { - err = d.refreshToken() + err = d.refreshToken(ctx) if err != nil { return nil, err, nil } - return d.requestReturnErrResp(uri, method, callback, true) + return d.requestReturnErrResp(ctx, limitTy, uri, method, callback, true) } return nil, fmt.Errorf("%s:%s", e.Code, e.Message), &e } @@ -136,7 +258,7 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base func (d *AliyundriveOpen) list(ctx context.Context, data base.Json) (*Files, error) { var resp Files - _, err := d.request("/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterList, "/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { @@ -165,7 +287,7 @@ func (d *AliyundriveOpen) getFiles(ctx context.Context, fileId string) ([]File, //"video_thumbnail_width": 480, //"image_thumbnail_width": 480, } - resp, err := d.limitList(ctx, data) + resp, err := d.list(ctx, data) if err != nil { return nil, err } From c07cd1bf5b99d762a8ec4f4e5f68eec2cf14056f Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 25 Mar 2026 20:48:09 +0800 Subject: [PATCH 609/659] feat(driver): add baidu youth driver --- drivers/all.go | 1 + drivers/baidu_youth/driver.go | 425 ++++++++++++++++++++++++ drivers/baidu_youth/meta.go | 40 +++ drivers/baidu_youth/types.go | 130 ++++++++ drivers/baidu_youth/util.go | 606 ++++++++++++++++++++++++++++++++++ internal/driver/proxy.go | 7 + server/common/check.go | 6 +- server/handles/fsread.go | 19 +- 8 files changed, 1228 insertions(+), 6 deletions(-) create mode 100644 drivers/baidu_youth/driver.go create mode 100644 drivers/baidu_youth/meta.go create mode 100644 drivers/baidu_youth/types.go create mode 100644 drivers/baidu_youth/util.go create mode 100644 internal/driver/proxy.go diff --git a/drivers/all.go b/drivers/all.go index a4fce9d0cac..1f096a86224 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -21,6 +21,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_netdisk" _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" + _ "github.com/alist-org/alist/v3/drivers/baidu_youth" _ "github.com/alist-org/alist/v3/drivers/bitqiu" _ "github.com/alist-org/alist/v3/drivers/chaoxing" _ "github.com/alist-org/alist/v3/drivers/cloudreve" diff --git a/drivers/baidu_youth/driver.go b/drivers/baidu_youth/driver.go new file mode 100644 index 00000000000..a0b6a4853ec --- /dev/null +++ b/drivers/baidu_youth/driver.go @@ -0,0 +1,425 @@ +package baidu_youth + +import ( + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" +) + +type BaiduYouth struct { + model.Storage + Addition + + uk int64 + bdstoken string + sk string + uploadThread int + upClient *resty.Client +} + +var ErrUploadIDExpired = errors.New("uploadid expired") + +func (d *BaiduYouth) Config() driver.Config { + return config +} + +func (d *BaiduYouth) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BaiduYouth) Init(ctx context.Context) error { + d.Cookie = strings.TrimSpace(d.Cookie) + if d.Storage.Addition != "" { + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(d.Storage.Addition), &raw); err == nil { + if _, ok := raw["force_proxy"]; !ok { + d.ForceProxy = true + } + } + } + d.upClient = base.NewRestyClient(). + SetTimeout(UPLOAD_TIMEOUT). + SetRetryCount(UPLOAD_RETRY_COUNT). + SetRetryWaitTime(UPLOAD_RETRY_WAIT_TIME). + SetRetryMaxWaitTime(UPLOAD_RETRY_MAX_WAIT_TIME) + + d.uploadThread, _ = strconv.Atoi(d.UploadThread) + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" + } + + u, err := url.Parse(d.UploadAPI) + if d.UploadAPI == "" || err != nil || u.Scheme == "" || u.Host == "" { + d.UploadAPI = UPLOAD_FALLBACK_API + } else { + d.UploadAPI = strings.TrimRight(d.UploadAPI, "/") + } + + uk, bdstoken, sk, err := d.getUserSession(ctx) + if err != nil { + return err + } + d.uk = uk + d.bdstoken = bdstoken + d.sk = sk + return nil +} + +func (d *BaiduYouth) ShouldProxyDownloads() bool { + return d.ForceProxy +} + +func (d *BaiduYouth) Drop(ctx context.Context) error { + d.uk = 0 + d.bdstoken = "" + d.sk = "" + return nil +} + +func (d *BaiduYouth) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetPath()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *BaiduYouth) Get(ctx context.Context, path string) (model.Obj, error) { + return d.getByPath(ctx, path) +} + +func (d *BaiduYouth) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if d.DownloadAPI == "crack" { + return d.linkCrack(ctx, file) + } + return d.linkOfficial(ctx, file) +} + +func (d *BaiduYouth) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + path := stdpath.Join(parentDir.GetPath(), dirName) + var resp CreateResp + _, err := d.postForm(ctx, "/youth/api/create", map[string]string{ + "a": "commit", + "bdstoken": d.bdstoken, + }, map[string]string{ + "block_list": "[]", + "isdir": "1", + "path": path, + }, &resp) + if err != nil { + return nil, err + } + if result := resp.ResultFile(); result.Path != "" || result.FsId != 0 { + return fileToObj(result), nil + } + return d.getByPath(ctx, path) +} + +func (d *BaiduYouth) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + _, err := d.manage(ctx, "move", []base.Json{ + { + "dest": dstDir.GetPath(), + "newname": srcObj.GetName(), + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + newPath := stdpath.Join(dstDir.GetPath(), srcObj.GetName()) + if obj, ok := srcObj.(*model.ObjThumb); ok { + obj.SetPath(newPath) + obj.Modified = time.Now() + return obj, nil + } + return d.getByPath(ctx, newPath) +} + +func (d *BaiduYouth) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + _, err := d.manage(ctx, "rename", []base.Json{ + { + "id": srcObj.GetID(), + "newname": newName, + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + newPath := stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName) + if obj, ok := srcObj.(*model.ObjThumb); ok { + obj.SetPath(newPath) + obj.Name = newName + obj.Modified = time.Now() + return obj, nil + } + return d.getByPath(ctx, newPath) +} + +func (d *BaiduYouth) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + newPath := stdpath.Join(dstDir.GetPath(), srcObj.GetName()) + _, err := d.manage(ctx, "copy", []base.Json{ + { + "dest": dstDir.GetPath(), + "newname": srcObj.GetName(), + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + if obj, ok := srcObj.(*model.ObjThumb); ok { + copied := *obj + copied.SetPath(newPath) + copied.Modified = time.Now() + return &copied, nil + } + // Youth copy returns success before /api/filemetas can resolve the new path. + // Avoid turning a successful copy into a false failure because the immediate + // post-copy lookup is temporarily unavailable. + return &model.Object{ + ID: newPath, + Path: newPath, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + HashInfo: srcObj.GetHash(), + }, nil +} + +func (d *BaiduYouth) Remove(ctx context.Context, obj model.Obj) error { + _, err := d.manage(ctx, "delete", []string{obj.GetPath()}) + return err +} + +func (d *BaiduYouth) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if stream.GetSize() < 1 { + return nil, ErrBaiduYouthEmptyFilesNotAllowed + } + + var ( + cache = stream.GetFile() + tmpF *os.File + err error + ) + if _, ok := cache.(io.ReaderAt); !ok { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF + } + + streamSize := stream.GetSize() + sliceSize := DefaultSliceSize + count := int(streamSize / sliceSize) + lastBlockSize := streamSize % sliceSize + if lastBlockSize > 0 { + count++ + } else { + lastBlockSize = sliceSize + } + + const sliceMD5Size int64 = 256 * utils.KB + blockList := make([]string, 0, count) + byteSize := sliceSize + fileMd5H := md5.New() + sliceMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceMd5H2Writer := utils.LimitWriter(sliceMd5H2, sliceMD5Size) + writers := []io.Writer{fileMd5H, sliceMd5H, sliceMd5H2Writer} + if tmpF != nil { + writers = append(writers, tmpF) + } + + written := int64(0) + for i := 1; i <= count; i++ { + if utils.IsCanceled(ctx) { + return nil, ctx.Err() + } + if i == count { + byteSize = lastBlockSize + } + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) + written += n + if err != nil && err != io.EOF { + return nil, err + } + blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) + sliceMd5H.Reset() + } + + if tmpF != nil { + if written != streamSize { + return nil, errs.NewErr(errs.StreamIncomplete, "temp file size mismatch: %d != %d", written, streamSize) + } + if _, err = tmpF.Seek(0, io.SeekStart); err != nil { + return nil, err + } + } + + contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) + sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) + blockListStr, err := utils.Json.MarshalToString(blockList) + if err != nil { + return nil, err + } + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) + mtime := stream.ModTime().Unix() + ctime := stream.CreateTime().Unix() + + progressKey := d.uploadProgressKey() + precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, progressKey, contentMd5) + if !ok { + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) + if err != nil { + return nil, err + } + } + + if precreateResp.ReturnType >= 2 { + result := precreateResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil + } + + cacheReaderAt, ok := cache.(io.ReaderAt) + if !ok { + return nil, fmt.Errorf("cache object must implement io.ReaderAt") + } + +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + completed := count - len(precreateResp.BlockList) + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue + } + + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "partseq": strconv.Itoa(partseq), + "path": path, + "type": "tmpfile", + "uploadid": precreateResp.Uploadid, + } + if precreateResp.Uploadsign != "" { + params["uploadsign"] = precreateResp.Uploadsign + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + if err := d.uploadSlice(ctx, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)); err != nil { + return err + } + precreateResp.BlockList[i] = -1 + success := completed + int(threadG.Success()) + 1 + up(float64(success) * 100 / float64(count)) + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(part int) bool { + return part >= 0 + }) + base.SaveUploadProgress(d, precreateResp, progressKey, contentMd5) + + if errors.Is(err, context.Canceled) { + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) + if err != nil { + return nil, err + } + if precreateResp.ReturnType >= 2 { + result := precreateResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil + } + base.SaveUploadProgress(d, precreateResp, progressKey, contentMd5) + continue uploadLoop + } + return nil, err + } + + var createResp CreateResp + _, err = d.createFile(ctx, path, stdpath.Dir(path), streamSize, precreateResp.Uploadid, precreateResp.Uploadsign, blockListStr, &createResp, mtime, ctime) + if err != nil { + return nil, err + } + + base.SaveUploadProgress(d, nil, progressKey, contentMd5) + result := createResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil +} + +var _ driver.Driver = (*BaiduYouth)(nil) +var _ driver.Getter = (*BaiduYouth)(nil) +var _ driver.MkdirResult = (*BaiduYouth)(nil) +var _ driver.MoveResult = (*BaiduYouth)(nil) +var _ driver.RenameResult = (*BaiduYouth)(nil) +var _ driver.CopyResult = (*BaiduYouth)(nil) +var _ driver.Remove = (*BaiduYouth)(nil) +var _ driver.PutResult = (*BaiduYouth)(nil) diff --git a/drivers/baidu_youth/meta.go b/drivers/baidu_youth/meta.go new file mode 100644 index 00000000000..7d5cb6a8698 --- /dev/null +++ b/drivers/baidu_youth/meta.go @@ -0,0 +1,40 @@ +package baidu_youth + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Cookie string `json:"cookie" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + ForceProxy bool `json:"force_proxy" type:"bool" default:"true" help:"Proxy downloads through AList. Disable to redirect the browser to a fresh Baidu direct link."` + DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` +} + +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" + UPLOAD_TIMEOUT = time.Minute * 30 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 + DefaultSliceSize int64 = 4 * 1024 * 1024 +) + +var config = driver.Config{ + Name: "BaiduYouth", + DefaultRoot: "/", + OnlyProxy: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BaiduYouth{} + }) +} diff --git a/drivers/baidu_youth/types.go b/drivers/baidu_youth/types.go new file mode 100644 index 00000000000..82024009403 --- /dev/null +++ b/drivers/baidu_youth/types.go @@ -0,0 +1,130 @@ +package baidu_youth + +import ( + "errors" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var ( + ErrBaiduYouthEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu youth") +) + +type File struct { + Category int `json:"category"` + FsId int64 `json:"fs_id"` + Thumbs struct { + Url3 string `json:"url3"` + } `json:"thumbs"` + Size int64 `json:"size"` + Path string `json:"path"` + ServerFilename string `json:"server_filename"` + Md5 string `json:"md5"` + Isdir int `json:"isdir"` + ServerCtime int64 `json:"server_ctime"` + ServerMtime int64 `json:"server_mtime"` + LocalMtime int64 `json:"local_mtime"` + LocalCtime int64 `json:"local_ctime"` + Ctime int64 `json:"ctime"` + Mtime int64 `json:"mtime"` + Dlink string `json:"dlink"` +} + +func fileToObj(f File) *model.ObjThumb { + if f.ServerFilename == "" { + f.ServerFilename = path.Base(f.Path) + } + if f.ServerCtime == 0 { + if f.LocalCtime != 0 { + f.ServerCtime = f.LocalCtime + } else { + f.ServerCtime = f.Ctime + } + } + if f.ServerMtime == 0 { + if f.LocalMtime != 0 { + f.ServerMtime = f.LocalMtime + } else { + f.ServerMtime = f.Mtime + } + } + return &model.ObjThumb{ + Object: model.Object{ + ID: strconv.FormatInt(f.FsId, 10), + Path: f.Path, + Name: f.ServerFilename, + Size: f.Size, + Modified: time.Unix(f.ServerMtime, 0), + Ctime: time.Unix(f.ServerCtime, 0), + IsFolder: f.Isdir == 1, + HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, + } +} + +type ListResp struct { + Errno int `json:"errno"` + List []File `json:"list"` +} + +type FileMetaResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Info []File `json:"info"` + List []File `json:"list"` +} + +type LocateDownloadResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Path string `json:"path"` + URL string `json:"url"` +} + +type MediaInfoResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Info File `json:"info"` +} + +type CreateResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Info File `json:"info"` + File +} + +func (r CreateResp) ResultFile() File { + if r.Info.Path != "" || r.Info.FsId != 0 { + return r.Info + } + return r.File +} + +type PrecreateResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + ReturnType int `json:"return_type"` + Path string `json:"path"` + Uploadid string `json:"uploadid"` + Uploadsign string `json:"uploadsign"` + BlockList []int `json:"block_list"` + Info File `json:"info"` + File +} + +func (r PrecreateResp) ResultFile() File { + if r.Info.Path != "" || r.Info.FsId != 0 { + return r.Info + } + return r.File +} diff --git a/drivers/baidu_youth/util.go b/drivers/baidu_youth/util.go new file mode 100644 index 00000000000..b83518fe350 --- /dev/null +++ b/drivers/baidu_youth/util.go @@ -0,0 +1,606 @@ +package baidu_youth + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + stdpath "path" + "strconv" + "strings" + "time" + "unicode" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + panBaseURL = "https://pan.baidu.com" + panReferer = panBaseURL + "/" + panUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36" + youthReferer = panBaseURL + "/youth/pan/main#/index?category=all" + youthAppID = "250528" + uploadAppID = "25179614" + videoAPIChannel = "android_15_25010PN30C_bd-netdisk_1523a" + videoAPIDevUID = "0%1" +) + +func (d *BaiduYouth) normalizeURL(furl string) string { + if strings.HasPrefix(furl, "http://") || strings.HasPrefix(furl, "https://") { + return furl + } + return panBaseURL + furl +} + +func (d *BaiduYouth) commonHeaders() map[string]string { + return map[string]string{ + "Accept": "application/json, text/plain, */*", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Cache-Control": "no-cache", + "Cookie": d.Cookie, + "Origin": panBaseURL, + "Pragma": "no-cache", + "Referer": youthReferer, + "User-Agent": panUserAgent, + "X-Requested-With": "XMLHttpRequest", + } +} + +func youthQueryParams() map[string]string { + return map[string]string{ + "app_id": youthAppID, + "clienttype": "0", + "web": "1", + } +} + +func youthUploadQueryParams() map[string]string { + return map[string]string{ + "app_id": uploadAppID, + "channel": "chunlei", + "clienttype": "0", + "web": "1", + } +} + +func extractBaiduMessage(body []byte) string { + for _, path := range [][]interface{}{ + {"show_msg"}, + {"errmsg"}, + {"error_msg"}, + {"error_description"}, + {"message"}, + } { + msg := utils.Json.Get(body, path...).ToString() + if msg != "" { + return msg + } + } + return "" +} + +func (d *BaiduYouth) doRequest(furl string, method string, defaultQuery map[string]string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R().SetHeaders(d.commonHeaders()) + if defaultQuery != nil { + req.SetQueryParams(defaultQuery) + } + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, d.normalizeURL(furl)) + if err != nil { + return nil, err + } + body := res.Body() + if errno := utils.Json.Get(body, "errno").ToInt(); errno != 0 { + msg := extractBaiduMessage(body) + if errno == -6 && msg == "" { + msg = "cookie expired or invalid" + } + if msg == "" { + msg = "request failed" + } + return nil, fmt.Errorf("[baidu_youth] %s (errno=%d)", msg, errno) + } + return body, nil +} + +func (d *BaiduYouth) get(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, youthQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) postForm(ctx context.Context, pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodPost, youthQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(form) + }, resp) +} + +func (d *BaiduYouth) getUpload(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, youthUploadQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) postUploadForm(ctx context.Context, pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodPost, youthUploadQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(form) + }, resp) +} + +func (d *BaiduYouth) getBare(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, nil, func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) getUserSK(ctx context.Context) (string, error) { + body, err := d.get(ctx, "/youth/api/report/user", map[string]string{ + "action": "sapi_auth", + "timestamp": strconv.FormatInt(time.Now().UnixMilli(), 10), + }, nil) + if err != nil { + return "", err + } + return utils.Json.Get(body, "uinfo").ToString(), nil +} + +func (d *BaiduYouth) getUserSession(ctx context.Context) (int64, string, string, error) { + body, err := d.get(ctx, "/youth/api/user/getinfo", map[string]string{ + "need_selfinfo": "1", + }, nil) + if err != nil { + return 0, "", "", err + } + uk := int64(utils.Json.Get(body, "records", 0, "uk").ToInt()) + bdstoken := utils.Json.Get(body, "records", 0, "bdstoken").ToString() + sk := utils.Json.Get(body, "records", 0, "sk").ToString() + if bdstoken == "" || uk == 0 || sk == "" { + body, err = d.getBare(ctx, "/api/gettemplatevariable", map[string]string{ + "fields": `["bdstoken","uk","sk"]`, + }, nil) + if err != nil { + return 0, "", "", err + } + if uk == 0 { + uk = int64(utils.Json.Get(body, "result", "uk").ToInt()) + } + if bdstoken == "" { + bdstoken = utils.Json.Get(body, "result", "bdstoken").ToString() + } + if sk == "" { + sk = utils.Json.Get(body, "result", "sk").ToString() + } + } + if sk == "" { + sk, _ = d.getUserSK(ctx) + } + if bdstoken == "" { + return 0, "", "", fmt.Errorf("failed to get bdstoken from baidu youth cookie") + } + if uk == 0 { + return 0, "", "", fmt.Errorf("failed to get uk from baidu youth cookie") + } + return uk, bdstoken, sk, nil +} + +func (d *BaiduYouth) getFiles(ctx context.Context, dir string) ([]File, error) { + page := 1 + num := 1000 + params := map[string]string{ + "dir": dir, + } + if d.OrderBy != "" { + params["order"] = d.OrderBy + if d.OrderDirection == "desc" { + params["desc"] = "1" + } else { + params["desc"] = "0" + } + } + files := make([]File, 0) + for { + params["page"] = strconv.Itoa(page) + params["num"] = strconv.Itoa(num) + var resp ListResp + _, err := d.get(ctx, "/youth/api/list", params, &resp) + if err != nil { + return nil, err + } + if len(resp.List) == 0 { + return files, nil + } + files = append(files, resp.List...) + if len(resp.List) < num { + return files, nil + } + page++ + } +} + +func (d *BaiduYouth) getFileByPath(ctx context.Context, path string) (File, error) { + if path == "/" { + return File{ + Path: "/", + ServerFilename: "/", + Isdir: 1, + }, nil + } + target, err := utils.Json.MarshalToString([]string{path}) + if err != nil { + return File{}, err + } + var resp FileMetaResp + _, err = d.get(ctx, "/api/filemetas", map[string]string{ + "target": target, + }, &resp) + if err != nil { + return File{}, err + } + if len(resp.Info) > 0 { + return resp.Info[0], nil + } + if len(resp.List) > 0 { + return resp.List[0], nil + } + return File{}, errs.NewErr(errs.ObjectNotFound, "baidu youth path not found: %s", path) +} + +func (d *BaiduYouth) getByPath(ctx context.Context, path string) (model.Obj, error) { + if path == "/" { + return &model.Object{ + Path: "/", + Name: "/", + IsFolder: true, + }, nil + } + file, err := d.getFileByPath(ctx, path) + if err != nil { + return nil, err + } + return fileToObj(file), nil +} + +func (d *BaiduYouth) linkOfficial(ctx context.Context, file model.Obj) (*model.Link, error) { + return d.buildDownloadLink(ctx, file) +} + +func (d *BaiduYouth) linkCrack(ctx context.Context, file model.Obj) (*model.Link, error) { + return d.buildDownloadLink(ctx, file) +} + +func (d *BaiduYouth) downloadHeaders() http.Header { + return http.Header{ + "Accept": []string{"*/*"}, + "Accept-Language": []string{"zh-CN,zh;q=0.9,en;q=0.8"}, + "Cache-Control": []string{"no-cache"}, + "Pragma": []string{"no-cache"}, + "Referer": []string{panReferer}, + "User-Agent": []string{panUserAgent}, + } +} + +func nextDPLogID() string { + return strconv.FormatInt(time.Now().UnixNano(), 10) +} + +func (d *BaiduYouth) getCurrentUserSK(ctx context.Context) (string, error) { + sk, err := d.getUserSK(ctx) + if err == nil && sk != "" { + return sk, nil + } + if d.sk != "" { + return d.sk, nil + } + if err != nil { + return "", err + } + return "", fmt.Errorf("baidu youth sk is empty") +} + +func (d *BaiduYouth) locatedownloadRand(sk string, nowMilli int64) string { + sum := sha1.Sum([]byte(strconv.FormatInt(d.uk, 10) + sk + strconv.FormatInt(nowMilli, 10) + "0")) + return hex.EncodeToString(sum[:]) +} + +func (d *BaiduYouth) locatedownloadSign(fileMD5 string, fileID string, nowMilli int64) string { + sum := md5.Sum([]byte(fileMD5 + "_" + strconv.FormatInt(d.uk, 10) + "_" + fileID + "_" + strconv.FormatInt(nowMilli, 10))) + return hex.EncodeToString(sum[:]) +} + +func (d *BaiduYouth) resolveDownloadMeta(ctx context.Context, file model.Obj) (string, string, string, error) { + parentDir := stdpath.Dir(file.GetPath()) + if parentDir == "." { + parentDir = "/" + } + + files, err := d.getFiles(ctx, parentDir) + if err != nil { + return "", "", "", err + } + for _, listedFile := range files { + if listedFile.Path != file.GetPath() { + continue + } + fileID := strconv.FormatInt(listedFile.FsId, 10) + if listedFile.Path != "" && fileID != "" && listedFile.Md5 != "" { + return listedFile.Path, fileID, listedFile.Md5, nil + } + return "", "", "", fmt.Errorf("baidu youth list metadata incomplete for %s", file.GetPath()) + } + return "", "", "", errs.NewErr(errs.ObjectNotFound, "baidu youth list metadata not found: %s", file.GetPath()) +} + +func normalizeLocatedownloadURL(rawURL string) (string, error) { + if rawURL == "" { + return "", fmt.Errorf("baidu youth locatedownload url is empty") + } + if !strings.Contains(rawURL, "response-cache-control=") { + sep := "&" + if !strings.Contains(rawURL, "?") { + sep = "?" + } + rawURL += sep + "response-cache-control=private" + } + return rawURL, nil +} + +func (d *BaiduYouth) getMediaInfoDLink(ctx context.Context, file model.Obj) (string, error) { + path, fileID, _, err := d.resolveDownloadMeta(ctx, file) + if err != nil { + return "", err + } + + var resp MediaInfoResp + _, err = d.doRequest("/youth/api/mediainfo", http.MethodGet, nil, func(req *resty.Request) { + req.SetContext(ctx) + req.SetQueryParams(map[string]string{ + "channel": videoAPIChannel, + "clienttype": "1", + "devuid": videoAPIDevUID, + "dlink": "1", + "fs_id": fileID, + "media": "1", + "nom3u8": "1", + "origin": "dlna", + "path": path, + "type": "VideoURL", + }) + }, &resp) + if err != nil { + return "", err + } + if resp.Info.Dlink == "" { + return "", fmt.Errorf("baidu youth video dlink not found for %s", path) + } + return resp.Info.Dlink, nil +} + +func (d *BaiduYouth) buildVideoLink(ctx context.Context, file model.Obj) (*model.Link, error) { + dlink, err := d.getMediaInfoDLink(ctx, file) + if err != nil { + return nil, err + } + return &model.Link{ + URL: dlink, + Header: http.Header{ + "Referer": []string{panReferer}, + }, + }, nil +} + +func (d *BaiduYouth) requestLocateDownloadURL(ctx context.Context, path string, fileID string, fileMD5 string, sk string) (string, error) { + nowMilli := time.Now().UnixMilli() + + var resp LocateDownloadResp + _, err := d.get(ctx, "/youth/api/locatedownload", map[string]string{ + "devuid": "0", + "dp-logid": nextDPLogID(), + "path": path, + "rand": d.locatedownloadRand(sk, nowMilli), + "sign": d.locatedownloadSign(fileMD5, fileID, nowMilli), + "time": strconv.FormatInt(nowMilli, 10), + }, &resp) + if err != nil { + return "", err + } + if resp.URL == "" { + return "", fmt.Errorf("baidu youth locatedownload url not found for %s", path) + } + return normalizeLocatedownloadURL(resp.URL) +} + +func (d *BaiduYouth) getLocateDownloadURL(ctx context.Context, file model.Obj) (string, error) { + path, fileID, fileMD5, err := d.resolveDownloadMeta(ctx, file) + if err != nil { + return "", err + } + + sk, err := d.getCurrentUserSK(ctx) + if err != nil { + return "", err + } + downloadURL, err := d.requestLocateDownloadURL(ctx, path, fileID, fileMD5, sk) + if err == nil { + return downloadURL, nil + } + + if !strings.Contains(err.Error(), "errno=-30006") { + return "", err + } + sk, refreshErr := d.getUserSK(ctx) + if refreshErr != nil { + return "", err + } + if sk == "" { + return "", err + } + return d.requestLocateDownloadURL(ctx, path, fileID, fileMD5, sk) +} + +func (d *BaiduYouth) buildDownloadLink(ctx context.Context, file model.Obj) (*model.Link, error) { + downloadURL, err := d.getLocateDownloadURL(ctx, file) + if err != nil { + return nil, err + } + return &model.Link{ + URL: downloadURL, + Header: d.downloadHeaders(), + }, nil +} + +func (d *BaiduYouth) manage(ctx context.Context, opera string, filelist any) ([]byte, error) { + marshal, err := utils.Json.MarshalToString(filelist) + if err != nil { + return nil, err + } + return d.postForm(ctx, "/youth/api/filemanager", map[string]string{ + "async": "0", + "bdstoken": d.bdstoken, + "onnest": "fail", + "opera": opera, + }, map[string]string{ + "filelist": marshal, + "ondup": "fail", + }, nil) +} + +func (d *BaiduYouth) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + form := map[string]string{ + "autoinit": "1", + "block_list": blockListStr, + "isdir": "0", + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "target_path": stdpath.Dir(path), + } + if contentMd5 != "" { + form["content-md5"] = contentMd5 + } + if sliceMd5 != "" { + form["slice-md5"] = sliceMd5 + } + joinTime(form, ctime, mtime) + var resp PrecreateResp + _, err := d.postUploadForm(ctx, "/youth/api/precreate", map[string]string{ + "bdstoken": d.bdstoken, + }, form, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *BaiduYouth) createFile(ctx context.Context, path, targetPath string, size int64, uploadID, uploadSign, blockList string, resp interface{}, mtime, ctime int64) ([]byte, error) { + form := map[string]string{ + "block_list": blockList, + "path": path, + "size": strconv.FormatInt(size, 10), + "target_path": targetPath, + "uploadid": uploadID, + } + if uploadSign != "" { + form["uploadsign"] = uploadSign + } + joinTime(form, ctime, mtime) + return d.postUploadForm(ctx, "/youth/api/create", map[string]string{ + "bdstoken": d.bdstoken, + "isdir": "0", + }, form, resp) +} + +func joinTime(form map[string]string, ctime, mtime int64) { + if ctime != 0 { + form["local_ctime"] = strconv.FormatInt(ctime, 10) + } + if mtime != 0 { + form["local_mtime"] = strconv.FormatInt(mtime, 10) + } +} + +func (d *BaiduYouth) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { + res, err := d.upClient.R(). + SetContext(ctx). + SetHeaders(d.commonHeaders()). + SetQueryParams(youthUploadQueryParams()). + SetQueryParams(params). + SetFileReader("file", fileName, file). + Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") + if err != nil { + return err + } + body := res.Body() + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + lower := strings.ToLower(string(body)) + if strings.Contains(lower, "uploadid") && (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + if errCode != 0 || errNo != 0 { + msg := extractBaiduMessage(body) + if msg == "" { + msg = "error uploading to baidu youth" + } + return errs.NewErr(errs.StreamIncomplete, "%s: %s", msg, string(body)) + } + return nil +} + +func (d *BaiduYouth) uploadProgressKey() string { + if d.uk != 0 { + return strconv.FormatInt(d.uk, 10) + } + sum := md5.Sum([]byte(d.Cookie)) + return hex.EncodeToString(sum[:]) +} + +func DecryptMd5(encryptMd5 string) string { + if encryptMd5 == "" { + return "" + } + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} diff --git a/internal/driver/proxy.go b/internal/driver/proxy.go new file mode 100644 index 00000000000..f94273c9f92 --- /dev/null +++ b/internal/driver/proxy.go @@ -0,0 +1,7 @@ +package driver + +// ProxyDriver lets a driver override the default "must proxy" download behavior +// on a per-storage basis. +type ProxyDriver interface { + ShouldProxyDownloads() bool +} diff --git a/server/common/check.go b/server/common/check.go index 34aaa41d93a..010d3131745 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -41,7 +41,11 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri // 2. storage.WebProxy // 3. proxy_types func ShouldProxy(storage driver.Driver, filename string) bool { - if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + if proxyDriver, ok := storage.(driver.ProxyDriver); ok { + if proxyDriver.ShouldProxyDownloads() || storage.GetStorage().WebProxy { + return true + } + } else if storage.Config().MustProxy() || storage.GetStorage().WebProxy { return true } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index c370f631fc0..67f84e5d70a 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -377,12 +377,21 @@ func FsGet(c *gin.Context) { common.ErrorResp(c, err, 500) return } + query := "" + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + query = "?sign=" + sign.Sign(reqPath) + } + forceRedirectRawURL := storage.GetStorage().Driver == "BaiduYouth" forceProxyRawURL := storage.GetStorage().Driver == "Quark" && utils.GetFileType(obj.GetName()) == conf.VIDEO - if storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL { - query := "" - if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { - query = "?sign=" + sign.Sign(reqPath) - } + if forceRedirectRawURL { + // Baidu Youth direct links are minted per request and are not stable enough + // to expose as fs/get raw_url. Return the local /d endpoint so the frontend + // obtains a fresh link on each download click. + rawURL = fmt.Sprintf("%s/d%s%s", + common.GetApiUrl(c.Request), + utils.EncodePath(reqPath, true), + query) + } else if storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL { if storage.GetStorage().DownProxyUrl != "" { rawURL = common.BuildDownProxyURL( storage.GetStorage().DownProxyUrl, From 094765ebcf2e9d55637db0ce5c75826eb27b1431 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 26 Mar 2026 10:59:50 +0800 Subject: [PATCH 610/659] feat(chunker): add storage driver --- drivers/all.go | 1 + drivers/chunker/driver.go | 381 +++++++++++++++++++++++ drivers/chunker/meta.go | 43 +++ drivers/chunker/types.go | 72 +++++ drivers/chunker/util.go | 572 +++++++++++++++++++++++++++++++++++ drivers/chunker/util_test.go | 115 +++++++ drivers/crypt/meta.go | 6 +- 7 files changed, 1187 insertions(+), 3 deletions(-) create mode 100644 drivers/chunker/driver.go create mode 100644 drivers/chunker/meta.go create mode 100644 drivers/chunker/types.go create mode 100644 drivers/chunker/util.go create mode 100644 drivers/chunker/util_test.go diff --git a/drivers/all.go b/drivers/all.go index a4fce9d0cac..7f6597561c1 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -23,6 +23,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/baidu_share" _ "github.com/alist-org/alist/v3/drivers/bitqiu" _ "github.com/alist-org/alist/v3/drivers/chaoxing" + _ "github.com/alist-org/alist/v3/drivers/chunker" _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" diff --git a/drivers/chunker/driver.go b/drivers/chunker/driver.go new file mode 100644 index 00000000000..004fde1c476 --- /dev/null +++ b/drivers/chunker/driver.go @@ -0,0 +1,381 @@ +package chunker + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "hash" + "io" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Chunker) Config() driver.Config { + return config +} + +func (d *Chunker) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Chunker) Init(ctx context.Context) error { + if d.ChunkSize == 0 { + d.ChunkSize = defaultChunkSize + } + if d.StartFrom == 0 { + d.StartFrom = defaultStartFrom + } + d.NameFormat = utils.GetNoneEmpty(d.NameFormat, defaultChunkNameFmt) + d.MetaFormat = utils.GetNoneEmpty(d.MetaFormat, defaultMetaFormat) + d.HashType = utils.GetNoneEmpty(d.HashType, defaultHashType) + + if err := d.setChunkNameFormat(d.NameFormat); err != nil { + return fmt.Errorf("invalid name_format: %w", err) + } + if err := d.validateOptions(); err != nil { + return err + } + + storage, err := fs.GetStorage(d.RemotePath, &fs.GetStoragesArgs{}) + if err != nil { + return fmt.Errorf("can't find remote storage: %w", err) + } + d.remoteStorage = storage + return nil +} + +func (d *Chunker) Drop(ctx context.Context) error { + d.remoteStorage = nil + return nil +} + +func (d *Chunker) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.listDirObjects(ctx, dir.GetPath(), args.Refresh) +} + +func (d *Chunker) Get(ctx context.Context, pathStr string) (model.Obj, error) { + if utils.PathEqual(pathStr, "/") { + return &model.Object{ + Name: "Root", + Path: "/", + IsFolder: true, + }, nil + } + parent, name := path.Split(utils.FixAndCleanPath(pathStr)) + if parent == "" { + parent = "/" + } + objs, err := d.listDirObjects(ctx, parent, false) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name { + return obj, nil + } + } + return nil, errs.ObjectNotFound +} + +func (d *Chunker) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj := d.linkedObject(file) + if obj == nil || !obj.Chunked { + actualPath, err := d.getActualPathForRemote(file.GetPath()) + if err != nil { + return nil, fmt.Errorf("failed to convert path to remote path: %w", err) + } + link, _, err := op.Link(ctx, d.remoteStorage, actualPath, args) + return link, err + } + + linkedParts := make([]linkedPart, 0, len(obj.Parts)) + baseClosers := utils.EmptyClosers() + for _, part := range obj.Parts { + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID) + if err != nil { + return nil, fmt.Errorf("failed to convert chunk path: %w", err) + } + link, _, err := op.Link(ctx, d.remoteStorage, actualPath, args) + if err != nil { + return nil, err + } + if link.MFile != nil { + baseClosers.Add(link.MFile) + } + if link.RangeReadCloser != nil { + baseClosers.Add(link.RangeReadCloser) + } + linkedParts = append(linkedParts, linkedPart{ + part: part, + link: link, + }) + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return d.openChunkReader(ctx, linkedParts, obj.GetSize(), httpRange) + }, + Closers: baseClosers, + }, + }, nil +} + +func (d *Chunker) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + dstDirActualPath, err := d.getActualPathForRemote(parentDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.MakeDir(ctx, d.remoteStorage, path.Join(dstDirActualPath, dirName)) +} + +func (d *Chunker) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + obj := d.linkedObject(srcObj) + if srcObj.IsDir() || obj == nil || !obj.Chunked { + srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Move(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + } + + dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + for _, logicalPath := range d.chunkPathsForObject(obj) { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + return err + } + if err := op.Move(ctx, d.remoteStorage, actualPath, dstRemoteActualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + obj := d.linkedObject(srcObj) + if srcObj.IsDir() || obj == nil || !obj.Chunked { + remoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Rename(ctx, d.remoteStorage, remoteActualPath, newName) + } + + for _, part := range obj.Parts { + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID) + if err != nil { + return err + } + newChunkName := d.chunkPartBaseName(path.Join(path.Dir(obj.GetPath()), newName), part.No, part.XactID) + if err := op.Rename(ctx, d.remoteStorage, actualPath, newChunkName); err != nil { + return err + } + } + if obj.UsesMeta { + actualPath, err := d.getActualPathForRemote(obj.GetPath()) + if err != nil { + return err + } + if err := op.Rename(ctx, d.remoteStorage, actualPath, newName); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + obj := d.linkedObject(srcObj) + if srcObj.IsDir() || obj == nil || !obj.Chunked { + srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Copy(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + } + + dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + for _, logicalPath := range d.chunkPathsForObject(obj) { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + return err + } + if err := op.Copy(ctx, d.remoteStorage, actualPath, dstRemoteActualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Remove(ctx context.Context, obj model.Obj) error { + chunkedObj := d.linkedObject(obj) + if obj.IsDir() || chunkedObj == nil || !chunkedObj.Chunked { + remoteActualPath, err := d.getActualPathForRemote(obj.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Remove(ctx, d.remoteStorage, remoteActualPath) + } + + for _, logicalPath := range d.chunkPathsForObject(chunkedObj) { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + return err + } + if err := op.Remove(ctx, d.remoteStorage, actualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { + dstDirActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + + existing := d.linkedObject(streamer.GetExist()) + logicalPath := path.Join(dstDir.GetPath(), streamer.GetName()) + if streamer.GetSize() <= d.ChunkSize { + if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, streamer, up, false); err != nil { + return err + } + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(logicalPath)) + } + + if up == nil { + up = func(float64) {} + } + + var ( + md5Hasher hash.Hash + sha1Hasher hash.Hash + writers []io.Writer + ) + switch d.HashType { + case "md5": + md5Hasher = md5.New() + writers = append(writers, md5Hasher) + case "sha1": + sha1Hasher = sha1.New() + writers = append(writers, sha1Hasher) + } + writers = append(writers, driver.NewProgress(streamer.GetSize(), up)) + + baseReader := io.TeeReader(streamer, io.MultiWriter(writers...)) + xactID := strconv.FormatInt(time.Now().UnixNano(), 36) + if len(xactID) > 9 { + xactID = xactID[len(xactID)-9:] + } + if len(xactID) < 4 { + xactID = fmt.Sprintf("%04s", xactID) + } + + chunkCount := 0 + remaining := streamer.GetSize() + keepPaths := []string{logicalPath} + for remaining > 0 { + chunkLen := utils.Min(remaining, d.ChunkSize) + chunkName := d.chunkPartBaseName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) + chunkPath := d.makeChunkName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) + partReader := driver.NewLimitedUploadStream(ctx, &driver.ReaderWithCtx{ + Reader: io.LimitReader(baseReader, chunkLen), + Ctx: ctx, + }) + partStream := &stream.FileStream{ + Obj: &model.Object{ + Name: chunkName, + Size: chunkLen, + Modified: streamer.ModTime(), + Ctime: streamer.CreateTime(), + IsFolder: false, + }, + Reader: partReader, + Mimetype: "application/octet-stream", + WebPutAsTask: streamer.NeedStore(), + ForceStreamUpload: true, + } + if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, partStream, nil, false); err != nil { + return err + } + keepPaths = append(keepPaths, chunkPath) + remaining -= chunkLen + chunkCount++ + } + + if d.MetaFormat == "simplejson" { + md5Value := "" + if md5Hasher != nil { + md5Value = hex.EncodeToString(md5Hasher.Sum(nil)) + } + sha1Value := "" + if sha1Hasher != nil { + sha1Value = hex.EncodeToString(sha1Hasher.Sum(nil)) + } + txn := xactID + metaData, err := marshalMetadata(streamer.GetSize(), chunkCount, md5Value, sha1Value, txn) + if err != nil { + return err + } + metaStream := &stream.FileStream{ + Obj: &model.Object{ + Name: streamer.GetName(), + Size: int64(len(metaData)), + Modified: streamer.ModTime(), + Ctime: streamer.CreateTime(), + IsFolder: false, + }, + Reader: bytes.NewReader(metaData), + Mimetype: "application/json", + WebPutAsTask: false, + ForceStreamUpload: true, + } + if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, metaStream, nil, false); err != nil { + return err + } + } else { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err == nil { + _ = op.Remove(ctx, d.remoteStorage, actualPath) + } + } + + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(keepPaths...)) +} + +func xactIDIfNeeded(metaFormat, xactID string) string { + if metaFormat == "simplejson" { + return xactID + } + return "" +} + +var _ driver.Driver = (*Chunker)(nil) diff --git a/drivers/chunker/meta.go b/drivers/chunker/meta.go new file mode 100644 index 00000000000..fabfbc67545 --- /dev/null +++ b/drivers/chunker/meta.go @@ -0,0 +1,43 @@ +package chunker + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + defaultChunkSize int64 = 2147483648 + defaultChunkNameFmt = "*.rclone_chunk.###" + defaultMetaFormat = "simplejson" + defaultHashType = "md5" + defaultStartFrom = 1 +) + +type Addition struct { + RemotePath string `json:"remote_path" required:"true" help:"AList mounted folder path used to store chunked data, e.g. /my-storage/chunks"` + ChunkSize int64 `json:"chunk_size" type:"number" required:"true" default:"2147483648" help:"Files larger than this will be split into chunks"` + NameFormat string `json:"name_format" required:"true" default:"*.rclone_chunk.###" help:"Compatible with rclone chunker naming"` + StartFrom int `json:"start_from" type:"number" required:"true" default:"1" help:"Chunk number base, usually 0 or 1"` + MetaFormat string `json:"meta_format" type:"select" required:"true" options:"simplejson,none" default:"simplejson" help:"simplejson is compatible with rclone chunker metadata"` + HashType string `json:"hash_type" type:"select" required:"true" options:"none,md5,sha1" default:"md5" help:"Hash stored in metadata for chunked files"` +} + +var config = driver.Config{ + Name: "Chunker", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: true, + NoCache: true, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Chunker{} + }) +} diff --git a/drivers/chunker/types.go b/drivers/chunker/types.go new file mode 100644 index 00000000000..e89fa45c5cd --- /dev/null +++ b/drivers/chunker/types.go @@ -0,0 +1,72 @@ +package chunker + +import ( + "regexp" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) + +const ( + ctrlTypeRegStr = `[a-z][a-z0-9]{2,6}` + tempSuffixFormat = `_%04s` + tempSuffixRegStr = `_([0-9a-z]{4,9})` + tempSuffixRegOld = `\.\.tmp_([0-9]{10,13})` + maxMetadataSizeRead = 1023 + maxMetadataSizeWrite = 255 + maxSafeChunkNumber = 10000000 + chunkerMetadataVerion = 2 +) + +var ctrlTypeRegexp = regexp.MustCompile(`^` + ctrlTypeRegStr + `$`) + +type Chunker struct { + model.Storage + Addition + remoteStorage driver.Driver + dataNameFmt string + nameRegexp *regexp.Regexp +} + +type metadataJSON struct { + Version *int `json:"ver"` + Size *int64 `json:"size"` + ChunkNum *int `json:"nchunks"` + MD5 string `json:"md5,omitempty"` + SHA1 string `json:"sha1,omitempty"` + XactID string `json:"txn,omitempty"` +} + +type chunkMetadata struct { + Version int + Size int64 + NChunks int + MD5 string + SHA1 string + XactID string +} + +type chunkPart struct { + No int + Size int64 + XactID string +} + +type groupInfo struct { + base model.Obj + partsByXact map[string]map[int]chunkPart +} + +type Object struct { + model.Object + Main model.Obj + Parts []chunkPart + Meta *chunkMetadata + Chunked bool + UsesMeta bool +} + +type linkedPart struct { + part chunkPart + link *model.Link +} diff --git a/drivers/chunker/util.go b/drivers/chunker/util.go new file mode 100644 index 00000000000..5711b2126f9 --- /dev/null +++ b/drivers/chunker/util.go @@ -0,0 +1,572 @@ +package chunker + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Chunker) validateOptions() error { + if d.RemotePath == "" { + return errors.New("remote_path is required") + } + if d.ChunkSize <= 0 { + return errors.New("chunk_size must be positive") + } + switch d.MetaFormat { + case "simplejson", "none": + default: + return fmt.Errorf("unsupported meta_format: %s", d.MetaFormat) + } + switch d.HashType { + case "none", "md5", "sha1": + default: + return fmt.Errorf("unsupported hash_type: %s", d.HashType) + } + if d.MetaFormat == "none" && d.HashType != "none" { + return fmt.Errorf("hash_type %q requires meta_format=simplejson", d.HashType) + } + return nil +} + +func (d *Chunker) setChunkNameFormat(pattern string) error { + if strings.Count(pattern, "*") != 1 { + return errors.New("pattern must have exactly one asterisk (*)") + } + hashCount := strings.Count(pattern, "#") + if hashCount < 1 { + return errors.New("pattern must contain a hash character (#)") + } + if strings.Index(pattern, "*") > strings.Index(pattern, "#") { + return errors.New("asterisk (*) in pattern must come before hashes (#)") + } + if ok, _ := regexp.MatchString("^[^#]*[#]+[^#]*$", pattern); !ok { + return errors.New("hashes (#) in pattern must be consecutive") + } + if dir, _ := path.Split(pattern); dir != "" { + return errors.New("directory separator prohibited") + } + if pattern[0] != '*' { + return errors.New("pattern must start with asterisk") + } + + reHashes := regexp.MustCompile("[#]+") + reDigits := "[0-9]+" + if hashCount > 1 { + reDigits = fmt.Sprintf("[0-9]{%d,}", hashCount) + } + reDataOrCtrl := fmt.Sprintf("(?:(%s)|_(%s))", reDigits, ctrlTypeRegStr) + + strRegex := regexp.QuoteMeta(pattern) + strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl) + strRegex = strings.Replace(strRegex, "\\*", "(.+?)", 1) + strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld) + d.nameRegexp = regexp.MustCompile(strRegex) + + fmtDigits := "%d" + if hashCount > 1 { + fmtDigits = fmt.Sprintf("%%0%dd", hashCount) + } + strFmt := strings.ReplaceAll(pattern, "%", "%%") + strFmt = strings.Replace(strFmt, "*", "%s", 1) + d.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits) + return nil +} + +func (d *Chunker) makeChunkName(filePath string, chunkNo int, xactID string) string { + dir, baseName := path.Split(filePath) + name := fmt.Sprintf(d.dataNameFmt, baseName, chunkNo+d.StartFrom) + if xactID != "" { + name += fmt.Sprintf(tempSuffixFormat, xactID) + } + return dir + name +} + +func (d *Chunker) parseChunkName(filePath string) (parentPath string, chunkNo int, ctrlType, xactID string) { + dir, name := path.Split(filePath) + match := d.nameRegexp.FindStringSubmatch(name) + if match == nil || match[1] == "" { + return "", -1, "", "" + } + + chunkNo = -1 + if match[2] != "" { + n, err := strconv.Atoi(match[2]) + if err != nil { + return "", -1, "", "" + } + chunkNo = n - d.StartFrom + if chunkNo < 0 { + return "", -1, "", "" + } + } + + if match[4] != "" { + xactID = match[4] + } + if match[5] != "" { + oldNum, err := strconv.ParseInt(match[5], 10, 64) + if err != nil || oldNum < 0 { + return "", -1, "", "" + } + xactID = fmt.Sprintf(tempSuffixFormat, strconv.FormatInt(oldNum, 36))[1:] + } + + return dir + match[1], chunkNo, match[3], xactID +} + +func marshalMetadata(size int64, nChunks int, md5Value, sha1Value, xactID string) ([]byte, error) { + version := chunkerMetadataVerion + if xactID == "" && version == 2 { + version = 1 + } + meta := metadataJSON{ + Version: &version, + Size: &size, + ChunkNum: &nChunks, + MD5: md5Value, + SHA1: sha1Value, + XactID: xactID, + } + data, err := json.Marshal(&meta) + if err == nil && len(data) >= maxMetadataSizeWrite { + return nil, errors.New("metadata can't be this big") + } + return data, err +} + +func unmarshalMetadata(data []byte) (*chunkMetadata, error) { + if len(data) > maxMetadataSizeWrite { + return nil, errors.New("metadata is too large") + } + if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' { + return nil, errors.New("invalid json") + } + var meta metadataJSON + if err := json.Unmarshal(data, &meta); err != nil { + return nil, err + } + if meta.Version == nil || meta.Size == nil || meta.ChunkNum == nil { + return nil, errors.New("missing required field") + } + if *meta.Version < 1 { + return nil, errors.New("wrong version") + } + if *meta.Size < 0 { + return nil, errors.New("negative file size") + } + if *meta.ChunkNum < 1 || *meta.ChunkNum > maxSafeChunkNumber { + return nil, errors.New("wrong number of chunks") + } + if meta.MD5 != "" { + if _, err := hex.DecodeString(meta.MD5); err != nil || len(meta.MD5) != 32 { + return nil, errors.New("wrong md5 hash") + } + } + if meta.SHA1 != "" { + if _, err := hex.DecodeString(meta.SHA1); err != nil || len(meta.SHA1) != 40 { + return nil, errors.New("wrong sha1 hash") + } + } + if *meta.Version > chunkerMetadataVerion { + return nil, errors.New("unknown metadata version") + } + return &chunkMetadata{ + Version: *meta.Version, + Size: *meta.Size, + NChunks: *meta.ChunkNum, + MD5: meta.MD5, + SHA1: meta.SHA1, + XactID: meta.XactID, + }, nil +} + +func (d *Chunker) joinRemotePath(logicalPath string) string { + logicalPath = utils.FixAndCleanPath(logicalPath) + if utils.PathEqual(logicalPath, "/") { + return d.RemotePath + } + return path.Join(d.RemotePath, logicalPath) +} + +func (d *Chunker) getActualPathForRemote(logicalPath string) (string, error) { + _, actualPath, err := op.GetStorageAndActualPath(d.joinRemotePath(logicalPath)) + return actualPath, err +} + +func (d *Chunker) getActualChunkPath(filePath string, chunkNo int, xactID string) (string, error) { + return d.getActualPathForRemote(d.makeChunkName(filePath, chunkNo, xactID)) +} + +func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bool) ([]model.Obj, error) { + remotePath := d.joinRemotePath(dirPath) + entries, err := fsList(ctx, remotePath, refresh) + if err != nil { + return nil, err + } + + groups := map[string]*groupInfo{} + var dirs []model.Obj + + for _, entry := range entries { + if entry.IsDir() { + dirs = append(dirs, &model.Object{ + Name: entry.GetName(), + Path: path.Join(dirPath, entry.GetName()), + Size: 0, + Modified: entry.ModTime(), + Ctime: entry.CreateTime(), + IsFolder: true, + HashInfo: entry.GetHash(), + }) + continue + } + + mainName, chunkNo, ctrlType, xactID := d.parseChunkName(entry.GetName()) + if mainName == "" { + g := groups[entry.GetName()] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[entry.GetName()] = g + } + g.base = entry + continue + } + if chunkNo < 0 || ctrlType != "" { + continue + } + g := groups[mainName] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[mainName] = g + } + if g.partsByXact[xactID] == nil { + g.partsByXact[xactID] = map[int]chunkPart{} + } + g.partsByXact[xactID][chunkNo] = chunkPart{ + No: chunkNo, + Size: entry.GetSize(), + XactID: xactID, + } + } + + result := make([]model.Obj, 0, len(dirs)+len(groups)) + result = append(result, dirs...) + for name, group := range groups { + obj, ok, err := d.buildListedObject(ctx, dirPath, name, group) + if err != nil { + return nil, err + } + if ok { + result = append(result, obj) + } + } + return result, nil +} + +func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, group *groupInfo) (model.Obj, bool, error) { + var meta *chunkMetadata + var err error + if group.base != nil && group.base.GetSize() <= maxMetadataSizeRead && len(group.partsByXact) > 0 { + meta, err = d.readMetadata(ctx, path.Join(dirPath, name), group.base.GetSize()) + if err != nil { + meta = nil + } + } + + if meta == nil && group.base != nil && len(group.partsByXact) == 0 { + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: group.base.GetSize(), + Modified: group.base.ModTime(), + Ctime: group.base.CreateTime(), + IsFolder: false, + HashInfo: group.base.GetHash(), + }, + Main: group.base, + }, true, nil + } + + selected := map[int]chunkPart{} + switch { + case meta != nil: + selected = group.partsByXact[meta.XactID] + case group.base == nil: + selected = group.partsByXact[""] + default: + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: group.base.GetSize(), + Modified: group.base.ModTime(), + Ctime: group.base.CreateTime(), + IsFolder: false, + HashInfo: group.base.GetHash(), + }, + Main: group.base, + }, true, nil + } + + parts := sortChunkParts(selected) + if len(parts) == 0 { + if meta != nil { + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: meta.Size, + Modified: group.base.ModTime(), + Ctime: group.base.CreateTime(), + IsFolder: false, + HashInfo: buildHashInfo(meta), + }, + Main: group.base, + Meta: meta, + Chunked: true, + UsesMeta: true, + }, true, nil + } + return nil, false, nil + } + + size := int64(0) + for _, part := range parts { + size += part.Size + } + modified := time.Time{} + ctime := time.Time{} + if group.base != nil { + modified = group.base.ModTime() + ctime = group.base.CreateTime() + } + if meta != nil { + size = meta.Size + } + + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: size, + Modified: modified, + Ctime: ctime, + IsFolder: false, + HashInfo: buildHashInfo(meta), + }, + Main: group.base, + Parts: parts, + Meta: meta, + Chunked: true, + UsesMeta: meta != nil, + }, true, nil +} + +func (d *Chunker) readMetadata(ctx context.Context, logicalPath string, size int64) (*chunkMetadata, error) { + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + return nil, err + } + link, obj, err := op.Link(ctx, d.remoteStorage, actualPath, model.LinkArgs{}) + if err != nil { + return nil, err + } + ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, link) + if err != nil { + return nil, err + } + defer ss.Close() + + reader, err := ss.RangeRead(http_range.Range{Start: 0, Length: size}) + if err != nil { + return nil, err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return unmarshalMetadata(data) +} + +func sortChunkParts(parts map[int]chunkPart) []chunkPart { + result := make([]chunkPart, 0, len(parts)) + for _, part := range parts { + result = append(result, part) + } + sort.Slice(result, func(i, j int) bool { + return result[i].No < result[j].No + }) + return result +} + +func buildHashInfo(meta *chunkMetadata) utils.HashInfo { + if meta == nil { + return utils.HashInfo{} + } + hashes := map[*utils.HashType]string{} + if meta.MD5 != "" { + hashes[utils.MD5] = meta.MD5 + } + if meta.SHA1 != "" { + hashes[utils.SHA1] = meta.SHA1 + } + return utils.NewHashInfoByMap(hashes) +} + +func (d *Chunker) linkedObject(obj model.Obj) *Object { + if linked, ok := obj.(*Object); ok { + return linked + } + return nil +} + +func (d *Chunker) chunkPathsForObject(obj *Object) []string { + if obj == nil { + return nil + } + paths := make([]string, 0, len(obj.Parts)+1) + if obj.Chunked && obj.UsesMeta { + paths = append(paths, obj.GetPath()) + } + if !obj.Chunked { + paths = append(paths, obj.GetPath()) + return paths + } + for _, part := range obj.Parts { + paths = append(paths, d.makeChunkName(obj.GetPath(), part.No, part.XactID)) + } + return paths +} + +func (d *Chunker) cleanupReplacedObject(ctx context.Context, obj *Object, keep map[string]struct{}) error { + if obj == nil { + return nil + } + var errs []error + for _, logicalPath := range d.chunkPathsForObject(obj) { + if _, ok := keep[logicalPath]; ok { + continue + } + actualPath, err := d.getActualPathForRemote(logicalPath) + if err != nil { + errs = append(errs, err) + continue + } + if err := op.Remove(ctx, d.remoteStorage, actualPath); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (d *Chunker) buildKeepSet(paths ...string) map[string]struct{} { + keep := make(map[string]struct{}, len(paths)) + for _, p := range paths { + if p == "" { + continue + } + keep[utils.FixAndCleanPath(p)] = struct{}{} + } + return keep +} + +func (d *Chunker) chunkPartBaseName(filePath string, chunkNo int, xactID string) string { + return path.Base(d.makeChunkName(filePath, chunkNo, xactID)) +} + +func (d *Chunker) openChunkReader(ctx context.Context, parts []linkedPart, totalSize int64, req http_range.Range) (io.ReadCloser, error) { + if req.Start < 0 || req.Start > totalSize { + return nil, fmt.Errorf("range start out of bound") + } + if req.Length < 0 || req.Start+req.Length > totalSize { + req.Length = totalSize - req.Start + } + if req.Length == 0 { + return io.NopCloser(strings.NewReader("")), nil + } + + var ( + readers []io.Reader + closers = utils.EmptyClosers() + offset int64 + remaining = req.Length + ) + for _, part := range parts { + partStart := offset + partEnd := offset + part.part.Size + offset = partEnd + if req.Start >= partEnd || remaining <= 0 { + continue + } + localStart := int64(0) + if req.Start > partStart { + localStart = req.Start - partStart + } + localLength := utils.Min(part.part.Size-localStart, remaining) + rc, err := d.openPartRange(ctx, part.link, part.part.Size, localStart, localLength) + if err != nil { + _ = closers.Close() + return nil, err + } + readers = append(readers, rc) + closers.Add(rc) + remaining -= localLength + } + if remaining > 0 { + _ = closers.Close() + return nil, errors.New("missing chunk data") + } + return utils.NewReadCloser(io.MultiReader(readers...), func() error { + return closers.Close() + }), nil +} + +func (d *Chunker) openPartRange(ctx context.Context, link *model.Link, size, offset, length int64) (io.ReadCloser, error) { + httpRange := http_range.Range{Start: offset, Length: length} + switch { + case link.MFile != nil: + return io.NopCloser(io.NewSectionReader(link.MFile, offset, length)), nil + case link.RangeReadCloser != nil: + return link.RangeReadCloser.RangeRead(ctx, httpRange) + case link.URL != "": + rrc, err := stream.GetRangeReadCloserFromLink(size, link) + if err != nil { + return nil, err + } + rc, err := rrc.RangeRead(ctx, httpRange) + if err != nil { + _ = rrc.Close() + return nil, err + } + return utils.NewReadCloser(rc, func() error { + return rrc.Close() + }), nil + default: + return nil, errors.New("chunk part has no readable link") + } +} + +func fsList(ctx context.Context, remotePath string, refresh bool) ([]model.Obj, error) { + return fs.List(ctx, remotePath, &fs.ListArgs{NoLog: true, Refresh: refresh}) +} diff --git a/drivers/chunker/util_test.go b/drivers/chunker/util_test.go new file mode 100644 index 00000000000..96f06051b75 --- /dev/null +++ b/drivers/chunker/util_test.go @@ -0,0 +1,115 @@ +package chunker + +import ( + "context" + "testing" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +func newTestChunker(t *testing.T) *Chunker { + t.Helper() + d := &Chunker{ + Addition: Addition{ + NameFormat: defaultChunkNameFmt, + StartFrom: defaultStartFrom, + }, + } + if err := d.setChunkNameFormat(d.NameFormat); err != nil { + t.Fatalf("setChunkNameFormat: %v", err) + } + return d +} + +func TestParseChunkName(t *testing.T) { + d := newTestChunker(t) + + mainName, chunkNo, ctrlType, xactID := d.parseChunkName("movie.mkv.rclone_chunk.001") + if mainName != "movie.mkv" || chunkNo != 0 || ctrlType != "" || xactID != "" { + t.Fatalf("unexpected parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) + } + + mainName, chunkNo, ctrlType, xactID = d.parseChunkName("movie.mkv.rclone_chunk.003_abcd") + if mainName != "movie.mkv" || chunkNo != 2 || ctrlType != "" || xactID != "abcd" { + t.Fatalf("unexpected temp parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) + } + + mainName, chunkNo, ctrlType, xactID = d.parseChunkName("movie.mkv.rclone_chunk._meta") + if mainName != "movie.mkv" || chunkNo != -1 || ctrlType != "meta" || xactID != "" { + t.Fatalf("unexpected control parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) + } +} + +func TestMarshalAndUnmarshalMetadata(t *testing.T) { + data, err := marshalMetadata(123, 2, "5d41402abc4b2a76b9719d911017c592", "", "") + if err != nil { + t.Fatalf("marshalMetadata: %v", err) + } + meta, err := unmarshalMetadata(data) + if err != nil { + t.Fatalf("unmarshalMetadata: %v", err) + } + if meta.Version != 1 || meta.Size != 123 || meta.NChunks != 2 || meta.MD5 == "" || meta.XactID != "" { + t.Fatalf("unexpected metadata: %+v", meta) + } + + data, err = marshalMetadata(456, 3, "", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "txn1") + if err != nil { + t.Fatalf("marshalMetadata with txn: %v", err) + } + meta, err = unmarshalMetadata(data) + if err != nil { + t.Fatalf("unmarshalMetadata with txn: %v", err) + } + if meta.Version != 2 || meta.Size != 456 || meta.NChunks != 3 || meta.SHA1 == "" || meta.XactID != "txn1" { + t.Fatalf("unexpected txn metadata: %+v", meta) + } +} + +func TestBuildListedObjectWithoutMetadata(t *testing.T) { + d := newTestChunker(t) + now := time.Now() + + obj, ok, err := d.buildListedObject(context.Background(), "/", "archive.bin", &groupInfo{ + partsByXact: map[string]map[int]chunkPart{ + "": { + 0: {No: 0, Size: 5}, + 1: {No: 1, Size: 7}, + }, + }, + }) + if err != nil { + t.Fatalf("buildListedObject: %v", err) + } + if !ok { + t.Fatal("expected grouped object") + } + grouped, ok := obj.(*Object) + if !ok { + t.Fatalf("expected *Object, got %T", obj) + } + if !grouped.Chunked || grouped.GetSize() != 12 || len(grouped.Parts) != 2 { + t.Fatalf("unexpected grouped object: %+v", grouped) + } + + raw, ok, err := d.buildListedObject(context.Background(), "/", "raw.txt", &groupInfo{ + base: &model.Object{ + Name: "raw.txt", + Size: 9, + Modified: now, + Ctime: now, + }, + partsByXact: map[string]map[int]chunkPart{}, + }) + if err != nil { + t.Fatalf("build raw listed object: %v", err) + } + if !ok { + t.Fatal("expected raw object") + } + rawObj := raw.(*Object) + if rawObj.Chunked || rawObj.GetSize() != 9 { + t.Fatalf("unexpected raw object: %+v", rawObj) + } +} diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index 180773a3f48..0878f63869f 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -13,16 +13,16 @@ type Addition struct { FileNameEnc string `json:"filename_encryption" type:"select" required:"true" options:"off,standard,obfuscate" default:"off"` DirNameEnc string `json:"directory_name_encryption" type:"select" required:"true" options:"false,true" default:"false"` - RemotePath string `json:"remote_path" required:"true" help:"This is where the encrypted data stores"` + RemotePath string `json:"remote_path" required:"true" help:"AList mounted folder path used to store encrypted data, e.g. /my-storage/secret"` Password string `json:"password" required:"true" confidential:"true" help:"the main password"` Salt string `json:"salt" confidential:"true" help:"If you don't know what is salt, treat it as a second password. Optional but recommended"` EncryptedSuffix string `json:"encrypted_suffix" required:"true" default:".bin" help:"for advanced user only! encrypted files will have this suffix"` FileNameEncoding string `json:"filename_encoding" type:"select" required:"true" options:"base64,base32,base32768" default:"base64" help:"for advanced user only!"` - Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` - ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` + ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ From ce6a192d62a9c2d2b95d4eba5985aef9415f0d75 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 26 Mar 2026 15:39:56 +0800 Subject: [PATCH 611/659] feat(chunker): support multi-target chunk storage --- drivers/chunker/driver.go | 196 ++++++++---- drivers/chunker/meta.go | 16 +- drivers/chunker/types.go | 37 ++- drivers/chunker/util.go | 570 +++++++++++++++++++++++++++-------- drivers/chunker/util_test.go | 106 ++++++- 5 files changed, 726 insertions(+), 199 deletions(-) diff --git a/drivers/chunker/driver.go b/drivers/chunker/driver.go index 004fde1c476..f05d4a9f89a 100644 --- a/drivers/chunker/driver.go +++ b/drivers/chunker/driver.go @@ -49,16 +49,28 @@ func (d *Chunker) Init(ctx context.Context) error { return err } - storage, err := fs.GetStorage(d.RemotePath, &fs.GetStoragesArgs{}) - if err != nil { - return fmt.Errorf("can't find remote storage: %w", err) + targetPaths := d.configuredRemotePaths() + d.remoteTargets = make([]remoteTarget, 0, len(targetPaths)) + for _, targetPath := range targetPaths { + storage, err := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if err != nil { + return fmt.Errorf("can't find remote storage %q: %w", targetPath, err) + } + d.remoteTargets = append(d.remoteTargets, remoteTarget{ + MountPath: targetPath, + Storage: storage, + }) + } + if len(d.remoteTargets) == 0 { + return fmt.Errorf("can't find remote storage: %w", errs.ObjectNotFound) } - d.remoteStorage = storage + d.remoteStorage = d.remoteTargets[0].Storage return nil } func (d *Chunker) Drop(ctx context.Context) error { d.remoteStorage = nil + d.remoteTargets = nil return nil } @@ -93,22 +105,26 @@ func (d *Chunker) Get(ctx context.Context, pathStr string) (model.Obj, error) { func (d *Chunker) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { obj := d.linkedObject(file) if obj == nil || !obj.Chunked { - actualPath, err := d.getActualPathForRemote(file.GetPath()) + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + actualPath, err := d.getActualPathForRemoteOnTarget(file.GetPath(), remoteIndex) if err != nil { return nil, fmt.Errorf("failed to convert path to remote path: %w", err) } - link, _, err := op.Link(ctx, d.remoteStorage, actualPath, args) + link, _, err := op.Link(ctx, d.remoteTargets[remoteIndex].Storage, actualPath, args) return link, err } linkedParts := make([]linkedPart, 0, len(obj.Parts)) baseClosers := utils.EmptyClosers() for _, part := range obj.Parts { - actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID) + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID, part.RemoteIndex) if err != nil { return nil, fmt.Errorf("failed to convert chunk path: %w", err) } - link, _, err := op.Link(ctx, d.remoteStorage, actualPath, args) + link, _, err := op.Link(ctx, d.remoteTargets[part.RemoteIndex].Storage, actualPath, args) if err != nil { return nil, err } @@ -135,37 +151,50 @@ func (d *Chunker) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Chunker) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - dstDirActualPath, err := d.getActualPathForRemote(parentDir.GetPath()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - return op.MakeDir(ctx, d.remoteStorage, path.Join(dstDirActualPath, dirName)) + return d.ensureDirOnAllTargets(ctx, path.Join(parentDir.GetPath(), dirName)) } func (d *Chunker) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj.IsDir() { + return d.moveDirAcrossTargets(ctx, srcObj.GetPath(), dstDir.GetPath()) + } obj := d.linkedObject(srcObj) - if srcObj.IsDir() || obj == nil || !obj.Chunked { - srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDir.GetPath()); err != nil { + return err + } + srcRemoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - return op.Move(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + return op.Move(ctx, d.remoteTargets[remoteIndex].Storage, srcRemoteActualPath, dstRemoteActualPath) } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - for _, logicalPath := range d.chunkPathsForObject(obj) { - actualPath, err := d.getActualPathForRemote(logicalPath) + ensuredTargets := map[int]struct{}{} + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := ensuredTargets[location.RemoteIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, location.RemoteIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[location.RemoteIndex] = struct{}{} + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) if err != nil { return err } - if err := op.Move(ctx, d.remoteStorage, actualPath, dstRemoteActualPath); err != nil { + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), location.RemoteIndex) + if err != nil { + return err + } + if err := op.Move(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath, dstRemoteActualPath); err != nil { return err } } @@ -173,31 +202,38 @@ func (d *Chunker) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Chunker) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if srcObj.IsDir() { + return d.renameDirAcrossTargets(ctx, srcObj.GetPath(), newName) + } obj := d.linkedObject(srcObj) - if srcObj.IsDir() || obj == nil || !obj.Chunked { - remoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + remoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - return op.Rename(ctx, d.remoteStorage, remoteActualPath, newName) + return op.Rename(ctx, d.remoteTargets[remoteIndex].Storage, remoteActualPath, newName) } for _, part := range obj.Parts { - actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID) + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID, part.RemoteIndex) if err != nil { return err } newChunkName := d.chunkPartBaseName(path.Join(path.Dir(obj.GetPath()), newName), part.No, part.XactID) - if err := op.Rename(ctx, d.remoteStorage, actualPath, newChunkName); err != nil { + if err := op.Rename(ctx, d.remoteTargets[part.RemoteIndex].Storage, actualPath, newChunkName); err != nil { return err } } if obj.UsesMeta { - actualPath, err := d.getActualPathForRemote(obj.GetPath()) + actualPath, err := d.getActualPathForRemoteOnTarget(obj.GetPath(), obj.MainRemoteIndex) if err != nil { return err } - if err := op.Rename(ctx, d.remoteStorage, actualPath, newName); err != nil { + if err := op.Rename(ctx, d.remoteTargets[obj.MainRemoteIndex].Storage, actualPath, newName); err != nil { return err } } @@ -205,29 +241,46 @@ func (d *Chunker) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *Chunker) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj.IsDir() { + return d.copyDirAcrossTargets(ctx, srcObj.GetPath(), dstDir.GetPath()) + } obj := d.linkedObject(srcObj) - if srcObj.IsDir() || obj == nil || !obj.Chunked { - srcRemoteActualPath, err := d.getActualPathForRemote(srcObj.GetPath()) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDir.GetPath()); err != nil { + return err + } + srcRemoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - return op.Copy(ctx, d.remoteStorage, srcRemoteActualPath, dstRemoteActualPath) + return op.Copy(ctx, d.remoteTargets[remoteIndex].Storage, srcRemoteActualPath, dstRemoteActualPath) } - dstRemoteActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) - if err != nil { - return fmt.Errorf("failed to convert path to remote path: %w", err) - } - for _, logicalPath := range d.chunkPathsForObject(obj) { - actualPath, err := d.getActualPathForRemote(logicalPath) + ensuredTargets := map[int]struct{}{} + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := ensuredTargets[location.RemoteIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, location.RemoteIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[location.RemoteIndex] = struct{}{} + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) if err != nil { return err } - if err := op.Copy(ctx, d.remoteStorage, actualPath, dstRemoteActualPath); err != nil { + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), location.RemoteIndex) + if err != nil { + return err + } + if err := op.Copy(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath, dstRemoteActualPath); err != nil { return err } } @@ -235,21 +288,28 @@ func (d *Chunker) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *Chunker) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + return d.removeDirAcrossTargets(ctx, obj.GetPath()) + } chunkedObj := d.linkedObject(obj) - if obj.IsDir() || chunkedObj == nil || !chunkedObj.Chunked { - remoteActualPath, err := d.getActualPathForRemote(obj.GetPath()) + if chunkedObj == nil || !chunkedObj.Chunked { + remoteIndex := 0 + if chunkedObj != nil { + remoteIndex = chunkedObj.MainRemoteIndex + } + remoteActualPath, err := d.getActualPathForRemoteOnTarget(obj.GetPath(), remoteIndex) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } - return op.Remove(ctx, d.remoteStorage, remoteActualPath) + return op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, remoteActualPath) } - for _, logicalPath := range d.chunkPathsForObject(chunkedObj) { - actualPath, err := d.getActualPathForRemote(logicalPath) + for _, location := range d.objectLocationsForObject(chunkedObj) { + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) if err != nil { return err } - if err := op.Remove(ctx, d.remoteStorage, actualPath); err != nil { + if err := op.Remove(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath); err != nil { return err } } @@ -257,18 +317,21 @@ func (d *Chunker) Remove(ctx context.Context, obj model.Obj) error { } func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { - dstDirActualPath, err := d.getActualPathForRemote(dstDir.GetPath()) + primaryDirActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), 0) if err != nil { return fmt.Errorf("failed to convert path to remote path: %w", err) } + if err := d.ensureDirOnTarget(ctx, 0, dstDir.GetPath()); err != nil { + return err + } existing := d.linkedObject(streamer.GetExist()) logicalPath := path.Join(dstDir.GetPath(), streamer.GetName()) if streamer.GetSize() <= d.ChunkSize { - if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, streamer, up, false); err != nil { + if err := op.Put(ctx, d.remoteTargets[0].Storage, primaryDirActualPath, streamer, up, false); err != nil { return err } - return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(logicalPath)) + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(d.targetLocation(logicalPath, 0))) } if up == nil { @@ -301,9 +364,21 @@ func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.File chunkCount := 0 remaining := streamer.GetSize() - keepPaths := []string{logicalPath} + keepLocations := make([]objectLocation, 0, len(d.remoteTargets)+1) + ensuredTargets := map[int]struct{}{0: {}} for remaining > 0 { chunkLen := utils.Min(remaining, d.ChunkSize) + targetIndex := d.chunkTargetIndex(chunkCount) + if _, ok := ensuredTargets[targetIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, targetIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[targetIndex] = struct{}{} + } + dstDirActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), targetIndex) + if err != nil { + return err + } chunkName := d.chunkPartBaseName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) chunkPath := d.makeChunkName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) partReader := driver.NewLimitedUploadStream(ctx, &driver.ReaderWithCtx{ @@ -323,10 +398,10 @@ func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.File WebPutAsTask: streamer.NeedStore(), ForceStreamUpload: true, } - if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, partStream, nil, false); err != nil { + if err := op.Put(ctx, d.remoteTargets[targetIndex].Storage, dstDirActualPath, partStream, nil, false); err != nil { return err } - keepPaths = append(keepPaths, chunkPath) + keepLocations = append(keepLocations, d.targetLocation(chunkPath, targetIndex)) remaining -= chunkLen chunkCount++ } @@ -358,17 +433,20 @@ func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.File WebPutAsTask: false, ForceStreamUpload: true, } - if err := op.Put(ctx, d.remoteStorage, dstDirActualPath, metaStream, nil, false); err != nil { + if err := op.Put(ctx, d.remoteTargets[0].Storage, primaryDirActualPath, metaStream, nil, false); err != nil { return err } + keepLocations = append(keepLocations, d.targetLocation(logicalPath, 0)) } else { - actualPath, err := d.getActualPathForRemote(logicalPath) - if err == nil { - _ = op.Remove(ctx, d.remoteStorage, actualPath) + for remoteIndex := range d.remoteTargets { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err == nil { + _ = op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, actualPath) + } } } - return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(keepPaths...)) + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(keepLocations...)) } func xactIDIfNeeded(metaFormat, xactID string) string { diff --git a/drivers/chunker/meta.go b/drivers/chunker/meta.go index fabfbc67545..27265fe3033 100644 --- a/drivers/chunker/meta.go +++ b/drivers/chunker/meta.go @@ -7,19 +7,21 @@ import ( const ( defaultChunkSize int64 = 2147483648 - defaultChunkNameFmt = "*.rclone_chunk.###" + defaultChunkNameFmt = "{name}.rclone_chunk.{chunk:3}" defaultMetaFormat = "simplejson" defaultHashType = "md5" defaultStartFrom = 1 ) type Addition struct { - RemotePath string `json:"remote_path" required:"true" help:"AList mounted folder path used to store chunked data, e.g. /my-storage/chunks"` - ChunkSize int64 `json:"chunk_size" type:"number" required:"true" default:"2147483648" help:"Files larger than this will be split into chunks"` - NameFormat string `json:"name_format" required:"true" default:"*.rclone_chunk.###" help:"Compatible with rclone chunker naming"` - StartFrom int `json:"start_from" type:"number" required:"true" default:"1" help:"Chunk number base, usually 0 or 1"` - MetaFormat string `json:"meta_format" type:"select" required:"true" options:"simplejson,none" default:"simplejson" help:"simplejson is compatible with rclone chunker metadata"` - HashType string `json:"hash_type" type:"select" required:"true" options:"none,md5,sha1" default:"md5" help:"Hash stored in metadata for chunked files"` + RemotePath string `json:"remote_path" required:"true" help:"Primary AList mounted folder path used to store metadata and small files, e.g. /my-storage/chunks"` + RemotePaths string `json:"remote_paths" type:"text" help:"Additional AList mounted folder paths, one per line. Chunk files will be distributed across remote_path and these extra paths."` + StoreChunksInPrimary bool `json:"store_chunks_in_primary" type:"bool" default:"true" help:"When extra remote paths are configured, also store chunk files in remote_path"` + ChunkSize int64 `json:"chunk_size" type:"number" required:"true" default:"2147483648" help:"Files larger than this will be split into chunks"` + NameFormat string `json:"name_format" required:"true" default:"{name}.rclone_chunk.{chunk:3}" help:"Magic tokens: {name}, {chunk}, {chunk:N}. Name token must appear before chunk token."` + StartFrom int `json:"start_from" type:"number" required:"true" default:"1" help:"Chunk number base, usually 0 or 1"` + MetaFormat string `json:"meta_format" type:"select" required:"true" options:"simplejson,none" default:"simplejson" help:"simplejson is compatible with rclone chunker metadata"` + HashType string `json:"hash_type" type:"select" required:"true" options:"none,md5,sha1" default:"md5" help:"Hash stored in metadata for chunked files"` } var config = driver.Config{ diff --git a/drivers/chunker/types.go b/drivers/chunker/types.go index e89fa45c5cd..ff0f6fc4d85 100644 --- a/drivers/chunker/types.go +++ b/drivers/chunker/types.go @@ -19,15 +19,32 @@ const ( ) var ctrlTypeRegexp = regexp.MustCompile(`^` + ctrlTypeRegStr + `$`) +var chunkTokenRegexp = regexp.MustCompile(`\{chunk(?::([0-9]+))?\}`) type Chunker struct { model.Storage Addition remoteStorage driver.Driver + remoteTargets []remoteTarget dataNameFmt string nameRegexp *regexp.Regexp } +type remoteTarget struct { + MountPath string + Storage driver.Driver +} + +type locatedObj struct { + Obj model.Obj + RemoteIndex int +} + +type objectLocation struct { + LogicalPath string + RemoteIndex int +} + type metadataJSON struct { Version *int `json:"ver"` Size *int64 `json:"size"` @@ -47,23 +64,25 @@ type chunkMetadata struct { } type chunkPart struct { - No int - Size int64 - XactID string + No int + Size int64 + XactID string + RemoteIndex int } type groupInfo struct { - base model.Obj + base *locatedObj partsByXact map[string]map[int]chunkPart } type Object struct { model.Object - Main model.Obj - Parts []chunkPart - Meta *chunkMetadata - Chunked bool - UsesMeta bool + Main model.Obj + MainRemoteIndex int + Parts []chunkPart + Meta *chunkMetadata + Chunked bool + UsesMeta bool } type linkedPart struct { diff --git a/drivers/chunker/util.go b/drivers/chunker/util.go index 5711b2126f9..ebb740e8194 100644 --- a/drivers/chunker/util.go +++ b/drivers/chunker/util.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" @@ -23,7 +24,7 @@ import ( ) func (d *Chunker) validateOptions() error { - if d.RemotePath == "" { + if strings.TrimSpace(d.RemotePath) == "" { return errors.New("remote_path is required") } if d.ChunkSize <= 0 { @@ -45,50 +46,113 @@ func (d *Chunker) validateOptions() error { return nil } -func (d *Chunker) setChunkNameFormat(pattern string) error { - if strings.Count(pattern, "*") != 1 { - return errors.New("pattern must have exactly one asterisk (*)") - } - hashCount := strings.Count(pattern, "#") - if hashCount < 1 { - return errors.New("pattern must contain a hash character (#)") - } - if strings.Index(pattern, "*") > strings.Index(pattern, "#") { - return errors.New("asterisk (*) in pattern must come before hashes (#)") +func (d *Chunker) configuredRemotePaths() []string { + seen := map[string]struct{}{} + paths := make([]string, 0, 1) + addPath := func(p string) { + p = strings.TrimSpace(p) + if p == "" { + return + } + p = utils.FixAndCleanPath(p) + if _, ok := seen[p]; ok { + return + } + seen[p] = struct{}{} + paths = append(paths, p) } - if ok, _ := regexp.MatchString("^[^#]*[#]+[^#]*$", pattern); !ok { - return errors.New("hashes (#) in pattern must be consecutive") + + addPath(d.RemotePath) + for _, line := range strings.Split(d.RemotePaths, "\n") { + addPath(line) } + return paths +} + +func (d *Chunker) setChunkNameFormat(pattern string) error { if dir, _ := path.Split(pattern); dir != "" { return errors.New("directory separator prohibited") } - if pattern[0] != '*' { - return errors.New("pattern must start with asterisk") + + nameStart, nameEnd, err := parseNameToken(pattern) + if err != nil { + return err + } + chunkStart, chunkEnd, chunkWidth, err := parseChunkToken(pattern) + if err != nil { + return err + } + if nameStart > chunkStart { + return errors.New("name token must come before chunk token") } - reHashes := regexp.MustCompile("[#]+") reDigits := "[0-9]+" - if hashCount > 1 { - reDigits = fmt.Sprintf("[0-9]{%d,}", hashCount) + if chunkWidth > 0 { + reDigits = fmt.Sprintf("[0-9]{%d,}", chunkWidth) } reDataOrCtrl := fmt.Sprintf("(?:(%s)|_(%s))", reDigits, ctrlTypeRegStr) - strRegex := regexp.QuoteMeta(pattern) - strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl) - strRegex = strings.Replace(strRegex, "\\*", "(.+?)", 1) - strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld) + beforeName := pattern[:nameStart] + between := pattern[nameEnd:chunkStart] + afterChunk := pattern[chunkEnd:] + + strRegex := fmt.Sprintf( + "^%s(.+?)%s%s%s(?:%s|%s)?$", + regexp.QuoteMeta(beforeName), + regexp.QuoteMeta(between), + reDataOrCtrl, + regexp.QuoteMeta(afterChunk), + tempSuffixRegStr, + tempSuffixRegOld, + ) d.nameRegexp = regexp.MustCompile(strRegex) fmtDigits := "%d" - if hashCount > 1 { - fmtDigits = fmt.Sprintf("%%0%dd", hashCount) - } - strFmt := strings.ReplaceAll(pattern, "%", "%%") - strFmt = strings.Replace(strFmt, "*", "%s", 1) - d.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits) + if chunkWidth > 0 { + fmtDigits = fmt.Sprintf("%%0%dd", chunkWidth) + } + d.dataNameFmt = strings.ReplaceAll(beforeName, "%", "%%") + + "%s" + + strings.ReplaceAll(between, "%", "%%") + + fmtDigits + + strings.ReplaceAll(afterChunk, "%", "%%") return nil } +func parseNameToken(pattern string) (start, end int, err error) { + nameMagicCount := strings.Count(pattern, "{name}") + switch nameMagicCount { + case 0: + return 0, 0, errors.New("pattern must contain one name token: {name}") + case 1: + default: + return 0, 0, errors.New("pattern must contain exactly one name token: {name}") + } + start = strings.Index(pattern, "{name}") + return start, start + len("{name}"), nil +} + +func parseChunkToken(pattern string) (start, end, width int, err error) { + chunkMatches := chunkTokenRegexp.FindAllStringSubmatchIndex(pattern, -1) + switch len(chunkMatches) { + case 0: + return 0, 0, 0, errors.New("pattern must contain one chunk token: {chunk} or {chunk:N}") + case 1: + default: + return 0, 0, 0, errors.New("pattern must contain exactly one chunk token: {chunk} or {chunk:N}") + } + match := chunkMatches[0] + start = match[0] + end = match[1] + if match[2] >= 0 && match[3] >= 0 { + width, err = strconv.Atoi(pattern[match[2]:match[3]]) + if err != nil || width <= 0 { + return 0, 0, 0, errors.New("chunk width in {chunk:N} must be a positive integer") + } + } + return start, end, width, nil +} + func (d *Chunker) makeChunkName(filePath string, chunkNo int, xactID string) string { dir, baseName := path.Split(filePath) name := fmt.Sprintf(d.dataNameFmt, baseName, chunkNo+d.StartFrom) @@ -197,75 +261,159 @@ func unmarshalMetadata(data []byte) (*chunkMetadata, error) { }, nil } -func (d *Chunker) joinRemotePath(logicalPath string) string { +func joinRemotePathWithBase(baseMountPath, logicalPath string) string { logicalPath = utils.FixAndCleanPath(logicalPath) if utils.PathEqual(logicalPath, "/") { - return d.RemotePath + return utils.FixAndCleanPath(baseMountPath) } - return path.Join(d.RemotePath, logicalPath) + return path.Join(utils.FixAndCleanPath(baseMountPath), logicalPath) +} + +func (d *Chunker) joinRemotePath(logicalPath string) string { + return joinRemotePathWithBase(d.RemotePath, logicalPath) +} + +func (d *Chunker) joinRemotePathForTarget(logicalPath string, remoteIndex int) string { + target := d.remoteTargets[remoteIndex] + return joinRemotePathWithBase(target.MountPath, logicalPath) } func (d *Chunker) getActualPathForRemote(logicalPath string) (string, error) { - _, actualPath, err := op.GetStorageAndActualPath(d.joinRemotePath(logicalPath)) + return d.getActualPathForRemoteOnTarget(logicalPath, 0) +} + +func (d *Chunker) getActualPathForRemoteOnTarget(logicalPath string, remoteIndex int) (string, error) { + _, actualPath, err := op.GetStorageAndActualPath(d.joinRemotePathForTarget(logicalPath, remoteIndex)) return actualPath, err } -func (d *Chunker) getActualChunkPath(filePath string, chunkNo int, xactID string) (string, error) { - return d.getActualPathForRemote(d.makeChunkName(filePath, chunkNo, xactID)) +func (d *Chunker) getActualChunkPath(filePath string, chunkNo int, xactID string, remoteIndex int) (string, error) { + return d.getActualPathForRemoteOnTarget(d.makeChunkName(filePath, chunkNo, xactID), remoteIndex) } -func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bool) ([]model.Obj, error) { - remotePath := d.joinRemotePath(dirPath) - entries, err := fsList(ctx, remotePath, refresh) - if err != nil { - return nil, err +func (d *Chunker) chunkTargetIndex(chunkNo int) int { + targetIndexes := d.chunkTargetIndexes() + if len(targetIndexes) == 0 { + return 0 + } + if chunkNo < 0 { + return targetIndexes[0] } + return targetIndexes[chunkNo%len(targetIndexes)] +} +func (d *Chunker) chunkTargetIndexes() []int { + if len(d.remoteTargets) <= 1 { + return []int{0} + } + if d.StoreChunksInPrimary { + targets := make([]int, 0, len(d.remoteTargets)) + for i := range d.remoteTargets { + targets = append(targets, i) + } + return targets + } + targets := make([]int, 0, len(d.remoteTargets)-1) + for i := 1; i < len(d.remoteTargets); i++ { + targets = append(targets, i) + } + if len(targets) == 0 { + return []int{0} + } + return targets +} + +func (d *Chunker) targetLocation(logicalPath string, remoteIndex int) objectLocation { + return objectLocation{ + LogicalPath: utils.FixAndCleanPath(logicalPath), + RemoteIndex: remoteIndex, + } +} + +func (d *Chunker) chunkLocation(filePath string, part chunkPart) objectLocation { + return d.targetLocation(d.makeChunkName(filePath, part.No, part.XactID), part.RemoteIndex) +} + +func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bool) ([]model.Obj, error) { groups := map[string]*groupInfo{} - var dirs []model.Obj - - for _, entry := range entries { - if entry.IsDir() { - dirs = append(dirs, &model.Object{ - Name: entry.GetName(), - Path: path.Join(dirPath, entry.GetName()), - Size: 0, - Modified: entry.ModTime(), - Ctime: entry.CreateTime(), - IsFolder: true, - HashInfo: entry.GetHash(), - }) - continue + dirMap := map[string]model.Obj{} + found := false + + for remoteIndex := range d.remoteTargets { + remotePath := d.joinRemotePathForTarget(dirPath, remoteIndex) + entries, err := fsList(ctx, remotePath, refresh) + if err != nil { + if errs.IsObjectNotFound(err) { + continue + } + return nil, err } + found = true + + for _, entry := range entries { + if entry.IsDir() { + if _, ok := dirMap[entry.GetName()]; !ok { + dirMap[entry.GetName()] = &model.Object{ + Name: entry.GetName(), + Path: path.Join(dirPath, entry.GetName()), + Size: 0, + Modified: entry.ModTime(), + Ctime: entry.CreateTime(), + IsFolder: true, + HashInfo: entry.GetHash(), + } + } + continue + } - mainName, chunkNo, ctrlType, xactID := d.parseChunkName(entry.GetName()) - if mainName == "" { - g := groups[entry.GetName()] + mainName, chunkNo, ctrlType, xactID := d.parseChunkName(entry.GetName()) + if mainName == "" { + g := groups[entry.GetName()] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[entry.GetName()] = g + } + if g.base == nil || remoteIndex < g.base.RemoteIndex { + g.base = &locatedObj{ + Obj: entry, + RemoteIndex: remoteIndex, + } + } + continue + } + if chunkNo < 0 || ctrlType != "" { + continue + } + g := groups[mainName] if g == nil { g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} - groups[entry.GetName()] = g + groups[mainName] = g + } + if g.partsByXact[xactID] == nil { + g.partsByXact[xactID] = map[int]chunkPart{} + } + part := chunkPart{ + No: chunkNo, + Size: entry.GetSize(), + XactID: xactID, + RemoteIndex: remoteIndex, + } + existing, ok := g.partsByXact[xactID][chunkNo] + if !ok || part.RemoteIndex < existing.RemoteIndex { + g.partsByXact[xactID][chunkNo] = part } - g.base = entry - continue - } - if chunkNo < 0 || ctrlType != "" { - continue - } - g := groups[mainName] - if g == nil { - g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} - groups[mainName] = g - } - if g.partsByXact[xactID] == nil { - g.partsByXact[xactID] = map[int]chunkPart{} - } - g.partsByXact[xactID][chunkNo] = chunkPart{ - No: chunkNo, - Size: entry.GetSize(), - XactID: xactID, } } + if !found && !utils.PathEqual(dirPath, "/") { + return nil, errs.ObjectNotFound + } + + dirs := make([]model.Obj, 0, len(dirMap)) + for _, obj := range dirMap { + dirs = append(dirs, obj) + } + result := make([]model.Obj, 0, len(dirs)+len(groups)) result = append(result, dirs...) for name, group := range groups { @@ -280,11 +428,180 @@ func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bo return result, nil } +func (d *Chunker) targetPathExists(ctx context.Context, remoteIndex int, logicalPath string) (bool, error) { + _, err := fs.Get(ctx, d.joinRemotePathForTarget(logicalPath, remoteIndex), &fs.GetArgs{NoLog: true}) + if err == nil { + return true, nil + } + if errs.IsObjectNotFound(err) { + return false, nil + } + return false, err +} + +func (d *Chunker) ensureDirOnTarget(ctx context.Context, remoteIndex int, logicalDirPath string) error { + logicalDirPath = utils.FixAndCleanPath(logicalDirPath) + if utils.PathEqual(logicalDirPath, "/") { + return nil + } + return fs.MakeDir(ctx, d.joinRemotePathForTarget(logicalDirPath, remoteIndex)) +} + +func (d *Chunker) ensureDirOnAllTargets(ctx context.Context, logicalDirPath string) error { + var errsList []error + for remoteIndex := range d.remoteTargets { + if err := d.ensureDirOnTarget(ctx, remoteIndex, logicalDirPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) existingLocationsForDir(ctx context.Context, logicalDirPath string) ([]int, error) { + locations := make([]int, 0, len(d.remoteTargets)) + for remoteIndex := range d.remoteTargets { + exists, err := d.targetPathExists(ctx, remoteIndex, logicalDirPath) + if err != nil { + return nil, err + } + if exists { + locations = append(locations, remoteIndex) + } + } + return locations, nil +} + +func (d *Chunker) dirLocationsOrAll(ctx context.Context, logicalDirPath string) ([]int, error) { + locations, err := d.existingLocationsForDir(ctx, logicalDirPath) + if err != nil { + return nil, err + } + if len(locations) > 0 { + return locations, nil + } + all := make([]int, 0, len(d.remoteTargets)) + for remoteIndex := range d.remoteTargets { + all = append(all, remoteIndex) + } + return all, nil +} + +func (d *Chunker) moveDirAcrossTargets(ctx context.Context, srcPath, dstDirPath string) error { + locations, err := d.existingLocationsForDir(ctx, srcPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDirPath); err != nil { + errsList = append(errsList, err) + continue + } + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + dstActualPath, err := d.getActualPathForRemoteOnTarget(dstDirPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Move(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, dstActualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) copyDirAcrossTargets(ctx context.Context, srcPath, dstDirPath string) error { + locations, err := d.dirLocationsOrAll(ctx, srcPath) + if err != nil { + return err + } + var errsList []error + for _, remoteIndex := range locations { + exists, err := d.targetPathExists(ctx, remoteIndex, srcPath) + if err != nil { + errsList = append(errsList, err) + continue + } + if !exists { + continue + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDirPath); err != nil { + errsList = append(errsList, err) + continue + } + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + dstActualPath, err := d.getActualPathForRemoteOnTarget(dstDirPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Copy(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, dstActualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) renameDirAcrossTargets(ctx context.Context, srcPath, newName string) error { + locations, err := d.existingLocationsForDir(ctx, srcPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Rename(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, newName); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) removeDirAcrossTargets(ctx context.Context, logicalPath string) error { + locations, err := d.existingLocationsForDir(ctx, logicalPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, actualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, group *groupInfo) (model.Obj, bool, error) { var meta *chunkMetadata var err error - if group.base != nil && group.base.GetSize() <= maxMetadataSizeRead && len(group.partsByXact) > 0 { - meta, err = d.readMetadata(ctx, path.Join(dirPath, name), group.base.GetSize()) + if group.base != nil && group.base.Obj.GetSize() <= maxMetadataSizeRead && len(group.partsByXact) > 0 { + meta, err = d.readMetadata(ctx, path.Join(dirPath, name), group.base.Obj.GetSize(), group.base.RemoteIndex) if err != nil { meta = nil } @@ -295,13 +612,14 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g Object: model.Object{ Name: name, Path: path.Join(dirPath, name), - Size: group.base.GetSize(), - Modified: group.base.ModTime(), - Ctime: group.base.CreateTime(), + Size: group.base.Obj.GetSize(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), IsFolder: false, - HashInfo: group.base.GetHash(), + HashInfo: group.base.Obj.GetHash(), }, - Main: group.base, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, }, true, nil } @@ -316,13 +634,14 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g Object: model.Object{ Name: name, Path: path.Join(dirPath, name), - Size: group.base.GetSize(), - Modified: group.base.ModTime(), - Ctime: group.base.CreateTime(), + Size: group.base.Obj.GetSize(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), IsFolder: false, - HashInfo: group.base.GetHash(), + HashInfo: group.base.Obj.GetHash(), }, - Main: group.base, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, }, true, nil } @@ -334,15 +653,16 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g Name: name, Path: path.Join(dirPath, name), Size: meta.Size, - Modified: group.base.ModTime(), - Ctime: group.base.CreateTime(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), IsFolder: false, HashInfo: buildHashInfo(meta), }, - Main: group.base, - Meta: meta, - Chunked: true, - UsesMeta: true, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, + Meta: meta, + Chunked: true, + UsesMeta: true, }, true, nil } return nil, false, nil @@ -355,13 +675,20 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g modified := time.Time{} ctime := time.Time{} if group.base != nil { - modified = group.base.ModTime() - ctime = group.base.CreateTime() + modified = group.base.Obj.ModTime() + ctime = group.base.Obj.CreateTime() } if meta != nil { size = meta.Size } + mainRemoteIndex := 0 + var mainObj model.Obj + if group.base != nil { + mainRemoteIndex = group.base.RemoteIndex + mainObj = group.base.Obj + } + return &Object{ Object: model.Object{ Name: name, @@ -372,20 +699,21 @@ func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, g IsFolder: false, HashInfo: buildHashInfo(meta), }, - Main: group.base, - Parts: parts, - Meta: meta, - Chunked: true, - UsesMeta: meta != nil, + Main: mainObj, + MainRemoteIndex: mainRemoteIndex, + Parts: parts, + Meta: meta, + Chunked: true, + UsesMeta: meta != nil, }, true, nil } -func (d *Chunker) readMetadata(ctx context.Context, logicalPath string, size int64) (*chunkMetadata, error) { - actualPath, err := d.getActualPathForRemote(logicalPath) +func (d *Chunker) readMetadata(ctx context.Context, logicalPath string, size int64, remoteIndex int) (*chunkMetadata, error) { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) if err != nil { return nil, err } - link, obj, err := op.Link(ctx, d.remoteStorage, actualPath, model.LinkArgs{}) + link, obj, err := op.Link(ctx, d.remoteTargets[remoteIndex].Storage, actualPath, model.LinkArgs{}) if err != nil { return nil, err } @@ -441,22 +769,22 @@ func (d *Chunker) linkedObject(obj model.Obj) *Object { return nil } -func (d *Chunker) chunkPathsForObject(obj *Object) []string { +func (d *Chunker) objectLocationsForObject(obj *Object) []objectLocation { if obj == nil { return nil } - paths := make([]string, 0, len(obj.Parts)+1) + locations := make([]objectLocation, 0, len(obj.Parts)+1) if obj.Chunked && obj.UsesMeta { - paths = append(paths, obj.GetPath()) + locations = append(locations, d.targetLocation(obj.GetPath(), obj.MainRemoteIndex)) } if !obj.Chunked { - paths = append(paths, obj.GetPath()) - return paths + locations = append(locations, d.targetLocation(obj.GetPath(), obj.MainRemoteIndex)) + return locations } for _, part := range obj.Parts { - paths = append(paths, d.makeChunkName(obj.GetPath(), part.No, part.XactID)) + locations = append(locations, d.chunkLocation(obj.GetPath(), part)) } - return paths + return locations } func (d *Chunker) cleanupReplacedObject(ctx context.Context, obj *Object, keep map[string]struct{}) error { @@ -464,29 +792,33 @@ func (d *Chunker) cleanupReplacedObject(ctx context.Context, obj *Object, keep m return nil } var errs []error - for _, logicalPath := range d.chunkPathsForObject(obj) { - if _, ok := keep[logicalPath]; ok { + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := keep[d.keepKey(location)]; ok { continue } - actualPath, err := d.getActualPathForRemote(logicalPath) + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) if err != nil { errs = append(errs, err) continue } - if err := op.Remove(ctx, d.remoteStorage, actualPath); err != nil { + if err := op.Remove(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath); err != nil { errs = append(errs, err) } } return errors.Join(errs...) } -func (d *Chunker) buildKeepSet(paths ...string) map[string]struct{} { - keep := make(map[string]struct{}, len(paths)) - for _, p := range paths { - if p == "" { +func (d *Chunker) keepKey(location objectLocation) string { + return fmt.Sprintf("%d:%s", location.RemoteIndex, utils.FixAndCleanPath(location.LogicalPath)) +} + +func (d *Chunker) buildKeepSet(locations ...objectLocation) map[string]struct{} { + keep := make(map[string]struct{}, len(locations)) + for _, location := range locations { + if location.LogicalPath == "" { continue } - keep[utils.FixAndCleanPath(p)] = struct{}{} + keep[d.keepKey(location)] = struct{}{} } return keep } diff --git a/drivers/chunker/util_test.go b/drivers/chunker/util_test.go index 96f06051b75..23786adc07e 100644 --- a/drivers/chunker/util_test.go +++ b/drivers/chunker/util_test.go @@ -41,6 +41,41 @@ func TestParseChunkName(t *testing.T) { } } +func TestNamedMagicNameFormat(t *testing.T) { + d := &Chunker{ + Addition: Addition{ + NameFormat: "chunk-{name}-{chunk:4}.bin", + StartFrom: defaultStartFrom, + }, + } + if err := d.setChunkNameFormat(d.NameFormat); err != nil { + t.Fatalf("setChunkNameFormat named magic: %v", err) + } + + got := d.makeChunkName("/video/movie.mkv", 0, "") + want := "/video/chunk-movie.mkv-0001.bin" + if got != want { + t.Fatalf("makeChunkName = %q, want %q", got, want) + } + + mainName, chunkNo, ctrlType, xactID := d.parseChunkName("chunk-movie.mkv-0003.bin") + if mainName != "movie.mkv" || chunkNo != 2 || ctrlType != "" || xactID != "" { + t.Fatalf("unexpected named magic parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) + } +} + +func TestLegacyNameFormatRejected(t *testing.T) { + d := &Chunker{ + Addition: Addition{ + NameFormat: "chunk-*.###.bin", + StartFrom: defaultStartFrom, + }, + } + if err := d.setChunkNameFormat(d.NameFormat); err == nil { + t.Fatal("expected legacy syntax to be rejected") + } +} + func TestMarshalAndUnmarshalMetadata(t *testing.T) { data, err := marshalMetadata(123, 2, "5d41402abc4b2a76b9719d911017c592", "", "") if err != nil { @@ -94,11 +129,14 @@ func TestBuildListedObjectWithoutMetadata(t *testing.T) { } raw, ok, err := d.buildListedObject(context.Background(), "/", "raw.txt", &groupInfo{ - base: &model.Object{ - Name: "raw.txt", - Size: 9, - Modified: now, - Ctime: now, + base: &locatedObj{ + Obj: &model.Object{ + Name: "raw.txt", + Size: 9, + Modified: now, + Ctime: now, + }, + RemoteIndex: 0, }, partsByXact: map[string]map[int]chunkPart{}, }) @@ -113,3 +151,61 @@ func TestBuildListedObjectWithoutMetadata(t *testing.T) { t.Fatalf("unexpected raw object: %+v", rawObj) } } + +func TestConfiguredRemotePaths(t *testing.T) { + d := &Chunker{ + Addition: Addition{ + RemotePath: "/s1/chunks", + RemotePaths: "\n/s2/chunks\n/s1/chunks\n /s3/chunks \n", + }, + } + got := d.configuredRemotePaths() + want := []string{"/s1/chunks", "/s2/chunks", "/s3/chunks"} + if len(got) != len(want) { + t.Fatalf("configuredRemotePaths length = %d, want %d (%v)", len(got), len(want), got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("configuredRemotePaths[%d] = %q, want %q", i, got[i], want[i]) + } + } +} + +func TestBuildKeepSetSeparatesRemoteTargets(t *testing.T) { + d := newTestChunker(t) + keep := d.buildKeepSet( + objectLocation{LogicalPath: "/movie.bin", RemoteIndex: 0}, + objectLocation{LogicalPath: "/movie.bin", RemoteIndex: 1}, + ) + if len(keep) != 2 { + t.Fatalf("buildKeepSet should keep distinct entries per remote target, got %d", len(keep)) + } +} + +func TestChunkTargetIndexes(t *testing.T) { + d := newTestChunker(t) + d.remoteTargets = []remoteTarget{{}, {}, {}} + d.StoreChunksInPrimary = true + if got := d.chunkTargetIndexes(); len(got) != 3 || got[0] != 0 || got[1] != 1 || got[2] != 2 { + t.Fatalf("chunkTargetIndexes with primary = %v", got) + } + if got := d.chunkTargetIndex(4); got != 1 { + t.Fatalf("chunkTargetIndex with primary = %d, want 1", got) + } + + d.StoreChunksInPrimary = false + if got := d.chunkTargetIndexes(); len(got) != 2 || got[0] != 1 || got[1] != 2 { + t.Fatalf("chunkTargetIndexes without primary = %v", got) + } + if got := d.chunkTargetIndex(0); got != 1 { + t.Fatalf("chunkTargetIndex without primary for first chunk = %d, want 1", got) + } + if got := d.chunkTargetIndex(3); got != 2 { + t.Fatalf("chunkTargetIndex without primary for chunk 3 = %d, want 2", got) + } + + d.remoteTargets = []remoteTarget{{}} + if got := d.chunkTargetIndex(0); got != 0 { + t.Fatalf("single target chunkTargetIndex = %d, want 0", got) + } +} From 428e04ee3c9cd2d04e985ed534ab17d54f8b8918 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 26 Mar 2026 15:54:27 +0800 Subject: [PATCH 612/659] test(chunker): remove util tests --- drivers/chunker/util_test.go | 211 ----------------------------------- 1 file changed, 211 deletions(-) delete mode 100644 drivers/chunker/util_test.go diff --git a/drivers/chunker/util_test.go b/drivers/chunker/util_test.go deleted file mode 100644 index 23786adc07e..00000000000 --- a/drivers/chunker/util_test.go +++ /dev/null @@ -1,211 +0,0 @@ -package chunker - -import ( - "context" - "testing" - "time" - - "github.com/alist-org/alist/v3/internal/model" -) - -func newTestChunker(t *testing.T) *Chunker { - t.Helper() - d := &Chunker{ - Addition: Addition{ - NameFormat: defaultChunkNameFmt, - StartFrom: defaultStartFrom, - }, - } - if err := d.setChunkNameFormat(d.NameFormat); err != nil { - t.Fatalf("setChunkNameFormat: %v", err) - } - return d -} - -func TestParseChunkName(t *testing.T) { - d := newTestChunker(t) - - mainName, chunkNo, ctrlType, xactID := d.parseChunkName("movie.mkv.rclone_chunk.001") - if mainName != "movie.mkv" || chunkNo != 0 || ctrlType != "" || xactID != "" { - t.Fatalf("unexpected parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) - } - - mainName, chunkNo, ctrlType, xactID = d.parseChunkName("movie.mkv.rclone_chunk.003_abcd") - if mainName != "movie.mkv" || chunkNo != 2 || ctrlType != "" || xactID != "abcd" { - t.Fatalf("unexpected temp parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) - } - - mainName, chunkNo, ctrlType, xactID = d.parseChunkName("movie.mkv.rclone_chunk._meta") - if mainName != "movie.mkv" || chunkNo != -1 || ctrlType != "meta" || xactID != "" { - t.Fatalf("unexpected control parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) - } -} - -func TestNamedMagicNameFormat(t *testing.T) { - d := &Chunker{ - Addition: Addition{ - NameFormat: "chunk-{name}-{chunk:4}.bin", - StartFrom: defaultStartFrom, - }, - } - if err := d.setChunkNameFormat(d.NameFormat); err != nil { - t.Fatalf("setChunkNameFormat named magic: %v", err) - } - - got := d.makeChunkName("/video/movie.mkv", 0, "") - want := "/video/chunk-movie.mkv-0001.bin" - if got != want { - t.Fatalf("makeChunkName = %q, want %q", got, want) - } - - mainName, chunkNo, ctrlType, xactID := d.parseChunkName("chunk-movie.mkv-0003.bin") - if mainName != "movie.mkv" || chunkNo != 2 || ctrlType != "" || xactID != "" { - t.Fatalf("unexpected named magic parse result: main=%q no=%d ctrl=%q txn=%q", mainName, chunkNo, ctrlType, xactID) - } -} - -func TestLegacyNameFormatRejected(t *testing.T) { - d := &Chunker{ - Addition: Addition{ - NameFormat: "chunk-*.###.bin", - StartFrom: defaultStartFrom, - }, - } - if err := d.setChunkNameFormat(d.NameFormat); err == nil { - t.Fatal("expected legacy syntax to be rejected") - } -} - -func TestMarshalAndUnmarshalMetadata(t *testing.T) { - data, err := marshalMetadata(123, 2, "5d41402abc4b2a76b9719d911017c592", "", "") - if err != nil { - t.Fatalf("marshalMetadata: %v", err) - } - meta, err := unmarshalMetadata(data) - if err != nil { - t.Fatalf("unmarshalMetadata: %v", err) - } - if meta.Version != 1 || meta.Size != 123 || meta.NChunks != 2 || meta.MD5 == "" || meta.XactID != "" { - t.Fatalf("unexpected metadata: %+v", meta) - } - - data, err = marshalMetadata(456, 3, "", "da39a3ee5e6b4b0d3255bfef95601890afd80709", "txn1") - if err != nil { - t.Fatalf("marshalMetadata with txn: %v", err) - } - meta, err = unmarshalMetadata(data) - if err != nil { - t.Fatalf("unmarshalMetadata with txn: %v", err) - } - if meta.Version != 2 || meta.Size != 456 || meta.NChunks != 3 || meta.SHA1 == "" || meta.XactID != "txn1" { - t.Fatalf("unexpected txn metadata: %+v", meta) - } -} - -func TestBuildListedObjectWithoutMetadata(t *testing.T) { - d := newTestChunker(t) - now := time.Now() - - obj, ok, err := d.buildListedObject(context.Background(), "/", "archive.bin", &groupInfo{ - partsByXact: map[string]map[int]chunkPart{ - "": { - 0: {No: 0, Size: 5}, - 1: {No: 1, Size: 7}, - }, - }, - }) - if err != nil { - t.Fatalf("buildListedObject: %v", err) - } - if !ok { - t.Fatal("expected grouped object") - } - grouped, ok := obj.(*Object) - if !ok { - t.Fatalf("expected *Object, got %T", obj) - } - if !grouped.Chunked || grouped.GetSize() != 12 || len(grouped.Parts) != 2 { - t.Fatalf("unexpected grouped object: %+v", grouped) - } - - raw, ok, err := d.buildListedObject(context.Background(), "/", "raw.txt", &groupInfo{ - base: &locatedObj{ - Obj: &model.Object{ - Name: "raw.txt", - Size: 9, - Modified: now, - Ctime: now, - }, - RemoteIndex: 0, - }, - partsByXact: map[string]map[int]chunkPart{}, - }) - if err != nil { - t.Fatalf("build raw listed object: %v", err) - } - if !ok { - t.Fatal("expected raw object") - } - rawObj := raw.(*Object) - if rawObj.Chunked || rawObj.GetSize() != 9 { - t.Fatalf("unexpected raw object: %+v", rawObj) - } -} - -func TestConfiguredRemotePaths(t *testing.T) { - d := &Chunker{ - Addition: Addition{ - RemotePath: "/s1/chunks", - RemotePaths: "\n/s2/chunks\n/s1/chunks\n /s3/chunks \n", - }, - } - got := d.configuredRemotePaths() - want := []string{"/s1/chunks", "/s2/chunks", "/s3/chunks"} - if len(got) != len(want) { - t.Fatalf("configuredRemotePaths length = %d, want %d (%v)", len(got), len(want), got) - } - for i := range want { - if got[i] != want[i] { - t.Fatalf("configuredRemotePaths[%d] = %q, want %q", i, got[i], want[i]) - } - } -} - -func TestBuildKeepSetSeparatesRemoteTargets(t *testing.T) { - d := newTestChunker(t) - keep := d.buildKeepSet( - objectLocation{LogicalPath: "/movie.bin", RemoteIndex: 0}, - objectLocation{LogicalPath: "/movie.bin", RemoteIndex: 1}, - ) - if len(keep) != 2 { - t.Fatalf("buildKeepSet should keep distinct entries per remote target, got %d", len(keep)) - } -} - -func TestChunkTargetIndexes(t *testing.T) { - d := newTestChunker(t) - d.remoteTargets = []remoteTarget{{}, {}, {}} - d.StoreChunksInPrimary = true - if got := d.chunkTargetIndexes(); len(got) != 3 || got[0] != 0 || got[1] != 1 || got[2] != 2 { - t.Fatalf("chunkTargetIndexes with primary = %v", got) - } - if got := d.chunkTargetIndex(4); got != 1 { - t.Fatalf("chunkTargetIndex with primary = %d, want 1", got) - } - - d.StoreChunksInPrimary = false - if got := d.chunkTargetIndexes(); len(got) != 2 || got[0] != 1 || got[1] != 2 { - t.Fatalf("chunkTargetIndexes without primary = %v", got) - } - if got := d.chunkTargetIndex(0); got != 1 { - t.Fatalf("chunkTargetIndex without primary for first chunk = %d, want 1", got) - } - if got := d.chunkTargetIndex(3); got != 2 { - t.Fatalf("chunkTargetIndex without primary for chunk 3 = %d, want 2", got) - } - - d.remoteTargets = []remoteTarget{{}} - if got := d.chunkTargetIndex(0); got != 0 { - t.Fatalf("single target chunkTargetIndex = %d, want 0", got) - } -} From bd725d547c52387068117515a3680312118c1642 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 27 Mar 2026 08:53:39 +0800 Subject: [PATCH 613/659] feat: add burn-after-read support for shares --- internal/db/share.go | 11 +++++++++++ internal/model/share.go | 2 ++ server/handles/share.go | 29 +++++++++++++++++++++++++++++ server/handles/share_public.go | 14 ++++++++++++++ 4 files changed, 56 insertions(+) diff --git a/internal/db/share.go b/internal/db/share.go index 54eb3d5f2f9..de96e62d2ee 100644 --- a/internal/db/share.go +++ b/internal/db/share.go @@ -64,3 +64,14 @@ func TouchShareDownload(shareID string) error { "download_count": gorm.Expr("download_count + ?", 1), }).Error } + +func ConsumeShare(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ? AND burn_after_read = ? AND consumed_at IS NULL", shareID, true). + Updates(map[string]interface{}{ + "enabled": false, + "consumed_at": now, + "last_access_at": now, + }).Error +} diff --git a/internal/model/share.go b/internal/model/share.go index a3fc90af008..4e7f5379484 100644 --- a/internal/model/share.go +++ b/internal/model/share.go @@ -11,12 +11,14 @@ type Share struct { IsDir bool `json:"is_dir"` PasswordHash string `json:"-" gorm:"size:64"` PasswordSalt string `json:"-" gorm:"size:32"` + BurnAfterRead bool `json:"burn_after_read" gorm:"default:false"` AllowPreview bool `json:"allow_preview" gorm:"default:true"` AllowDownload bool `json:"allow_download" gorm:"default:true"` Enabled bool `json:"enabled" gorm:"default:true;index"` ViewCount int64 `json:"view_count"` DownloadCount int64 `json:"download_count"` LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` ExpiresAt *time.Time `json:"expires_at"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` diff --git a/server/handles/share.go b/server/handles/share.go index 4626da8f0a2..f0a44bac664 100644 --- a/server/handles/share.go +++ b/server/handles/share.go @@ -26,6 +26,7 @@ type CreateShareReq struct { Name string `json:"name"` Password string `json:"password"` ExpireHours int64 `json:"expire_hours"` + BurnAfterRead *bool `json:"burn_after_read"` AllowPreview *bool `json:"allow_preview"` AllowDownload *bool `json:"allow_download"` } @@ -59,12 +60,14 @@ type ShareResp struct { RootPath string `json:"root_path"` IsDir bool `json:"is_dir"` HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` AllowPreview bool `json:"allow_preview"` AllowDownload bool `json:"allow_download"` Enabled bool `json:"enabled"` ViewCount int64 `json:"view_count"` DownloadCount int64 `json:"download_count"` LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` ExpiresAt *time.Time `json:"expires_at"` CreatedAt time.Time `json:"created_at"` UpdatedAt time.Time `json:"updated_at"` @@ -76,9 +79,11 @@ type PublicShareInfoResp struct { Name string `json:"name"` IsDir bool `json:"is_dir"` HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` AllowPreview bool `json:"allow_preview"` AllowDownload bool `json:"allow_download"` Authed bool `json:"authed"` + ConsumedAt *time.Time `json:"consumed_at"` ExpiresAt *time.Time `json:"expires_at"` CreatedAt time.Time `json:"created_at"` } @@ -123,12 +128,14 @@ func toShareResp(c *gin.Context, share *model.Share) ShareResp { RootPath: share.RootPath, IsDir: share.IsDir, HasPassword: share.HasPassword(), + BurnAfterRead: share.BurnAfterRead, AllowPreview: share.AllowPreview, AllowDownload: share.AllowDownload, Enabled: share.Enabled, ViewCount: share.ViewCount, DownloadCount: share.DownloadCount, LastAccessAt: share.LastAccessAt, + ConsumedAt: share.ConsumedAt, ExpiresAt: share.ExpiresAt, CreatedAt: share.CreatedAt, UpdatedAt: share.UpdatedAt, @@ -177,6 +184,10 @@ func getShareAccessToken(c *gin.Context, fallback string) string { func ensureShareAvailable(c *gin.Context, share *model.Share) bool { now := time.Now() + if share.ConsumedAt != nil { + common.ErrorStrResp(c, "share has been consumed", 410) + return false + } if !share.Enabled { common.ErrorStrResp(c, "share is disabled", 404) return false @@ -188,6 +199,19 @@ func ensureShareAvailable(c *gin.Context, share *model.Share) bool { return true } +func consumeShareIfNeeded(share *model.Share) error { + if !share.BurnAfterRead || share.ConsumedAt != nil { + return nil + } + now := time.Now() + if err := db.ConsumeShare(share.ShareID); err != nil { + return err + } + share.Enabled = false + share.ConsumedAt = &now + return nil +} + func ensureShareAccess(c *gin.Context, share *model.Share, token string) bool { if !share.HasPassword() { return true @@ -303,6 +327,10 @@ func CreateShare(c *gin.Context) { if req.AllowDownload != nil { allowDownload = *req.AllowDownload } + burnAfterRead := false + if req.BurnAfterRead != nil { + burnAfterRead = *req.BurnAfterRead + } var expiresAt *time.Time if req.ExpireHours > 0 { expires := time.Now().Add(time.Duration(req.ExpireHours) * time.Hour) @@ -314,6 +342,7 @@ func CreateShare(c *gin.Context) { Name: normalizeShareName(obj, req.Name), RootPath: reqPath, IsDir: obj.IsDir(), + BurnAfterRead: burnAfterRead, AllowPreview: allowPreview, AllowDownload: allowDownload, Enabled: true, diff --git a/server/handles/share_public.go b/server/handles/share_public.go index 36476bae466..e1f7cf17012 100644 --- a/server/handles/share_public.go +++ b/server/handles/share_public.go @@ -40,9 +40,11 @@ func GetPublicShareInfo(c *gin.Context) { Name: share.Name, IsDir: share.IsDir, HasPassword: share.HasPassword(), + BurnAfterRead: share.BurnAfterRead, AllowPreview: share.AllowPreview, AllowDownload: share.AllowDownload, Authed: authed, + ConsumedAt: share.ConsumedAt, ExpiresAt: share.ExpiresAt, CreatedAt: share.CreatedAt, }) @@ -138,6 +140,10 @@ func ListPublicShare(c *gin.Context) { } content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) } + if err := consumeShareIfNeeded(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } common.SuccessResp(c, PublicShareListResp{ Content: content, Total: int64(total), @@ -219,6 +225,10 @@ func ShareDown(c *gin.Context) { return } _ = db.TouchShareDownload(share.ShareID) + if err := consumeShareIfNeeded(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } c.Set("path", targetPath) Down(c) } @@ -254,6 +264,10 @@ func ShareProxy(c *gin.Context) { common.ErrorStrResp(c, "directory preview is not supported", 400) return } + if err := consumeShareIfNeeded(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } c.Set("path", targetPath) Proxy(c) } From 4f60c3126138e5852467ea5f28dd2c2058133212 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 27 Mar 2026 18:56:10 +0800 Subject: [PATCH 614/659] feat: extend share lifecycle controls --- internal/db/share.go | 63 ++++++- internal/model/share.go | 31 ++- server/handles/share.go | 331 ++++++++++++++++++++++++++------- server/handles/share_public.go | 31 +-- server/router.go | 2 + 5 files changed, 369 insertions(+), 89 deletions(-) diff --git a/internal/db/share.go b/internal/db/share.go index de96e62d2ee..c92e546755e 100644 --- a/internal/db/share.go +++ b/internal/db/share.go @@ -5,6 +5,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "gorm.io/gorm" + "gorm.io/gorm/clause" ) func GetShareByShareID(shareID string) (*model.Share, error) { @@ -15,6 +16,14 @@ func GetShareByShareID(shareID string) (*model.Share, error) { return &share, nil } +func GetShareByCreatorAndShareID(creatorID uint, shareID string) (*model.Share, error) { + var share model.Share + if err := db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Take(&share).Error; err != nil { + return nil, err + } + return &share, nil +} + func ShareIDExists(shareID string) bool { var count int64 if err := db.Model(&model.Share{}).Where("share_id = ?", shareID).Count(&count).Error; err != nil { @@ -23,6 +32,14 @@ func ShareIDExists(shareID string) bool { return count > 0 } +func ShareIDExistsExceptID(shareID string, id uint) bool { + var count int64 + if err := db.Model(&model.Share{}).Where("share_id = ? AND id <> ?", shareID, id).Count(&count).Error; err != nil { + return false + } + return count > 0 +} + func CreateShare(share *model.Share) error { return db.Create(share).Error } @@ -45,6 +62,12 @@ func DeleteShareByShareID(creatorID uint, shareID string) error { return db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Delete(&model.Share{}).Error } +func DisableShareByShareID(creatorID uint, shareID string) error { + return db.Model(&model.Share{}). + Where("creator_id = ? AND share_id = ?", creatorID, shareID). + Update("enabled", false).Error +} + func TouchShareView(shareID string) error { now := time.Now() return db.Model(&model.Share{}). @@ -65,13 +88,37 @@ func TouchShareDownload(shareID string) error { }).Error } -func ConsumeShare(shareID string) error { - now := time.Now() - return db.Model(&model.Share{}). - Where("share_id = ? AND burn_after_read = ? AND consumed_at IS NULL", shareID, true). - Updates(map[string]interface{}{ - "enabled": false, - "consumed_at": now, +func RecordShareAccess(shareID string) (*model.Share, error) { + var updated model.Share + err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + Where("share_id = ?", shareID). + Take(&updated).Error; err != nil { + return err + } + + now := time.Now() + updated.AccessCount++ + updated.LastAccessAt = &now + updates := map[string]interface{}{ + "access_count": updated.AccessCount, "last_access_at": now, - }).Error + } + + limit := updated.EffectiveAccessLimit() + if limit > 0 && updated.AccessCount >= limit { + updated.Enabled = false + updated.ConsumedAt = &now + updates["enabled"] = false + updates["consumed_at"] = now + } + + return tx.Model(&model.Share{}). + Where("id = ?", updated.ID). + Updates(updates).Error + }) + if err != nil { + return nil, err + } + return &updated, nil } diff --git a/internal/model/share.go b/internal/model/share.go index 4e7f5379484..30fbb8cb7d4 100644 --- a/internal/model/share.go +++ b/internal/model/share.go @@ -12,6 +12,8 @@ type Share struct { PasswordHash string `json:"-" gorm:"size:64"` PasswordSalt string `json:"-" gorm:"size:32"` BurnAfterRead bool `json:"burn_after_read" gorm:"default:false"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` AllowPreview bool `json:"allow_preview" gorm:"default:true"` AllowDownload bool `json:"allow_download" gorm:"default:true"` Enabled bool `json:"enabled" gorm:"default:true;index"` @@ -28,6 +30,33 @@ func (s Share) HasPassword() bool { return s.PasswordHash != "" } +func (s Share) EffectiveAccessLimit() int64 { + if s.AccessLimit > 0 { + return s.AccessLimit + } + if s.BurnAfterRead { + return 1 + } + return 0 +} + +func (s Share) RemainingAccesses() int64 { + limit := s.EffectiveAccessLimit() + if limit <= 0 { + return 0 + } + remaining := limit - s.AccessCount + if remaining < 0 { + return 0 + } + return remaining +} + +func (s Share) IsConsumed() bool { + limit := s.EffectiveAccessLimit() + return s.ConsumedAt != nil || (limit > 0 && s.AccessCount >= limit) +} + func (s Share) IsExpired(now time.Time) bool { - return s.ExpiresAt != nil && s.ExpiresAt.Before(now) + return s.ExpiresAt != nil && !s.ExpiresAt.After(now) } diff --git a/server/handles/share.go b/server/handles/share.go index f0a44bac664..99a693b7a5f 100644 --- a/server/handles/share.go +++ b/server/handles/share.go @@ -5,6 +5,7 @@ import ( "fmt" "net/url" stdpath "path" + "regexp" "strings" "time" @@ -21,16 +22,32 @@ import ( const shareAccessTokenLifetime = 24 * time.Hour +var shareIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,32}$`) + type CreateShareReq struct { Path string `json:"path" binding:"required"` + ShareID string `json:"share_id"` Name string `json:"name"` Password string `json:"password"` + ExpireAt string `json:"expire_at"` ExpireHours int64 `json:"expire_hours"` + AccessLimit int64 `json:"access_limit"` BurnAfterRead *bool `json:"burn_after_read"` AllowPreview *bool `json:"allow_preview"` AllowDownload *bool `json:"allow_download"` } +type UpdateShareReq struct { + ShareID string `json:"share_id" binding:"required"` + NewShareID string `json:"new_share_id"` + Name string `json:"name"` + Password string `json:"password"` + ExpireAt *string `json:"expire_at"` + AccessLimit *int64 `json:"access_limit"` + AllowPreview *bool `json:"allow_preview"` + AllowDownload *bool `json:"allow_download"` +} + type ShareDeleteReq struct { ShareID string `json:"share_id" binding:"required"` } @@ -54,38 +71,44 @@ type PublicShareListReq struct { } type ShareResp struct { - ID uint `json:"id"` - ShareID string `json:"share_id"` - Name string `json:"name"` - RootPath string `json:"root_path"` - IsDir bool `json:"is_dir"` - HasPassword bool `json:"has_password"` - BurnAfterRead bool `json:"burn_after_read"` - AllowPreview bool `json:"allow_preview"` - AllowDownload bool `json:"allow_download"` - Enabled bool `json:"enabled"` - ViewCount int64 `json:"view_count"` - DownloadCount int64 `json:"download_count"` - LastAccessAt *time.Time `json:"last_access_at"` - ConsumedAt *time.Time `json:"consumed_at"` - ExpiresAt *time.Time `json:"expires_at"` - CreatedAt time.Time `json:"created_at"` - UpdatedAt time.Time `json:"updated_at"` - URL string `json:"url"` + ID uint `json:"id"` + ShareID string `json:"share_id"` + Name string `json:"name"` + RootPath string `json:"root_path"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + RemainingAccesses int64 `json:"remaining_accesses"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Enabled bool `json:"enabled"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` } type PublicShareInfoResp struct { - ShareID string `json:"share_id"` - Name string `json:"name"` - IsDir bool `json:"is_dir"` - HasPassword bool `json:"has_password"` - BurnAfterRead bool `json:"burn_after_read"` - AllowPreview bool `json:"allow_preview"` - AllowDownload bool `json:"allow_download"` - Authed bool `json:"authed"` - ConsumedAt *time.Time `json:"consumed_at"` - ExpiresAt *time.Time `json:"expires_at"` - CreatedAt time.Time `json:"created_at"` + ShareID string `json:"share_id"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + RemainingAccesses int64 `json:"remaining_accesses"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Authed bool `json:"authed"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` } type PublicShareObjResp struct { @@ -121,33 +144,41 @@ func shareURL(c *gin.Context, shareID string) string { } func toShareResp(c *gin.Context, share *model.Share) ShareResp { + accessLimit := share.EffectiveAccessLimit() return ShareResp{ - ID: share.ID, - ShareID: share.ShareID, - Name: share.Name, - RootPath: share.RootPath, - IsDir: share.IsDir, - HasPassword: share.HasPassword(), - BurnAfterRead: share.BurnAfterRead, - AllowPreview: share.AllowPreview, - AllowDownload: share.AllowDownload, - Enabled: share.Enabled, - ViewCount: share.ViewCount, - DownloadCount: share.DownloadCount, - LastAccessAt: share.LastAccessAt, - ConsumedAt: share.ConsumedAt, - ExpiresAt: share.ExpiresAt, - CreatedAt: share.CreatedAt, - UpdatedAt: share.UpdatedAt, - URL: shareURL(c, share.ShareID), + ID: share.ID, + ShareID: share.ShareID, + Name: share.Name, + RootPath: share.RootPath, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + BurnAfterRead: accessLimit == 1, + AccessLimit: accessLimit, + AccessCount: share.AccessCount, + RemainingAccesses: share.RemainingAccesses(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Enabled: share.Enabled, + ViewCount: share.ViewCount, + DownloadCount: share.DownloadCount, + LastAccessAt: share.LastAccessAt, + ConsumedAt: share.ConsumedAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + UpdatedAt: share.UpdatedAt, + URL: shareURL(c, share.ShareID), } } func normalizeShareName(obj model.Obj, name string) string { + return normalizeOptionalShareName(name, obj.GetName()) +} + +func normalizeOptionalShareName(name, fallback string) string { if strings.TrimSpace(name) != "" { return strings.TrimSpace(name) } - return obj.GetName() + return fallback } func generateShareID() (string, error) { @@ -164,6 +195,84 @@ func sharePasswordHash(password, salt string) string { return model.HashPwd(model.StaticHash(password), salt) } +func validateCustomShareID(shareID string) error { + if shareID == "" { + return nil + } + if !shareIDPattern.MatchString(shareID) { + return fmt.Errorf("share_id must be 1-32 characters of letters, numbers, underscore or hyphen") + } + return nil +} + +func resolveRequestedShareID(rawShareID, fallback string, excludeID uint) (string, error) { + shareID := strings.TrimSpace(rawShareID) + if shareID == "" { + if fallback != "" { + return fallback, nil + } + return generateShareID() + } + if err := validateCustomShareID(shareID); err != nil { + return "", err + } + if excludeID == 0 { + if db.ShareIDExists(shareID) { + return "", fmt.Errorf("share link already exists") + } + return shareID, nil + } + if db.ShareIDExistsExceptID(shareID, excludeID) { + return "", fmt.Errorf("share link already exists") + } + return shareID, nil +} + +func normalizeShareAccessLimit(accessLimit int64, burnAfterRead *bool) (int64, bool, error) { + if accessLimit < 0 { + return 0, false, fmt.Errorf("access_limit must be 0 or greater") + } + if accessLimit == 0 && burnAfterRead != nil && *burnAfterRead { + accessLimit = 1 + } + return accessLimit, accessLimit == 1, nil +} + +func parseShareExpireAt(raw string) (*time.Time, error) { + value := strings.TrimSpace(raw) + if value == "" { + return nil, nil + } + if parsed, err := time.Parse(time.RFC3339, value); err == nil { + return &parsed, nil + } + layouts := []string{ + "2006-01-02T15:04:05", + "2006-01-02T15:04", + "2006-01-02 15:04:05", + } + for _, layout := range layouts { + if parsed, err := time.ParseInLocation(layout, value, time.Local); err == nil { + return &parsed, nil + } + } + return nil, fmt.Errorf("invalid expire_at") +} + +func resolveShareExpireAt(expireAt string, expireHours int64) (*time.Time, error) { + if strings.TrimSpace(expireAt) != "" { + return parseShareExpireAt(expireAt) + } + if expireHours < 0 { + return nil, fmt.Errorf("expire_hours must be 0 or greater") + } + if expireHours == 0 { + return nil, nil + } + expires := time.Now().Add(time.Duration(expireHours) * time.Hour) + return &expires, nil +} + func sharePasswordMatched(share *model.Share, password string) bool { if !share.HasPassword() { return true @@ -184,7 +293,7 @@ func getShareAccessToken(c *gin.Context, fallback string) string { func ensureShareAvailable(c *gin.Context, share *model.Share) bool { now := time.Now() - if share.ConsumedAt != nil { + if share.IsConsumed() { common.ErrorStrResp(c, "share has been consumed", 410) return false } @@ -199,16 +308,14 @@ func ensureShareAvailable(c *gin.Context, share *model.Share) bool { return true } -func consumeShareIfNeeded(share *model.Share) error { - if !share.BurnAfterRead || share.ConsumedAt != nil { - return nil - } - now := time.Now() - if err := db.ConsumeShare(share.ShareID); err != nil { +func recordShareAccess(share *model.Share) error { + updated, err := db.RecordShareAccess(share.ShareID) + if err != nil { return err } - share.Enabled = false - share.ConsumedAt = &now + if updated != nil { + *share = *updated + } return nil } @@ -314,9 +421,9 @@ func CreateShare(c *gin.Context) { common.ErrorResp(c, err, 500) return } - shareID, err := generateShareID() + shareID, err := resolveRequestedShareID(req.ShareID, "", 0) if err != nil { - common.ErrorResp(c, err, 500, true) + common.ErrorResp(c, err, 400) return } allowPreview := true @@ -327,14 +434,15 @@ func CreateShare(c *gin.Context) { if req.AllowDownload != nil { allowDownload = *req.AllowDownload } - burnAfterRead := false - if req.BurnAfterRead != nil { - burnAfterRead = *req.BurnAfterRead + accessLimit, burnAfterRead, err := normalizeShareAccessLimit(req.AccessLimit, req.BurnAfterRead) + if err != nil { + common.ErrorResp(c, err, 400) + return } - var expiresAt *time.Time - if req.ExpireHours > 0 { - expires := time.Now().Add(time.Duration(req.ExpireHours) * time.Hour) - expiresAt = &expires + expiresAt, err := resolveShareExpireAt(req.ExpireAt, req.ExpireHours) + if err != nil { + common.ErrorResp(c, err, 400) + return } share := &model.Share{ ShareID: shareID, @@ -343,6 +451,7 @@ func CreateShare(c *gin.Context) { RootPath: reqPath, IsDir: obj.IsDir(), BurnAfterRead: burnAfterRead, + AccessLimit: accessLimit, AllowPreview: allowPreview, AllowDownload: allowDownload, Enabled: true, @@ -359,6 +468,78 @@ func CreateShare(c *gin.Context) { common.SuccessResp(c, toShareResp(c, share)) } +func UpdateShare(c *gin.Context) { + var req UpdateShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + share, err := db.GetShareByCreatorAndShareID(user.ID, req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + + shareID, err := resolveRequestedShareID(req.NewShareID, share.ShareID, share.ID) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + + allowPreview := share.AllowPreview + if req.AllowPreview != nil { + allowPreview = *req.AllowPreview + } + allowDownload := share.AllowDownload + if req.AllowDownload != nil { + allowDownload = *req.AllowDownload + } + + accessLimit := share.EffectiveAccessLimit() + burnAfterRead := accessLimit == 1 + if req.AccessLimit != nil { + accessLimit, burnAfterRead, err = normalizeShareAccessLimit(*req.AccessLimit, nil) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + expiresAt := share.ExpiresAt + if req.ExpireAt != nil { + expiresAt, err = parseShareExpireAt(*req.ExpireAt) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + share.ShareID = shareID + share.Name = normalizeOptionalShareName(req.Name, share.Name) + share.BurnAfterRead = burnAfterRead + share.AccessLimit = accessLimit + share.AllowPreview = allowPreview + share.AllowDownload = allowDownload + share.ExpiresAt = expiresAt + if req.Password != "" { + share.PasswordSalt = random.String(16) + share.PasswordHash = sharePasswordHash(req.Password, share.PasswordSalt) + } + if share.Enabled && accessLimit > 0 && share.AccessCount >= accessLimit { + now := time.Now() + share.Enabled = false + if share.ConsumedAt == nil { + share.ConsumedAt = &now + } + } + if err := db.UpdateShare(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, toShareResp(c, share)) +} + func ListShares(c *gin.Context) { var req model.PageReq if err := c.ShouldBind(&req); err != nil { @@ -382,6 +563,24 @@ func ListShares(c *gin.Context) { }) } +func DisableShare(c *gin.Context) { + var req ShareDeleteReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if _, err := db.GetShareByCreatorAndShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 404) + return + } + if err := db.DisableShareByShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + func DeleteShare(c *gin.Context) { var req ShareDeleteReq if err := c.ShouldBind(&req); err != nil { diff --git a/server/handles/share_public.go b/server/handles/share_public.go index e1f7cf17012..22aba45bc4c 100644 --- a/server/handles/share_public.go +++ b/server/handles/share_public.go @@ -36,17 +36,20 @@ func GetPublicShareInfo(c *gin.Context) { _ = db.TouchShareView(share.ShareID) } common.SuccessResp(c, PublicShareInfoResp{ - ShareID: share.ShareID, - Name: share.Name, - IsDir: share.IsDir, - HasPassword: share.HasPassword(), - BurnAfterRead: share.BurnAfterRead, - AllowPreview: share.AllowPreview, - AllowDownload: share.AllowDownload, - Authed: authed, - ConsumedAt: share.ConsumedAt, - ExpiresAt: share.ExpiresAt, - CreatedAt: share.CreatedAt, + ShareID: share.ShareID, + Name: share.Name, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + BurnAfterRead: share.EffectiveAccessLimit() == 1, + AccessLimit: share.EffectiveAccessLimit(), + AccessCount: share.AccessCount, + RemainingAccesses: share.RemainingAccesses(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Authed: authed, + ConsumedAt: share.ConsumedAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, }) } @@ -140,7 +143,7 @@ func ListPublicShare(c *gin.Context) { } content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) } - if err := consumeShareIfNeeded(share); err != nil { + if err := recordShareAccess(share); err != nil { common.ErrorResp(c, err, 500, true) return } @@ -225,7 +228,7 @@ func ShareDown(c *gin.Context) { return } _ = db.TouchShareDownload(share.ShareID) - if err := consumeShareIfNeeded(share); err != nil { + if err := recordShareAccess(share); err != nil { common.ErrorResp(c, err, 500, true) return } @@ -264,7 +267,7 @@ func ShareProxy(c *gin.Context) { common.ErrorStrResp(c, "directory preview is not supported", 400) return } - if err := consumeShareIfNeeded(share); err != nil { + if err := recordShareAccess(share); err != nil { common.ErrorResp(c, err, 500, true) return } diff --git a/server/router.go b/server/router.go index e63b667297e..34df40096b4 100644 --- a/server/router.go +++ b/server/router.go @@ -111,6 +111,8 @@ func Init(e *gin.Engine) { _fs(auth.Group("/fs")) share := auth.Group("/share", middlewares.AuthNotGuest) share.POST("/create", handles.CreateShare) + share.POST("/update", handles.UpdateShare) + share.POST("/disable", handles.DisableShare) share.GET("/list", handles.ListShares) share.POST("/delete", handles.DeleteShare) _task(auth.Group("/task", middlewares.AuthNotGuest)) From fe0c5b8dc52de5ad6e67a3301478f6ddd219ffbd Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:04:29 +0800 Subject: [PATCH 615/659] feat(driver/streamtape): add response types for remote download, file info, and conversion status --- drivers/streamtape/types.go | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/drivers/streamtape/types.go b/drivers/streamtape/types.go index 4e470ff927e..347e89d8a67 100644 --- a/drivers/streamtape/types.go +++ b/drivers/streamtape/types.go @@ -52,3 +52,46 @@ type createFolderResult struct { type uploadURLResult struct { URL string `json:"url"` } + +type remoteDlAddResult struct { + ID string `json:"id"` + FolderID string `json:"folderid"` +} + +type remoteDlStatusResult map[string]remoteDlStatusItem + +type remoteDlStatusItem struct { + ID string `json:"id"` + RemoteURL string `json:"remoteurl"` + Status string `json:"status"` + BytesLoaded interface{} `json:"bytes_loaded"` + BytesTotal interface{} `json:"bytes_total"` + FolderID string `json:"folderid"` + Added string `json:"added"` + LastUpdate string `json:"last_update"` + ExtID bool `json:"extid"` + URL bool `json:"url"` +} + +type fileInfoResult map[string]fileInfoItem + +type fileInfoItem struct { + ID string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + Converted bool `json:"converted"` + Status int `json:"status"` +} + +type conversionResult []conversionItem + +type conversionItem struct { + Name string `json:"name"` + FolderID string `json:"folderid"` + Status string `json:"status"` + Progress int `json:"progress"` + Retries int `json:"retries"` + Link string `json:"link"` + LinkID string `json:"linkid"` +} From 11254ddba154b135a530423d8bd98331a6210deb Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:06:12 +0800 Subject: [PATCH 616/659] feat(driver/streamtape): add remote upload ID helpers --- drivers/streamtape/util.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/drivers/streamtape/util.go b/drivers/streamtape/util.go index f6d8373763e..51ad73d59bb 100644 --- a/drivers/streamtape/util.go +++ b/drivers/streamtape/util.go @@ -144,3 +144,20 @@ func extractFileIDFromUploadBody(body []byte) string { } return "" } + +const remoteUploadPrefix = "ru:" + +func encodeRemoteUploadID(id string) string { + return remoteUploadPrefix + id +} + +func remoteUploadIDFromObjID(id string) string { + if strings.HasPrefix(id, remoteUploadPrefix) { + return strings.TrimPrefix(id, remoteUploadPrefix) + } + return "" +} + +func isRemoteUploadID(id string) bool { + return strings.HasPrefix(id, remoteUploadPrefix) +} From d59c58916cf6529f4a8d5785b426f079e49acdd4 Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:08:26 +0800 Subject: [PATCH 617/659] feat(driver/streamtape): implement PutURL for remote uploads --- drivers/streamtape/driver.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go index 042df42cbe6..7eb9d6c9e33 100644 --- a/drivers/streamtape/driver.go +++ b/drivers/streamtape/driver.go @@ -381,6 +381,35 @@ func (d *Streamtape) Put(ctx context.Context, dstDir model.Obj, file model.FileS }, nil } +// PutURL initiates a remote upload from an external URL +func (d *Streamtape) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + params := map[string]string{ + "url": url, + } + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + if name != "" { + params["name"] = name + } + + var result remoteDlAddResult + if err := d.callAPI(ctx, "/remotedl/add", params, &result); err != nil { + return nil, err + } + + return &model.Object{ + ID: encodeRemoteUploadID(result.ID), + Name: name, + IsFolder: false, + }, nil +} + func (d *Streamtape) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { return nil, errs.NotImplement } From 6761fe7a1ac69b21bfcac54c7385b4ac96426987 Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:11:35 +0800 Subject: [PATCH 618/659] feat(driver/streamtape): implement Other method with remotedl_status, remotedl_remove, file_info, thumbnail, conversion_status --- drivers/streamtape/driver.go | 112 +++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go index 7eb9d6c9e33..b261f86100e 100644 --- a/drivers/streamtape/driver.go +++ b/drivers/streamtape/driver.go @@ -426,4 +426,116 @@ func (d *Streamtape) ArchiveDecompress(ctx context.Context, srcObj, dstDir model return nil, errs.NotImplement } +func (d *Streamtape) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch strings.ToLower(args.Method) { + case "remotedl_status": + return d.remoteDlStatus(ctx, args) + case "remotedl_remove": + return d.remoteDlRemove(ctx, args) + case "file_info": + return d.fileInfo(ctx, args) + case "thumbnail": + return d.thumbnail(ctx, args) + case "conversion_status": + return d.conversionStatus(ctx, args) + default: + return nil, errs.NotSupport + } +} + +func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) + if uploadID == "" { + // Try to get from data + if data, ok := args.Data.(map[string]interface{}); ok { + if id, ok := data["id"].(string); ok { + uploadID = id + } + } + } + if uploadID == "" { + return nil, fmt.Errorf("remote upload ID required") + } + + var result remoteDlStatusResult + if err := d.callAPI(ctx, "/remotedl/status", map[string]string{"id": uploadID}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) remoteDlRemove(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) + if uploadID == "" { + // Try to get from data + if data, ok := args.Data.(map[string]interface{}); ok { + if id, ok := data["id"].(string); ok { + uploadID = id + } + } + } + if uploadID == "" { + return nil, fmt.Errorf("remote upload ID required") + } + + if err := d.callAPI(ctx, "/remotedl/remove", map[string]string{"id": uploadID}, nil); err != nil { + return nil, err + } + return true, nil +} + +func (d *Streamtape) fileInfo(ctx context.Context, args model.OtherArgs) (interface{}, error) { + var fileIDs string + if data, ok := args.Data.(map[string]interface{}); ok { + if ids, ok := data["file_ids"].(string); ok { + fileIDs = ids + } + } + if fileIDs == "" { + fileIDs = fileIDFromObjID(args.Obj.GetID()) + } + if fileIDs == "" { + return nil, fmt.Errorf("file IDs required") + } + + var result fileInfoResult + if err := d.callAPI(ctx, "/file/info", map[string]string{"file": fileIDs}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) thumbnail(ctx context.Context, args model.OtherArgs) (interface{}, error) { + fileID := fileIDFromObjID(args.Obj.GetID()) + if fileID == "" { + return nil, fmt.Errorf("file ID required") + } + + var result string + if err := d.callAPI(ctx, "/file/getsplash", map[string]string{"file": fileID}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) conversionStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + isFailed := false + if data, ok := args.Data.(map[string]interface{}); ok { + if t, ok := data["type"].(string); ok && t == "failed" { + isFailed = true + } + } + + endpoint := "/file/runningconverts" + if isFailed { + endpoint = "/file/failedconverts" + } + + var result conversionResult + if err := d.callAPI(ctx, endpoint, nil, &result); err != nil { + return nil, err + } + return result, nil +} + var _ driver.Driver = (*Streamtape)(nil) From 24aaf0ce2c5642b1e5154e429ad68bc50418e3c5 Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:13:59 +0800 Subject: [PATCH 619/659] refactor(driver/streamtape): extract remote upload ID helper to reduce duplication --- drivers/streamtape/driver.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go index b261f86100e..f632d22f31f 100644 --- a/drivers/streamtape/driver.go +++ b/drivers/streamtape/driver.go @@ -443,10 +443,9 @@ func (d *Streamtape) Other(ctx context.Context, args model.OtherArgs) (interface } } -func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { +func (d *Streamtape) extractRemoteUploadID(args model.OtherArgs) (string, error) { uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) if uploadID == "" { - // Try to get from data if data, ok := args.Data.(map[string]interface{}); ok { if id, ok := data["id"].(string); ok { uploadID = id @@ -454,7 +453,15 @@ func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) ( } } if uploadID == "" { - return nil, fmt.Errorf("remote upload ID required") + return "", fmt.Errorf("remote upload ID required") + } + return uploadID, nil +} + +func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID, err := d.extractRemoteUploadID(args) + if err != nil { + return nil, err } var result remoteDlStatusResult @@ -465,17 +472,9 @@ func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) ( } func (d *Streamtape) remoteDlRemove(ctx context.Context, args model.OtherArgs) (interface{}, error) { - uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) - if uploadID == "" { - // Try to get from data - if data, ok := args.Data.(map[string]interface{}); ok { - if id, ok := data["id"].(string); ok { - uploadID = id - } - } - } - if uploadID == "" { - return nil, fmt.Errorf("remote upload ID required") + uploadID, err := d.extractRemoteUploadID(args) + if err != nil { + return nil, err } if err := d.callAPI(ctx, "/remotedl/remove", map[string]string{"id": uploadID}, nil); err != nil { From 066386fac4cb53d75e0513d79f7dbf03aec39fdd Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:15:40 +0800 Subject: [PATCH 620/659] feat(driver/streamtape): add Sha256 upload support and move-to-root alert --- drivers/streamtape/driver.go | 3 +++ drivers/streamtape/meta.go | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go index f632d22f31f..9ce32e25b69 100644 --- a/drivers/streamtape/driver.go +++ b/drivers/streamtape/driver.go @@ -331,6 +331,9 @@ func (d *Streamtape) Put(ctx context.Context, dstDir model.Obj, file model.FileS if folderID != "" && folderID != "0" { params["folder"] = folderID } + if d.Sha256 != "" { + params["sha256"] = d.Sha256 + } var uploadURL uploadURLResult if err := d.callAPI(ctx, "/file/ul", params, &uploadURL); err != nil { diff --git a/drivers/streamtape/meta.go b/drivers/streamtape/meta.go index 50b09f14d41..50dd90291ef 100644 --- a/drivers/streamtape/meta.go +++ b/drivers/streamtape/meta.go @@ -14,6 +14,7 @@ type Addition struct { RangeConcurrency int `json:"range_concurrency" type:"number" default:"4" help:"Chunk mode concurrent upstream requests"` RangePercent int `json:"range_percent" type:"number" default:"15" help:"Percent mode part size percentage (1-100)"` EnableRangeControl bool `json:"enable_range_control" default:"true" help:"Enable driver-level range shaping for smoother streaming"` + Sha256 string `json:"sha256" help:"Expected SHA256 hash for upload verification (optional)"` } var config = driver.Config{ @@ -26,7 +27,7 @@ var config = driver.Config{ NeedMs: false, DefaultRoot: "0", CheckStatus: false, - Alert: "", + Alert: "Moving files to root folder is not supported by Streamtape API", NoOverwriteUpload: false, ProxyRangeOption: true, } From 9dd6823391080e9b67c69c084d3089fc636e49bf Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Thu, 26 Mar 2026 23:17:32 +0800 Subject: [PATCH 621/659] fix(driver/streamtape): add warning type prefix to Alert message --- drivers/streamtape/meta.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/streamtape/meta.go b/drivers/streamtape/meta.go index 50dd90291ef..55e97894dbe 100644 --- a/drivers/streamtape/meta.go +++ b/drivers/streamtape/meta.go @@ -27,7 +27,7 @@ var config = driver.Config{ NeedMs: false, DefaultRoot: "0", CheckStatus: false, - Alert: "Moving files to root folder is not supported by Streamtape API", + Alert: "warning|Moving files to root folder is not supported by Streamtape API", NoOverwriteUpload: false, ProxyRangeOption: true, } From e3f3fc40f026186cc91b83c11ac23f0d2627d6a8 Mon Sep 17 00:00:00 2001 From: Liam Wang Date: Sat, 28 Mar 2026 13:57:40 +0800 Subject: [PATCH 622/659] test(streamtape): add comprehensive unit tests for Streamtape driver - Initialize RestyClient with custom settings for testing - Add TestDriverList to verify listing folders and files - Add TestDriverPutURL to test remote upload functionality and status checks - Add TestDriverFileInfo to validate file_info method via Other call - Add TestDriverThumbnail to test thumbnail retrieval via Other call - Add TestDriverConversionStatus to test conversion status queries (running and failed) --- drivers/streamtape/api_test.go | 224 +++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 drivers/streamtape/api_test.go diff --git a/drivers/streamtape/api_test.go b/drivers/streamtape/api_test.go new file mode 100644 index 00000000000..d12fc1320ab --- /dev/null +++ b/drivers/streamtape/api_test.go @@ -0,0 +1,224 @@ +package streamtape + +import ( + "context" + "crypto/tls" + "testing" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" +) + +const ( + testLogin = "" + testKey = "" +) + +func init() { + // Initialize RestyClient for testing + base.RestyClient = resty.New(). + SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). + SetRetryCount(3). + SetRetryResetReaders(true). + SetTimeout(30 * time.Second). + SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) +} + +func newTestDriver() *Streamtape { + return &Streamtape{ + Addition: Addition{ + APILogin: testLogin, + APIKey: testKey, + }, + } +} + +// TestDriverList tests List method +func TestDriverList(t *testing.T) { + d := newTestDriver() + d.RootID.RootFolderID = "0" + + ctx := context.Background() + objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("List failed: %v", err) + } + t.Logf("List returned %d objects", len(objs)) + for _, obj := range objs { + t.Logf(" - %s (folder=%v)", obj.GetName(), obj.IsDir()) + } +} + +// TestDriverPutURL tests PutURL method - remote upload +func TestDriverPutURL(t *testing.T) { + d := newTestDriver() + d.RootID.RootFolderID = "0" + + t.Logf("Driver initialized: APILogin=%s, APIKey=%s, RootFolderID=%s", + d.APILogin, d.APIKey, d.RootID.RootFolderID) + + ctx := context.Background() + // Pass a valid root directory object instead of nil + rootDir := &model.Object{ID: "d:0", IsFolder: true} + obj, err := d.PutURL(ctx, rootDir, "test.txt", "https://example.com/test.txt") + if err != nil { + t.Fatalf("PutURL failed: %v", err) + } + t.Logf("PutURL returned: id=%s name=%s", obj.GetID(), obj.GetName()) + + // Extract upload ID + uploadID := remoteUploadIDFromObjID(obj.GetID()) + if uploadID == "" { + t.Fatal("PutURL returned invalid ID format") + } + t.Logf("Upload ID: %s", uploadID) + + // Pass Obj with the upload ID so extractRemoteUploadID can work + uploadObj := &model.Object{ID: obj.GetID()} + + // Test remotedl_status via Other method + statusResult, err := d.Other(ctx, model.OtherArgs{ + Obj: uploadObj, + Method: "remotedl_status", + Data: map[string]interface{}{"id": uploadID}, + }) + if err != nil { + t.Fatalf("remotedl_status failed: %v", err) + } + t.Logf("remotedl_status: %+v", statusResult) + + // Test remotedl_remove via Other method + removeResult, err := d.Other(ctx, model.OtherArgs{ + Obj: uploadObj, + Method: "remotedl_remove", + Data: map[string]interface{}{"id": uploadID}, + }) + if err != nil { + t.Fatalf("remotedl_remove failed: %v", err) + } + t.Logf("remotedl_remove: %v", removeResult) +} + +// TestDriverFileInfo tests file_info via Other method +func TestDriverFileInfo(t *testing.T) { + d := newTestDriver() + + // First get a file ID from list + ctx := context.Background() + objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("List failed: %v", err) + } + + // Find a file in subfolders + var fileID string + var fileName string + for _, obj := range objs { + if obj.IsDir() { + subObjs, err := d.List(ctx, obj, model.ListArgs{}) + if err != nil { + continue + } + for _, subObj := range subObjs { + if !subObj.IsDir() { + fileID = subObj.GetID() + fileName = subObj.GetName() + break + } + } + if fileID != "" { + break + } + } + } + + if fileID == "" { + t.Skip("No files found for file_info test") + } + t.Logf("Testing file_info with file: %s (id=%s)", fileName, fileID) + + // Test file_info via Other method + infoResult, err := d.Other(ctx, model.OtherArgs{ + Method: "file_info", + Obj: &model.Object{ID: fileID}, + }) + if err != nil { + t.Fatalf("file_info failed: %v", err) + } + t.Logf("file_info: %+v", infoResult) +} + +// TestDriverThumbnail tests thumbnail via Other method +func TestDriverThumbnail(t *testing.T) { + d := newTestDriver() + + // First get a file ID from list + ctx := context.Background() + objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) + if err != nil { + t.Fatalf("List failed: %v", err) + } + + // Find a file in subfolders + var fileID string + for _, obj := range objs { + if obj.IsDir() { + subObjs, err := d.List(ctx, obj, model.ListArgs{}) + if err != nil { + continue + } + for _, subObj := range subObjs { + if !subObj.IsDir() { + fileID = subObj.GetID() + break + } + } + if fileID != "" { + break + } + } + } + + if fileID == "" { + t.Skip("No files found for thumbnail test") + } + t.Logf("Testing thumbnail with file id=%s", fileID) + + // Test thumbnail via Other method + thumbResult, err := d.Other(ctx, model.OtherArgs{ + Method: "thumbnail", + Obj: &model.Object{ID: fileID}, + }) + if err != nil { + t.Fatalf("thumbnail failed: %v", err) + } + t.Logf("thumbnail: %v", thumbResult) +} + +// TestDriverConversionStatus tests conversion_status via Other method +func TestDriverConversionStatus(t *testing.T) { + d := newTestDriver() + + ctx := context.Background() + + // Test running conversions + runningResult, err := d.Other(ctx, model.OtherArgs{ + Method: "conversion_status", + }) + if err != nil { + t.Fatalf("conversion_status (running) failed: %v", err) + } + t.Logf("conversion_status (running): %+v", runningResult) + + // Test failed conversions + failedResult, err := d.Other(ctx, model.OtherArgs{ + Method: "conversion_status", + Data: map[string]interface{}{"type": "failed"}, + }) + if err != nil { + t.Fatalf("conversion_status (failed) failed: %v", err) + } + t.Logf("conversion_status (failed): %+v", failedResult) +} \ No newline at end of file From 1d05bdb0d575d845eb1096ad8a5aa353f6628a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Sat, 28 Mar 2026 14:45:30 +0800 Subject: [PATCH 623/659] chore(streamtape): delete drivers/streamtape/api_test.go --- drivers/streamtape/api_test.go | 224 --------------------------------- 1 file changed, 224 deletions(-) delete mode 100644 drivers/streamtape/api_test.go diff --git a/drivers/streamtape/api_test.go b/drivers/streamtape/api_test.go deleted file mode 100644 index d12fc1320ab..00000000000 --- a/drivers/streamtape/api_test.go +++ /dev/null @@ -1,224 +0,0 @@ -package streamtape - -import ( - "context" - "crypto/tls" - "testing" - "time" - - "github.com/alist-org/alist/v3/drivers/base" - "github.com/alist-org/alist/v3/internal/model" - "github.com/go-resty/resty/v2" -) - -const ( - testLogin = "" - testKey = "" -) - -func init() { - // Initialize RestyClient for testing - base.RestyClient = resty.New(). - SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"). - SetRetryCount(3). - SetRetryResetReaders(true). - SetTimeout(30 * time.Second). - SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) -} - -func newTestDriver() *Streamtape { - return &Streamtape{ - Addition: Addition{ - APILogin: testLogin, - APIKey: testKey, - }, - } -} - -// TestDriverList tests List method -func TestDriverList(t *testing.T) { - d := newTestDriver() - d.RootID.RootFolderID = "0" - - ctx := context.Background() - objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) - if err != nil { - t.Fatalf("List failed: %v", err) - } - t.Logf("List returned %d objects", len(objs)) - for _, obj := range objs { - t.Logf(" - %s (folder=%v)", obj.GetName(), obj.IsDir()) - } -} - -// TestDriverPutURL tests PutURL method - remote upload -func TestDriverPutURL(t *testing.T) { - d := newTestDriver() - d.RootID.RootFolderID = "0" - - t.Logf("Driver initialized: APILogin=%s, APIKey=%s, RootFolderID=%s", - d.APILogin, d.APIKey, d.RootID.RootFolderID) - - ctx := context.Background() - // Pass a valid root directory object instead of nil - rootDir := &model.Object{ID: "d:0", IsFolder: true} - obj, err := d.PutURL(ctx, rootDir, "test.txt", "https://example.com/test.txt") - if err != nil { - t.Fatalf("PutURL failed: %v", err) - } - t.Logf("PutURL returned: id=%s name=%s", obj.GetID(), obj.GetName()) - - // Extract upload ID - uploadID := remoteUploadIDFromObjID(obj.GetID()) - if uploadID == "" { - t.Fatal("PutURL returned invalid ID format") - } - t.Logf("Upload ID: %s", uploadID) - - // Pass Obj with the upload ID so extractRemoteUploadID can work - uploadObj := &model.Object{ID: obj.GetID()} - - // Test remotedl_status via Other method - statusResult, err := d.Other(ctx, model.OtherArgs{ - Obj: uploadObj, - Method: "remotedl_status", - Data: map[string]interface{}{"id": uploadID}, - }) - if err != nil { - t.Fatalf("remotedl_status failed: %v", err) - } - t.Logf("remotedl_status: %+v", statusResult) - - // Test remotedl_remove via Other method - removeResult, err := d.Other(ctx, model.OtherArgs{ - Obj: uploadObj, - Method: "remotedl_remove", - Data: map[string]interface{}{"id": uploadID}, - }) - if err != nil { - t.Fatalf("remotedl_remove failed: %v", err) - } - t.Logf("remotedl_remove: %v", removeResult) -} - -// TestDriverFileInfo tests file_info via Other method -func TestDriverFileInfo(t *testing.T) { - d := newTestDriver() - - // First get a file ID from list - ctx := context.Background() - objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) - if err != nil { - t.Fatalf("List failed: %v", err) - } - - // Find a file in subfolders - var fileID string - var fileName string - for _, obj := range objs { - if obj.IsDir() { - subObjs, err := d.List(ctx, obj, model.ListArgs{}) - if err != nil { - continue - } - for _, subObj := range subObjs { - if !subObj.IsDir() { - fileID = subObj.GetID() - fileName = subObj.GetName() - break - } - } - if fileID != "" { - break - } - } - } - - if fileID == "" { - t.Skip("No files found for file_info test") - } - t.Logf("Testing file_info with file: %s (id=%s)", fileName, fileID) - - // Test file_info via Other method - infoResult, err := d.Other(ctx, model.OtherArgs{ - Method: "file_info", - Obj: &model.Object{ID: fileID}, - }) - if err != nil { - t.Fatalf("file_info failed: %v", err) - } - t.Logf("file_info: %+v", infoResult) -} - -// TestDriverThumbnail tests thumbnail via Other method -func TestDriverThumbnail(t *testing.T) { - d := newTestDriver() - - // First get a file ID from list - ctx := context.Background() - objs, err := d.List(ctx, &model.Object{ID: "0", IsFolder: true}, model.ListArgs{}) - if err != nil { - t.Fatalf("List failed: %v", err) - } - - // Find a file in subfolders - var fileID string - for _, obj := range objs { - if obj.IsDir() { - subObjs, err := d.List(ctx, obj, model.ListArgs{}) - if err != nil { - continue - } - for _, subObj := range subObjs { - if !subObj.IsDir() { - fileID = subObj.GetID() - break - } - } - if fileID != "" { - break - } - } - } - - if fileID == "" { - t.Skip("No files found for thumbnail test") - } - t.Logf("Testing thumbnail with file id=%s", fileID) - - // Test thumbnail via Other method - thumbResult, err := d.Other(ctx, model.OtherArgs{ - Method: "thumbnail", - Obj: &model.Object{ID: fileID}, - }) - if err != nil { - t.Fatalf("thumbnail failed: %v", err) - } - t.Logf("thumbnail: %v", thumbResult) -} - -// TestDriverConversionStatus tests conversion_status via Other method -func TestDriverConversionStatus(t *testing.T) { - d := newTestDriver() - - ctx := context.Background() - - // Test running conversions - runningResult, err := d.Other(ctx, model.OtherArgs{ - Method: "conversion_status", - }) - if err != nil { - t.Fatalf("conversion_status (running) failed: %v", err) - } - t.Logf("conversion_status (running): %+v", runningResult) - - // Test failed conversions - failedResult, err := d.Other(ctx, model.OtherArgs{ - Method: "conversion_status", - Data: map[string]interface{}{"type": "failed"}, - }) - if err != nil { - t.Fatalf("conversion_status (failed) failed: %v", err) - } - t.Logf("conversion_status (failed): %+v", failedResult) -} \ No newline at end of file From a0545cb4366ce62060bfee402ea149a98a757883 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Sat, 28 Mar 2026 22:46:33 +0800 Subject: [PATCH 624/659] chore(deps): bump github.com/shoenig/go-m1cpu to v0.2.1 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index bc2475c548d..17c09201ab0 100644 --- a/go.mod +++ b/go.mod @@ -250,7 +250,7 @@ require ( github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/shirou/gopsutil/v3 v3.24.4 // indirect - github.com/shoenig/go-m1cpu v0.2.0 // indirect + github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index e6b04779ede..370ac6d206f 100644 --- a/go.sum +++ b/go.sum @@ -591,6 +591,8 @@ github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFt github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/go-m1cpu v0.2.0 h1:t4GNqvPZ84Vjtpboo/kT3pIkbaK3vc+JIlD/Wz1zSFY= github.com/shoenig/go-m1cpu v0.2.0/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= +github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= +github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= From c4595e59f3ecd81ad508c5c310a1f15d6ff09b7f Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Mon, 30 Mar 2026 10:00:03 +0800 Subject: [PATCH 625/659] Fix share access accounting and validation --- internal/db/share.go | 12 ++++---- server/handles/share.go | 55 +++++++++++++++++++++++++++++----- server/handles/share_public.go | 27 ++++++++--------- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/internal/db/share.go b/internal/db/share.go index c92e546755e..711c3f278c3 100644 --- a/internal/db/share.go +++ b/internal/db/share.go @@ -24,20 +24,20 @@ func GetShareByCreatorAndShareID(creatorID uint, shareID string) (*model.Share, return &share, nil } -func ShareIDExists(shareID string) bool { +func ShareIDExists(shareID string) (bool, error) { var count int64 if err := db.Model(&model.Share{}).Where("share_id = ?", shareID).Count(&count).Error; err != nil { - return false + return false, err } - return count > 0 + return count > 0, nil } -func ShareIDExistsExceptID(shareID string, id uint) bool { +func ShareIDExistsExceptID(shareID string, id uint) (bool, error) { var count int64 if err := db.Model(&model.Share{}).Where("share_id = ? AND id <> ?", shareID, id).Count(&count).Error; err != nil { - return false + return false, err } - return count > 0 + return count > 0, nil } func CreateShare(share *model.Share) error { diff --git a/server/handles/share.go b/server/handles/share.go index 99a693b7a5f..4de7da63672 100644 --- a/server/handles/share.go +++ b/server/handles/share.go @@ -2,7 +2,9 @@ package handles import ( "crypto/subtle" + "errors" "fmt" + "net/http" "net/url" stdpath "path" "regexp" @@ -24,6 +26,11 @@ const shareAccessTokenLifetime = 24 * time.Hour var shareIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,32}$`) +var ( + errShareIDInvalid = errors.New("share_id must be 1-32 characters of letters, numbers, underscore or hyphen") + errShareIDExists = errors.New("share link already exists") +) + type CreateShareReq struct { Path string `json:"path" binding:"required"` ShareID string `json:"share_id"` @@ -184,7 +191,11 @@ func normalizeOptionalShareName(name, fallback string) string { func generateShareID() (string, error) { for range 10 { shareID := random.String(8) - if !db.ShareIDExists(shareID) { + exists, err := db.ShareIDExists(shareID) + if err != nil { + return "", err + } + if !exists { return shareID, nil } } @@ -200,7 +211,7 @@ func validateCustomShareID(shareID string) error { return nil } if !shareIDPattern.MatchString(shareID) { - return fmt.Errorf("share_id must be 1-32 characters of letters, numbers, underscore or hyphen") + return errShareIDInvalid } return nil } @@ -217,13 +228,21 @@ func resolveRequestedShareID(rawShareID, fallback string, excludeID uint) (strin return "", err } if excludeID == 0 { - if db.ShareIDExists(shareID) { - return "", fmt.Errorf("share link already exists") + exists, err := db.ShareIDExists(shareID) + if err != nil { + return "", fmt.Errorf("check share id availability: %w", err) + } + if exists { + return "", errShareIDExists } return shareID, nil } - if db.ShareIDExistsExceptID(shareID, excludeID) { - return "", fmt.Errorf("share link already exists") + exists, err := db.ShareIDExistsExceptID(shareID, excludeID) + if err != nil { + return "", fmt.Errorf("check share id availability: %w", err) + } + if exists { + return "", errShareIDExists } return shareID, nil } @@ -334,6 +353,10 @@ func ensureShareAccess(c *gin.Context, share *model.Share, token string) bool { return true } +func shouldTrackShareContentAccess(c *gin.Context) bool { + return c.Request.Method != http.MethodHead +} + func resolveShareTarget(share *model.Share, rawRelPath string) (string, string, error) { cleanRelPath := utils.FixAndCleanPath(rawRelPath) if !share.IsDir && cleanRelPath != "/" { @@ -349,6 +372,14 @@ func resolveShareTarget(share *model.Share, rawRelPath string) (string, string, return target, cleanRelPath, nil } +func resolveShareWildcardTarget(share *model.Share, rawPath string) (string, string, error) { + path, err := url.PathUnescape(rawPath) + if err != nil { + return "", "", err + } + return resolveShareTarget(share, strings.TrimPrefix(path, "/")) +} + func buildPublicShareAssetURL(c *gin.Context, prefix, shareID, relPath, token string, preview bool) string { base := common.GetApiUrl(c.Request) + prefix + shareID cleanPath := utils.FixAndCleanPath(relPath) @@ -423,7 +454,11 @@ func CreateShare(c *gin.Context) { } shareID, err := resolveRequestedShareID(req.ShareID, "", 0) if err != nil { - common.ErrorResp(c, err, 400) + if errors.Is(err, errShareIDInvalid) || errors.Is(err, errShareIDExists) { + common.ErrorResp(c, err, 400) + return + } + common.ErrorResp(c, err, 500, true) return } allowPreview := true @@ -483,7 +518,11 @@ func UpdateShare(c *gin.Context) { shareID, err := resolveRequestedShareID(req.NewShareID, share.ShareID, share.ID) if err != nil { - common.ErrorResp(c, err, 400) + if errors.Is(err, errShareIDInvalid) || errors.Is(err, errShareIDExists) { + common.ErrorResp(c, err, 400) + return + } + common.ErrorResp(c, err, 500, true) return } diff --git a/server/handles/share_public.go b/server/handles/share_public.go index 22aba45bc4c..83ca79f3591 100644 --- a/server/handles/share_public.go +++ b/server/handles/share_public.go @@ -2,7 +2,6 @@ package handles import ( stdpath "path" - "strings" "time" "github.com/alist-org/alist/v3/internal/db" @@ -143,10 +142,6 @@ func ListPublicShare(c *gin.Context) { } content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) } - if err := recordShareAccess(share); err != nil { - common.ErrorResp(c, err, 500, true) - return - } common.SuccessResp(c, PublicShareListResp{ Content: content, Total: int64(total), @@ -213,7 +208,7 @@ func ShareDown(c *gin.Context) { if !ensureShareAccess(c, share, token) { return } - targetPath, _, err := resolveShareTarget(share, strings.TrimPrefix(c.Param("path"), "/")) + targetPath, _, err := resolveShareWildcardTarget(share, c.Param("path")) if err != nil { common.ErrorResp(c, err, 400) return @@ -227,10 +222,12 @@ func ShareDown(c *gin.Context) { common.ErrorStrResp(c, "directory download is not supported", 400) return } - _ = db.TouchShareDownload(share.ShareID) - if err := recordShareAccess(share); err != nil { - common.ErrorResp(c, err, 500, true) - return + if shouldTrackShareContentAccess(c) { + _ = db.TouchShareDownload(share.ShareID) + if err := recordShareAccess(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } } c.Set("path", targetPath) Down(c) @@ -253,7 +250,7 @@ func ShareProxy(c *gin.Context) { if !ensureShareAccess(c, share, token) { return } - targetPath, _, err := resolveShareTarget(share, strings.TrimPrefix(c.Param("path"), "/")) + targetPath, _, err := resolveShareWildcardTarget(share, c.Param("path")) if err != nil { common.ErrorResp(c, err, 400) return @@ -267,9 +264,11 @@ func ShareProxy(c *gin.Context) { common.ErrorStrResp(c, "directory preview is not supported", 400) return } - if err := recordShareAccess(share); err != nil { - common.ErrorResp(c, err, 500, true) - return + if shouldTrackShareContentAccess(c) { + if err := recordShareAccess(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } } c.Set("path", targetPath) Proxy(c) From 71898a4443f8217f62d29ea0f735b84848f3cc42 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 31 Mar 2026 16:27:12 +0800 Subject: [PATCH 626/659] feat: add 360AI YunPan driver --- drivers/all.go | 1 + drivers/yunpan360/driver.go | 286 ++++++++++ drivers/yunpan360/meta.go | 34 ++ drivers/yunpan360/types.go | 462 +++++++++++++++ drivers/yunpan360/upload.go | 926 +++++++++++++++++++++++++++++++ drivers/yunpan360/upload_test.go | 25 + drivers/yunpan360/util.go | 909 ++++++++++++++++++++++++++++++ drivers/yunpan360/util_test.go | 109 ++++ 8 files changed, 2752 insertions(+) create mode 100644 drivers/yunpan360/driver.go create mode 100644 drivers/yunpan360/meta.go create mode 100644 drivers/yunpan360/types.go create mode 100644 drivers/yunpan360/upload.go create mode 100644 drivers/yunpan360/upload_test.go create mode 100644 drivers/yunpan360/util.go create mode 100644 drivers/yunpan360/util_test.go diff --git a/drivers/all.go b/drivers/all.go index a4fce9d0cac..ef872771961 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -83,6 +83,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/wopan" _ "github.com/alist-org/alist/v3/drivers/wukong" _ "github.com/alist-org/alist/v3/drivers/yandex_disk" + _ "github.com/alist-org/alist/v3/drivers/yunpan360" ) // All do nothing,just for import diff --git a/drivers/yunpan360/driver.go b/drivers/yunpan360/driver.go new file mode 100644 index 00000000000..c92e918a0ad --- /dev/null +++ b/drivers/yunpan360/driver.go @@ -0,0 +1,286 @@ +package yunpan360 + +import ( + "context" + "errors" + stdpath "path" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Yunpan360 struct { + model.Storage + Addition + + authMu sync.Mutex + cachedOpenAuth *OpenAuthInfo + openAuthExpire time.Time + + cachedCookieSession *CookieDownloadSession + cookieSessionExpire time.Time +} + +func (d *Yunpan360) Config() driver.Config { + return config +} + +func (d *Yunpan360) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Yunpan360) Init(ctx context.Context) error { + if d.PageSize <= 0 { + d.PageSize = 100 + } + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + if d.RootFolderPath == "" { + d.RootFolderPath = "/" + } + d.OrderDirection = strings.ToLower(strings.TrimSpace(d.OrderDirection)) + if d.OrderDirection != "desc" { + d.OrderDirection = "asc" + } + d.AuthType = strings.ToLower(strings.TrimSpace(d.AuthType)) + if d.AuthType == "" { + d.AuthType = authTypeCookie + } + d.SubChannel = strings.TrimSpace(d.SubChannel) + if d.SubChannel == "" { + d.SubChannel = defaultSubChannel + } + d.EcsEnv = strings.ToLower(strings.TrimSpace(d.EcsEnv)) + if d.EcsEnv == "" { + d.EcsEnv = openEnvProd + } + d.Cookie = strings.TrimSpace(d.Cookie) + d.APIKey = strings.TrimSpace(d.APIKey) + d.OwnerQID = strings.TrimSpace(d.OwnerQID) + d.DownloadToken = strings.TrimSpace(d.DownloadToken) + d.cachedOpenAuth = nil + d.openAuthExpire = time.Time{} + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} + + switch d.authMode() { + case authTypeAPIKey: + if d.APIKey == "" { + return errors.New("api_key is empty") + } + _, err := d.openUserInfo(ctx) + return err + case authTypeCookie: + if d.Cookie == "" { + return errors.New("cookie is empty") + } + // Web download URLs require browser-session headers; force local proxying + // so AList can forward Referer/Origin instead of exposing a bare 302 URL. + d.WebProxy = true + _, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + return err + default: + return errors.New("invalid auth_type") + } +} + +func (d *Yunpan360) Drop(ctx context.Context) error { + d.cachedOpenAuth = nil + d.openAuthExpire = time.Time{} + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} + return nil +} + +func (d *Yunpan360) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + dirPath := dir.GetPath() + if dirPath == "" { + dirPath = d.RootFolderPath + } + + objs := make([]model.Obj, 0, d.PageSize) + for page := 0; ; page++ { + resp, err := d.listPage(ctx, dirPath, page, d.PageSize) + if err != nil { + return nil, err + } + pageObjs := resp.Objects(dirPath) + for _, item := range pageObjs { + objs = append(objs, item) + } + if len(pageObjs) == 0 { + break + } + if d.authMode() == authTypeAPIKey { + if len(pageObjs) < d.PageSize { + break + } + continue + } + if !resp.GetHasNextPage() { + break + } + } + return objs, nil +} + +func (d *Yunpan360) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if d.authMode() == authTypeCookie { + resp, err := d.cookieDownloadURL(ctx, file) + if err != nil { + return nil, err + } + downloadURL := strings.TrimSpace(resp.GetURL()) + if downloadURL == "" { + return nil, errors.New("download url is empty") + } + return &model.Link{ + URL: downloadURL, + Header: map[string][]string{ + "Accept": {"text/javascript, text/html, application/xml, text/xml, */*"}, + "Origin": {baseURL}, + "Referer": {baseURL + indexPath}, + }, + }, nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + resp, err := d.openDownloadURL(ctx, file) + if err != nil { + return nil, err + } + downloadURL := strings.TrimSpace(resp.GetURL()) + if downloadURL == "" { + return nil, errors.New("download url is empty") + } + return &model.Link{URL: downloadURL}, nil +} + +func (d *Yunpan360) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.authMode() == authTypeCookie { + fullPath := ensureDirAPIPath(stdpath.Join(parentDir.GetPath(), dirName)) + resp, err := d.cookieMakeDir(ctx, fullPath) + if err != nil { + return nil, err + } + return &YunpanObject{ + Object: model.Object{ + ID: resp.Data.NID, + Path: normalizeRemotePath(fullPath), + Name: dirName, + Size: 0, + Modified: time.Now(), + IsFolder: true, + }, + }, nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + fullPath := ensureDirAPIPath(stdpath.Join(parentDir.GetPath(), dirName)) + resp, err := d.openMakeDir(ctx, fullPath) + if err != nil { + return nil, err + } + obj := &model.Object{ + ID: resp.Data.NID, + Path: normalizeRemotePath(fullPath), + Name: dirName, + Size: 0, + Modified: time.Now(), + IsFolder: true, + } + return obj, nil +} + +func (d *Yunpan360) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.authMode() == authTypeCookie { + srcPath := apiPathForObj(srcObj) + dstPath := ensureDirAPIPath(dstDir.GetPath()) + if err := d.cookieMove(ctx, srcPath, dstPath); err != nil { + return nil, err + } + return cloneObj(srcObj, stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetName()), nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + srcPath := apiPathForObj(srcObj) + dstPath := ensureDirAPIPath(dstDir.GetPath()) + if err := d.openMove(ctx, srcPath, dstPath); err != nil { + return nil, err + } + return cloneObj(srcObj, stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetName()), nil +} + +func (d *Yunpan360) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.authMode() == authTypeCookie { + targetName := strings.TrimSuffix(strings.TrimSpace(newName), "/") + if targetName == "" { + return nil, errors.New("new name is empty") + } + if err := d.cookieRename(ctx, srcObj, targetName); err != nil { + return nil, err + } + parentPath := stdpath.Dir(srcObj.GetPath()) + if parentPath == "." { + parentPath = "/" + } + return cloneObj(srcObj, stdpath.Join(parentPath, targetName), targetName), nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + srcPath := apiPathForObj(srcObj) + targetName := newName + if srcObj.IsDir() { + targetName = ensureDirSuffix(newName) + } + if err := d.openRename(ctx, srcPath, targetName); err != nil { + return nil, err + } + + parentPath := stdpath.Dir(srcObj.GetPath()) + if parentPath == "." { + parentPath = "/" + } + return cloneObj(srcObj, stdpath.Join(parentPath, strings.TrimSuffix(newName, "/")), strings.TrimSuffix(newName, "/")), nil +} + +func (d *Yunpan360) Remove(ctx context.Context, obj model.Obj) error { + if d.authMode() == authTypeCookie { + return d.cookieRecycle(ctx, obj) + } + if d.authMode() != authTypeAPIKey { + return errs.NotImplement + } + return d.openDelete(ctx, apiPathForObj(obj)) +} + +func (d *Yunpan360) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.authMode() == authTypeCookie { + return nil, errs.NotImplement + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + return d.putOpenFile(ctx, dstDir, file, up) +} + +func (d *Yunpan360) authMode() string { + if d.AuthType == authTypeAPIKey { + return authTypeAPIKey + } + return authTypeCookie +} + +var _ driver.Driver = (*Yunpan360)(nil) diff --git a/drivers/yunpan360/meta.go b/drivers/yunpan360/meta.go new file mode 100644 index 00000000000..5c4e1c8d245 --- /dev/null +++ b/drivers/yunpan360/meta.go @@ -0,0 +1,34 @@ +package yunpan360 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + AuthType string `json:"auth_type" type:"select" options:"cookie,api_key" default:"cookie"` + Cookie string `json:"cookie" type:"text" help:"Cookie copied from a logged-in yunpan.com session; used when auth_type=cookie"` + OwnerQID string `json:"owner_qid" type:"text" help:"Optional owner_qid for cookie-mode download; leave empty to auto-detect"` + DownloadToken string `json:"download_token" type:"text" help:"Optional web token for cookie-mode download; leave empty to auto-detect"` + APIKey string `json:"api_key" type:"text" help:"360 AI YunPan API key; used when auth_type=api_key"` + EcsEnv string `json:"ecs_env" type:"select" options:"prod,test,hgtest" default:"prod"` + SubChannel string `json:"sub_channel" default:"open"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + PageSize int `json:"page_size" type:"number" default:"100" help:"List page size"` +} + +var config = driver.Config{ + Name: "360AIYunPan", + LocalSort: false, + CheckStatus: true, + NoUpload: false, + DefaultRoot: "/", + Alert: "info|api_key mode supports list/link/upload/mkdir/rename/move/delete; cookie mode supports list/link/mkdir/rename/move/delete only, and forces web proxy because direct download URLs require web headers.", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Yunpan360{} + }) +} diff --git a/drivers/yunpan360/types.go b/drivers/yunpan360/types.go new file mode 100644 index 00000000000..f5c1b7a2135 --- /dev/null +++ b/drivers/yunpan360/types.go @@ -0,0 +1,462 @@ +package yunpan360 + +import ( + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + authTypeCookie = "cookie" + authTypeAPIKey = "api_key" + openEnvProd = "prod" + defaultSubChannel = "open" + openSignSecret = "e7b24b112a44fdd9ee93bdf998c6ca0e" + openClientID = "e4757e933b6486c08ed206ecb6d5d9e684fcb4e2" + openClientSecret = "885fd3231f1c1e37c9f462261a09b8c38cde0c2b" + openClientSecretQA = "b11b8fff1c75a5d227c8cc93aaeb0bb70c8eee47" +) + +type BaseResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` +} + +type CookieDownloadSession struct { + OwnerQID string + Token string +} + +type ListResp interface { + Objects(parentPath string) []model.Obj + GetHasNextPage() bool +} + +type CookieListResp struct { + BaseResp + Token string `json:"token"` + OwnerQid string `json:"owner_qid"` + Qid string `json:"qid"` + Data []ListItem `json:"data"` + HasNextPage bool `json:"has_next_page"` +} + +func (r *CookieListResp) Objects(parentPath string) []model.Obj { + ownerQID := r.GetOwnerQID() + return utils.MustSliceConvert(r.Data, func(src ListItem) model.Obj { + return src.toObj(parentPath, ownerQID, r.Token) + }) +} + +func (r *CookieListResp) GetHasNextPage() bool { + return r.HasNextPage +} + +func (r *CookieListResp) GetOwnerQID() string { + return firstNonEmpty(r.OwnerQid, r.Qid) +} + +type ListItem struct { + NID string `json:"nid"` + FileName string `json:"file_name"` + FilePath string `json:"file_path"` + FileSize string `json:"file_size"` + IsDir bool `json:"is_dir"` + Fhash string `json:"fhash"` + CreateTime string `json:"create_time"` + ModifyTime string `json:"modify_time"` + Mtime string `json:"mtime"` + ServerTime string `json:"server_time"` + Preview string `json:"preview"` + Thumb string `json:"thumb"` + SrcPic string `json:"srcpic"` + OwnerQid string `json:"owner_qid"` + Qid string `json:"qid"` + Token string `json:"token"` +} + +func (i ListItem) toObj(parentPath, ownerQID, token string) model.Obj { + objPath := normalizeRemotePath(i.FilePath) + if objPath == "" || !pathLooksLikeObject(objPath, i.FileName) { + objPath = joinRemotePath(parentPath, i.FileName) + } + thumb := "" + if !i.IsDir { + thumb = absoluteURL(firstNonEmpty(i.Thumb, i.SrcPic, i.Preview)) + } + + return &YunpanObject{ + Object: model.Object{ + ID: i.NID, + Path: objPath, + Name: i.FileName, + Size: parseSize(i.FileSize), + Modified: parseYunpanTime(i.ModifyTime, i.Mtime), + Ctime: parseYunpanTime(i.CreateTime, i.ServerTime), + IsFolder: i.IsDir, + HashInfo: parseHash(i.Fhash), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + OwnerQID: firstNonEmpty(i.OwnerQid, i.Qid, ownerQID), + DownloadToken: firstNonEmpty(i.Token, token), + } +} + +func parseSize(raw string) int64 { + size, _ := strconv.ParseInt(raw, 10, 64) + return size +} + +func parseHash(raw string) utils.HashInfo { + if len(raw) == 40 { + return utils.NewHashInfo(utils.SHA1, raw) + } + return utils.HashInfo{} +} + +func parseYunpanTime(unixStr, text string) time.Time { + if t := parseUnixTime(unixStr); !t.IsZero() { + return t + } + return parseTextTime(text) +} + +func parseUnixTime(raw string) time.Time { + if raw != "" { + sec, err := strconv.ParseInt(raw, 10, 64) + if err == nil && sec > 0 { + return time.Unix(sec, 0) + } + } + return time.Time{} +} + +func parseTextTime(text string) time.Time { + if text == "" { + return time.Time{} + } + t, err := time.ParseInLocation("2006-01-02 15:04:05", text, utils.CNLoc) + if err == nil { + return t + } + return time.Time{} +} + +func normalizeRemotePath(p string) string { + if p == "" { + return "" + } + if p != "/" { + p = strings.TrimSuffix(p, "/") + } + return utils.FixAndCleanPath(p) +} + +type OpenAuthResp struct { + BaseResp + Data struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + AccessTokenExpire int64 `json:"access_token_expire"` + Qid string `json:"qid"` + } `json:"data"` +} + +type OpenAuthInfo struct { + AccessToken string + Qid string + Token string + SubChannel string +} + +type OpenListResp struct { + BaseResp + Data struct { + NodeList []OpenNode `json:"node_list"` + List []OpenNode `json:"list"` + Data []OpenNode `json:"data"` + TotalCount int `json:"total_count"` + Total int `json:"total"` + PageNum int `json:"page_num"` + } `json:"data"` +} + +func (r *OpenListResp) Objects(parentPath string) []model.Obj { + nodes := r.Data.NodeList + if len(nodes) == 0 { + nodes = r.Data.List + } + if len(nodes) == 0 { + nodes = r.Data.Data + } + return utils.MustSliceConvert(nodes, func(src OpenNode) model.Obj { + return src.toObj(parentPath) + }) +} + +func (r *OpenListResp) GetHasNextPage() bool { + total := r.Data.TotalCount + if total <= 0 { + total = r.Data.Total + } + if total <= 0 { + return false + } + loaded := len(r.Data.NodeList) + if loaded == 0 { + loaded = len(r.Data.List) + } + if loaded == 0 { + loaded = len(r.Data.Data) + } + return loaded > 0 && loaded < total +} + +type OpenNode struct { + NID string `json:"nid"` + Name string `json:"name"` + FName string `json:"fname"` + Path string `json:"path"` + FPath string `json:"fpath"` + Type interface{} `json:"type"` + IsDir interface{} `json:"is_dir"` + CountSize interface{} `json:"count_size"` + Size interface{} `json:"size"` + CreateTime interface{} `json:"create_time"` + ModifyTime interface{} `json:"modify_time"` + MTime interface{} `json:"mtime"` + FileHash string `json:"file_hash"` + Fhash string `json:"fhash"` + Thumb string `json:"thumb"` + Preview string `json:"preview"` + SrcPic string `json:"srcpic"` +} + +func (n OpenNode) toObj(parentPath string) model.Obj { + name := firstNonEmpty(strings.TrimSpace(n.Name), strings.TrimSpace(n.FName)) + objPath := normalizeRemotePath(firstNonEmpty(n.FPath, n.Path)) + if objPath == "" || !pathLooksLikeObject(objPath, name) { + objPath = joinRemotePath(parentPath, name) + } + isDir := parseOpenDir(n.IsDir, n.Type) + thumb := "" + if !isDir { + thumb = absoluteURL(firstNonEmpty(n.Thumb, n.SrcPic, n.Preview)) + } + + return &YunpanObject{ + Object: model.Object{ + ID: n.NID, + Path: objPath, + Name: name, + Size: parseAnySize(n.CountSize, n.Size), + Modified: parseAnyTime(n.ModifyTime, n.MTime), + Ctime: parseAnyTime(n.CreateTime), + IsFolder: isDir, + HashInfo: parseHash(firstNonEmpty(n.FileHash, n.Fhash)), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + } +} + +type OpenUserInfoResp struct { + BaseResp + Data map[string]interface{} `json:"data"` +} + +type OpenMkdirResp struct { + BaseResp + Data struct { + NID string `json:"nid"` + } `json:"data"` +} + +type CookieMkdirResp struct { + BaseResp + Data struct { + NID string `json:"nid"` + } `json:"data"` +} + +type CookieMoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + IsAsync bool `json:"is_async"` + } `json:"data"` +} + +type CookieRecycleResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + IsAsync bool `json:"is_async"` + } `json:"data"` +} + +type CookieAsyncQueryResp struct { + BaseResp + Data map[string]CookieAsyncTask `json:"data"` +} + +type CookieAsyncTask struct { + MessageID string `json:"message_id"` + SendTime string `json:"send_time"` + Status int `json:"status"` + Action string `json:"action"` + Errno int `json:"errno"` + Errstr string `json:"errstr"` + Result string `json:"result"` +} + +type CookieDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"download_url"` + Store string `json:"store"` + Host string `json:"host"` + } `json:"data"` +} + +func (r *CookieDownloadResp) GetURL() string { + return r.Data.DownloadURL +} + +type OpenDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"downloadUrl"` + } `json:"data"` + DownloadURL string `json:"downloadUrl"` +} + +func (r *OpenDownloadResp) GetURL() string { + return firstNonEmpty(r.Data.DownloadURL, r.DownloadURL) +} + +type YunpanObject struct { + model.Object + model.Thumbnail + OwnerQID string + DownloadToken string +} + +func parseAnySize(values ...interface{}) int64 { + for _, value := range values { + switch v := value.(type) { + case string: + if v == "" { + continue + } + size, err := strconv.ParseInt(v, 10, 64) + if err == nil { + return size + } + case float64: + return int64(v) + case int64: + return v + case int: + return int64(v) + } + } + return 0 +} + +func parseAnyTime(values ...interface{}) time.Time { + for _, value := range values { + switch v := value.(type) { + case string: + if t := parseUnixTime(v); !t.IsZero() { + return t + } + if t := parseTextTime(v); !t.IsZero() { + return t + } + case float64: + if v > 0 { + return time.Unix(int64(v), 0) + } + case int64: + if v > 0 { + return time.Unix(v, 0) + } + case int: + if v > 0 { + return time.Unix(int64(v), 0) + } + } + } + return time.Time{} +} + +func parseOpenDir(values ...interface{}) bool { + for _, value := range values { + switch v := value.(type) { + case bool: + if v { + return true + } + case string: + switch strings.ToLower(strings.TrimSpace(v)) { + case "1", "true", "dir", "folder": + return true + } + case float64: + if int64(v) == 1 { + return true + } + case int64: + if v == 1 { + return true + } + case int: + if v == 1 { + return true + } + } + } + return false +} + +func pathLooksLikeObject(objPath, name string) bool { + if objPath == "" || name == "" { + return false + } + return strings.TrimSuffix(stdPathBase(objPath), "/") == strings.TrimSuffix(name, "/") +} + +func stdPathBase(p string) string { + if p == "/" { + return "/" + } + idx := strings.LastIndex(strings.TrimSuffix(p, "/"), "/") + if idx < 0 { + return p + } + return p[idx+1:] +} + +func joinRemotePath(parentPath, name string) string { + parentPath = normalizeRemotePath(parentPath) + if parentPath == "" { + parentPath = "/" + } + return normalizeRemotePath(strings.TrimSuffix(parentPath, "/") + "/" + strings.TrimPrefix(name, "/")) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} diff --git a/drivers/yunpan360/upload.go b/drivers/yunpan360/upload.go new file mode 100644 index 00000000000..38c3fded57b --- /dev/null +++ b/drivers/yunpan360/upload.go @@ -0,0 +1,926 @@ +package yunpan360 + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + stdpath "path" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + aliststream "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + yunpanUploadChunkSize = int64(512 * 1024) + yunpanUploadBoundary = "WebKitFormBoundaryQ5OJVvzZwEkg4ttY" + yunpanUploadVersion = "1.0.1" + yunpanUploadDevType = "ecs_openapi" + yunpanUploadDevName = "EYUN_WEB_UPLOAD" +) + +type openUploadPlan struct { + DirPath string + TargetPath string + FileName string + Size int64 + FileHash string + FileSHA1 string + FileSum string + CreatedAt int64 + DeviceID string + Chunks []openUploadChunk +} + +type openUploadChunk struct { + Index int + Offset int64 + Size int64 + Hash string +} + +type openUploadDetectResp struct { + BaseResp + Data struct { + Exists []openUploadDuplicate `json:"exists"` + IsSlice int `json:"is_slice"` + } `json:"data"` +} + +type openUploadDuplicate struct { + FullName string `json:"fullName"` +} + +type openUploadAddressResp struct { + BaseResp + Data struct { + HTTP string `json:"http"` + Addr1 string `json:"addr_1"` + Addr2 string `json:"addr_2"` + Backup string `json:"backup"` + TK string `json:"tk"` + GroupSize string `json:"group_size"` + AutoCommit interface{} `json:"autoCommit"` + IsHTTPS interface{} `json:"is_https"` + } `json:"data"` +} + +type openUploadRequestResp struct { + BaseResp + Data struct { + Tid string `json:"tid"` + BlockInfo []map[string]interface{} `json:"block_info"` + } `json:"data"` +} + +type openUploadFinalizeResp struct { + BaseResp + Data map[string]interface{} `json:"data"` +} + +type uploadEnvelope struct { + Errno *int `json:"errno"` + Errmsg string `json:"errmsg"` + Data json.RawMessage `json:"data"` +} + +func (d *Yunpan360) putOpenFile(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return nil, err + } + + dirPath := dstDir.GetPath() + if dirPath == "" { + dirPath = d.RootFolderPath + } + dirPath = ensureDirAPIPath(dirPath) + targetPath := joinRemotePath(dirPath, file.GetName()) + + cached, err := d.cacheUploadSource(ctx, file, progressRange(up, 0, 5)) + if err != nil { + return nil, err + } + defer func() { + _, _ = cached.Seek(0, io.SeekStart) + }() + + plan, err := d.buildUploadPlan(ctx, cached, targetPath, file.GetSize(), file.ModTime(), progressRange(up, 5, 10)) + if err != nil { + return nil, err + } + + detectResp, err := d.openDetectUpload(ctx, auth, plan) + if err != nil { + return nil, err + } + if detectResp.Data.IsSlice == 0 { + detectResp.Data.IsSlice = 1 + } + + addrResp, err := d.openGetUploadAddress(ctx, auth, plan) + if err != nil { + return nil, err + } + + finalResp := &openUploadFinalizeResp{BaseResp: BaseResp{Errno: 0}, Data: map[string]interface{}{}} + if strings.TrimSpace(addrResp.Data.HTTP) == "" { + finalResp.Data = map[string]interface{}{"autoCommit": true} + if tk := strings.TrimSpace(addrResp.Data.TK); tk != "" { + finalResp.Data["tk"] = tk + finalResp.Data["autoCommit"] = false + } + } else { + reqResp, err := d.openRequestUpload(ctx, auth, plan, addrResp) + if err != nil { + return nil, err + } + if err := d.openUploadBlocks(ctx, auth, cached, plan, addrResp, reqResp, up); err != nil { + return nil, err + } + finalResp, err = d.openCommitUpload(ctx, auth, plan, addrResp, reqResp) + if err != nil { + return nil, err + } + } + + if err := d.openFinalizeUpload(ctx, auth, finalResp); err != nil { + return nil, err + } + if up != nil { + up(100) + } + + obj, err := d.findUploadedObject(ctx, targetPath) + if err == nil { + return obj, nil + } + if !errors.Is(err, errs.ObjectNotFound) { + return nil, err + } + + return &model.Object{ + Path: normalizeRemotePath(targetPath), + Name: file.GetName(), + Size: file.GetSize(), + Modified: time.Now(), + Ctime: time.Now(), + HashInfo: utils.NewHashInfo(utils.SHA1, firstNonEmpty(plan.FileSHA1, plan.FileHash)), + }, nil +} + +func (d *Yunpan360) cacheUploadSource(ctx context.Context, file model.FileStreamer, up driver.UpdateProgress) (model.File, error) { + if cached := file.GetFile(); cached != nil { + _, _ = cached.Seek(0, io.SeekStart) + if up != nil { + up(100) + } + return cached, nil + } + if up == nil { + return file.CacheFullInTempFile() + } + return aliststream.CacheFullInTempFileAndUpdateProgress(file, up) +} + +func (d *Yunpan360) buildUploadPlan(ctx context.Context, cached model.File, targetPath string, size int64, modTime time.Time, up driver.UpdateProgress) (*openUploadPlan, error) { + if _, err := cached.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + createdAt := time.Now().Unix() + if !modTime.IsZero() { + createdAt = modTime.Unix() + } + plan := &openUploadPlan{ + DirPath: ensureDirAPIPath(stdpath.Dir(targetPath)), + TargetPath: normalizeRemotePath(targetPath), + FileName: stdpath.Base(targetPath), + Size: size, + CreatedAt: createdAt, + DeviceID: sha1HexString("node-sdk-" + runtime.Version()), + } + + if plan.DirPath == "./" || plan.DirPath == "." { + plan.DirPath = "/" + } + + totalChunks := 0 + if size > 0 { + totalChunks = int((size + yunpanUploadChunkSize - 1) / yunpanUploadChunkSize) + } + chunks := make([]openUploadChunk, 0, totalChunks) + var hashConcat strings.Builder + var hashed int64 + + for idx := 0; idx < totalChunks; idx++ { + if err := ctx.Err(); err != nil { + return nil, err + } + offset := int64(idx) * yunpanUploadChunkSize + chunkSize := yunpanUploadChunkSize + if remain := size - offset; remain < chunkSize { + chunkSize = remain + } + + chunkHash, err := sha1HexReader(io.NewSectionReader(cached, offset, chunkSize)) + if err != nil { + return nil, err + } + + chunks = append(chunks, openUploadChunk{ + Index: idx + 1, + Offset: offset, + Size: chunkSize, + Hash: chunkHash, + }) + hashConcat.WriteString(chunkHash) + hashed += chunkSize + reportByteProgress(up, hashed, size) + } + + plan.Chunks = chunks + plan.FileHash = sha1HexString(hashConcat.String()) + if _, err := cached.Seek(0, io.SeekStart); err != nil { + return nil, err + } + sha1Hasher := sha1.New() + md5Hasher := md5.New() + if _, err := io.Copy(io.MultiWriter(sha1Hasher, md5Hasher), cached); err != nil { + return nil, err + } + plan.FileSHA1 = hex.EncodeToString(sha1Hasher.Sum(nil)) + plan.FileSum = hex.EncodeToString(md5Hasher.Sum(nil)) + if size == 0 && up != nil { + up(100) + } + return plan, nil +} + +func (d *Yunpan360) openDetectUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan) (*openUploadDetectResp, error) { + payload, err := json.Marshal([]map[string]interface{}{ + {"fname": plan.FileName, "fsize": plan.Size}, + }) + if err != nil { + return nil, err + } + + signParams := map[string]string{ + "data": string(payload), + "path": plan.DirPath, + } + body, contentType, err := createMultipartForm("", map[string]string{ + "qid": auth.Qid, + "data": string(payload), + "path": plan.DirPath, + "sign": openSign(auth.AccessToken, auth.Qid, "Sync.detectFileExists", signParams), + }, nil) + if err != nil { + return nil, err + } + + var resp openUploadDetectResp + err = d.uploadRequest(ctx, http.MethodPost, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.detectFileExists", nil), auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openGetUploadAddress(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan) (*openUploadAddressResp, error) { + query := d.uploadCookieParams(auth, plan, "") + signParams := map[string]string{ + "access_token": auth.AccessToken, + "fhash": plan.FileHash, + "fname": plan.TargetPath, + "fsize": strconv.FormatInt(plan.Size, 10), + } + query["sign"] = openSign(auth.AccessToken, auth.Qid, "Sync.getUploadFileAddr", signParams) + + var resp openUploadAddressResp + err := d.uploadRequest(ctx, http.MethodGet, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.getUploadFileAddr", query), auth.AccessToken, "", nil, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openRequestUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan, addrResp *openUploadAddressResp) (*openUploadRequestResp, error) { + chunkInfos := make([]map[string]interface{}, 0, len(plan.Chunks)) + for _, chunk := range plan.Chunks { + chunkInfos = append(chunkInfos, map[string]interface{}{ + "bhash": chunk.Hash, + "bidx": chunk.Index, + "boffset": chunk.Offset, + "bsize": chunk.Size, + }) + } + payload, err := json.Marshal(map[string]interface{}{ + "request": map[string]interface{}{ + "block_info": chunkInfos, + }, + }) + if err != nil { + return nil, err + } + + body, contentType, err := createMultipartForm( + yunpanUploadBoundary, + d.uploadCookieParams(auth, plan, strings.TrimSpace(addrResp.Data.TK)), + &multipartFile{ + FieldName: "file", + FileName: "file.dat", + ContentType: "application/octet-stream", + Content: payload, + }, + ) + if err != nil { + return nil, err + } + + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.request4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var resp openUploadRequestResp + err = d.uploadRequest(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openUploadBlocks(ctx context.Context, auth *OpenAuthInfo, cached model.File, plan *openUploadPlan, addrResp *openUploadAddressResp, reqResp *openUploadRequestResp, up driver.UpdateProgress) error { + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.block4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var uploaded int64 + for _, chunk := range plan.Chunks { + info := reqResp.blockInfoForChunk(chunk.Index) + if info.found() > 0 { + uploaded += chunk.Size + reportUploadProgress(up, uploaded, plan.Size) + continue + } + + chunkBytes := make([]byte, chunk.Size) + if _, err := cached.ReadAt(chunkBytes, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { + return err + } + + fields := map[string]string{ + "bhash": chunk.Hash, + "bidx": strconv.Itoa(chunk.Index), + "boffset": strconv.FormatInt(chunk.Offset, 10), + "bsize": strconv.FormatInt(chunk.Size, 10), + "filename": plan.TargetPath, + "filesize": strconv.FormatInt(plan.Size, 10), + "q": info.stringValue("q"), + "t": info.stringValue("t"), + "token": auth.Token, + "tid": info.stringValue("tid"), + } + for key, value := range info.extraFields() { + fields[key] = value + } + + body, contentType, err := createMultipartForm( + yunpanUploadBoundary, + fields, + &multipartFile{ + FieldName: "file", + FileName: "file.dat", + ContentType: "application/octet-stream", + Content: chunkBytes, + }, + ) + if err != nil { + return err + } + + chunkStart := uploaded + chunkSize := chunk.Size + err = d.uploadRequestWithProgress(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, func(p float64) { + done := chunkStart + int64(float64(chunkSize)*(p/100.0)) + reportUploadProgress(up, done, plan.Size) + }, nil) + if err != nil { + return err + } + + uploaded += chunk.Size + reportUploadProgress(up, uploaded, plan.Size) + } + return nil +} + +func (d *Yunpan360) openCommitUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan, addrResp *openUploadAddressResp, reqResp *openUploadRequestResp) (*openUploadFinalizeResp, error) { + body, contentType, err := createMultipartForm("", map[string]string{ + "q": "", + "t": "", + "token": auth.Token, + "tid": strings.TrimSpace(reqResp.Data.Tid), + }, nil) + if err != nil { + return nil, err + } + + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.commit4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var resp openUploadFinalizeResp + err = d.uploadRequest(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openFinalizeUpload(ctx context.Context, auth *OpenAuthInfo, resp *openUploadFinalizeResp) error { + if resp == nil || resp.autoCommit() { + return nil + } + tk := strings.TrimSpace(resp.stringValue("tk")) + if tk == "" { + return errors.New("upload tk is empty") + } + + signParams := map[string]string{ + "tk": tk, + } + body, contentType, err := createMultipartForm("", map[string]string{ + "qid": auth.Qid, + "tk": tk, + "sign": openSign(auth.AccessToken, auth.Qid, "Sync.addFileToApi", signParams), + }, nil) + if err != nil { + return err + } + + return d.uploadRequest(ctx, http.MethodPost, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.addFileToApi", nil), auth.AccessToken, contentType, body, nil) +} + +func (d *Yunpan360) findUploadedObject(ctx context.Context, targetPath string) (model.Obj, error) { + targetPath = normalizeRemotePath(targetPath) + parentPath := normalizeRemotePath(stdpath.Dir(targetPath)) + if parentPath == "." || parentPath == "" { + parentPath = "/" + } + targetName := stdpath.Base(targetPath) + + for page := 0; ; page++ { + resp, err := d.listPage(ctx, parentPath, page, d.PageSize) + if err != nil { + return nil, err + } + pageObjs := resp.Objects(parentPath) + for _, obj := range pageObjs { + if normalizeRemotePath(obj.GetPath()) == targetPath || obj.GetName() == targetName { + return obj, nil + } + } + if len(pageObjs) == 0 || len(pageObjs) < d.PageSize { + break + } + } + return nil, errs.ObjectNotFound +} + +func (d *Yunpan360) uploadCookieParams(auth *OpenAuthInfo, plan *openUploadPlan, uploadTK string) map[string]string { + params := map[string]string{ + "owner_qid": auth.Qid, + "fname": plan.TargetPath, + "fsize": strconv.FormatInt(plan.Size, 10), + "fctime": strconv.FormatInt(plan.CreatedAt, 10), + "fmtime": strconv.FormatInt(plan.CreatedAt, 10), + "fhash": plan.FileHash, + "qid": auth.Qid, + "fattr": "0", + "token": auth.Token, + "devtype": yunpanUploadDevType, + } + if uploadTK != "" { + params["tk"] = uploadTK + } + return params +} + +func (d *Yunpan360) uploadDataParams(auth *OpenAuthInfo, plan *openUploadPlan) map[string]string { + return map[string]string{ + "owner_qid": auth.Qid, + "qid": auth.Qid, + "devtype": yunpanUploadDevType, + "devid": plan.DeviceID, + "v": yunpanUploadVersion, + "ofmt": "json", + "devname": yunpanUploadDevName, + "rtick": strconv.FormatInt(time.Now().UnixMilli(), 10), + } +} + +func (d *Yunpan360) uploadBaseURL(addrResp *openUploadAddressResp) string { + host := "" + isHTTPS := false + if addrResp != nil { + host = strings.TrimSpace(addrResp.Data.HTTP) + isHTTPS = parseOpenDir(addrResp.Data.IsHTTPS) + } + scheme := "http" + if isHTTPS { + scheme = "https" + } + if host == "" { + return openAPIURL(d.EcsEnv) + } + return fmt.Sprintf("%s://%s/intf.php", scheme, host) +} + +func (d *Yunpan360) uploadRequest(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, out interface{}) error { + return d.uploadRequestWithProgress(ctx, method, reqURL, accessToken, contentType, body, nil, out) +} + +func (d *Yunpan360) uploadRequestWithProgress(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, progress driver.UpdateProgress, out interface{}) error { + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + if err := sleepWithContext(ctx, time.Duration(attempt)*500*time.Millisecond); err != nil { + return err + } + } + err := d.doUploadRequest(ctx, method, reqURL, accessToken, contentType, body, progress, out) + if err == nil { + return nil + } + lastErr = err + if ctx.Err() != nil { + return ctx.Err() + } + } + return lastErr +} + +func (d *Yunpan360) doUploadRequest(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, progress driver.UpdateProgress, out interface{}) error { + var bodyReader io.ReadCloser + if body != nil { + reader := &driver.SimpleReaderWithSize{ + Reader: bytes.NewReader(body), + Size: int64(len(body)), + } + if progress != nil { + bodyReader = driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: reader, + UpdateProgress: progress, + }) + } else { + bodyReader = driver.NewLimitedUploadStream(ctx, reader) + } + } + + req, err := http.NewRequestWithContext(ctx, method, reqURL, bodyReader) + if err != nil { + if bodyReader != nil { + _ = bodyReader.Close() + } + return err + } + if body != nil { + req.ContentLength = int64(len(body)) + } + req.Header.Set("Accept", "application/json") + if accessToken != "" { + req.Header.Set("Access-Token", accessToken) + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode >= http.StatusBadRequest { + return fmt.Errorf("yunpan upload request failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(respBody))) + } + return decodeUploadResp(respBody, out) +} + +func decodeUploadResp(body []byte, out interface{}) error { + var env uploadEnvelope + if err := utils.Json.Unmarshal(body, &env); err != nil { + return err + } + if env.Errno != nil && *env.Errno != 0 { + if env.Errmsg == "" { + return fmt.Errorf("yunpan upload request failed: errno=%d", *env.Errno) + } + return errors.New(env.Errmsg) + } + if env.Errno == nil && strings.TrimSpace(env.Errmsg) != "" && len(env.Data) > 0 && string(env.Data) == "[]" { + return errors.New(env.Errmsg) + } + if out == nil { + return nil + } + if err := utils.Json.Unmarshal(body, out); err != nil { + if strings.TrimSpace(env.Errmsg) != "" { + return errors.New(env.Errmsg) + } + return err + } + return nil +} + +type multipartFile struct { + FieldName string + FileName string + ContentType string + Content []byte +} + +func createMultipartForm(boundary string, fields map[string]string, file *multipartFile) ([]byte, string, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + if boundary != "" { + if err := writer.SetBoundary(boundary); err != nil { + return nil, "", err + } + } + + for key, value := range fields { + if err := writer.WriteField(key, value); err != nil { + return nil, "", err + } + } + + if file != nil { + partHeader := make(textproto.MIMEHeader) + partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, file.FieldName, file.FileName)) + partHeader.Set("Content-Type", firstNonEmpty(file.ContentType, "application/octet-stream")) + part, err := writer.CreatePart(partHeader) + if err != nil { + return nil, "", err + } + if _, err := part.Write(file.Content); err != nil { + return nil, "", err + } + } + + if err := writer.Close(); err != nil { + return nil, "", err + } + return body.Bytes(), writer.FormDataContentType(), nil +} + +func buildJSQueryURL(baseURL, method string, params map[string]string) string { + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + + var builder strings.Builder + builder.WriteString(baseURL) + if strings.Contains(baseURL, "?") { + builder.WriteByte('&') + } else { + builder.WriteByte('?') + } + builder.WriteString("method=") + builder.WriteString(jsQueryEscape(method)) + for _, key := range keys { + value := params[key] + if value == "" { + continue + } + builder.WriteByte('&') + builder.WriteString(key) + builder.WriteByte('=') + builder.WriteString(jsQueryEscape(value)) + } + return builder.String() +} + +func buildQueryURL(baseURL string, params map[string]string) string { + u, err := url.Parse(baseURL) + if err != nil { + return baseURL + } + u.RawQuery = encodeSortedQuery(params) + return u.String() +} + +func encodeSortedQuery(params map[string]string) string { + if len(params) == 0 { + return "" + } + q := make(url.Values, len(params)) + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + q.Set(key, params[key]) + } + return q.Encode() +} + +func appendHostQuery(rawURL, host string) string { + host = strings.TrimSpace(host) + if host == "" { + return rawURL + } + return rawURL + "&host=" + jsQueryEscape(host) +} + +func jsQueryEscape(raw string) string { + replacer := strings.NewReplacer( + "+", "%20", + "%21", "!", + "%27", "'", + "%28", "(", + "%29", ")", + "%2A", "*", + "%7E", "~", + ) + return replacer.Replace(url.QueryEscape(raw)) +} + +func sha1HexReader(r io.Reader) (string, error) { + h := sha1.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func sha1HexString(raw string) string { + sum := sha1.Sum([]byte(raw)) + return hex.EncodeToString(sum[:]) +} + +func progressRange(up driver.UpdateProgress, start, end float64) driver.UpdateProgress { + if up == nil { + return nil + } + return model.UpdateProgressWithRange(up, start, end) +} + +func reportByteProgress(up driver.UpdateProgress, done, total int64) { + if up == nil { + return + } + if total <= 0 { + up(100) + return + } + up(float64(done) / float64(total) * 100) +} + +func reportUploadProgress(up driver.UpdateProgress, done, total int64) { + if up == nil { + return + } + if total <= 0 { + up(100) + return + } + if done > total { + done = total + } + up(10 + float64(done)/float64(total)*90) +} + +func sleepWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +type blockInfoMap map[string]interface{} + +func (r *openUploadRequestResp) blockInfoForChunk(index int) blockInfoMap { + if index <= 0 || index > len(r.Data.BlockInfo) { + return blockInfoMap{} + } + return blockInfoMap(r.Data.BlockInfo[index-1]) +} + +func (m blockInfoMap) stringValue(key string) string { + value, ok := m[key] + if !ok || value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case float64: + return strconv.FormatInt(int64(v), 10) + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + case bool: + if v { + return "1" + } + return "0" + default: + return fmt.Sprint(v) + } +} + +func (m blockInfoMap) found() int64 { + raw := strings.TrimSpace(m.stringValue("found")) + if raw == "" { + return 0 + } + value, _ := strconv.ParseInt(raw, 10, 64) + return value +} + +func (m blockInfoMap) extraFields() map[string]string { + extras := make(map[string]string) + for key := range m { + switch key { + case "bhash", "bidx", "boffset", "bsize", "filename", "filesize", "q", "t", "token", "tid", "found", "url": + continue + } + value := strings.TrimSpace(m.stringValue(key)) + if value != "" { + extras[key] = value + } + } + return extras +} + +func (r *openUploadFinalizeResp) autoCommit() bool { + raw, ok := r.Data["autoCommit"] + if !ok { + return false + } + switch v := raw.(type) { + case bool: + return v + case string: + return strings.EqualFold(v, "true") || v == "1" + case float64: + return int64(v) == 1 + case int: + return v == 1 + case int64: + return v == 1 + default: + return false + } +} + +func (r *openUploadFinalizeResp) stringValue(key string) string { + if r == nil || r.Data == nil { + return "" + } + value, ok := r.Data[key] + if !ok || value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case float64: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + case int: + return strconv.Itoa(v) + default: + return fmt.Sprint(v) + } +} diff --git a/drivers/yunpan360/upload_test.go b/drivers/yunpan360/upload_test.go new file mode 100644 index 00000000000..092570ff5e8 --- /dev/null +++ b/drivers/yunpan360/upload_test.go @@ -0,0 +1,25 @@ +package yunpan360 + +import ( + "bytes" + "testing" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +func TestBuildUploadPlanComputesFileSHA1AndMD5(t *testing.T) { + d := &Yunpan360{} + data := []byte("hello yunpan") + file := model.NewNopMFile(bytes.NewReader(data)) + plan, err := d.buildUploadPlan(t.Context(), file, "/hello.txt", int64(len(data)), time.Unix(1700000000, 0), nil) + if err != nil { + t.Fatalf("buildUploadPlan() error = %v", err) + } + if plan.FileSHA1 != "254ec33af17332a3964145f8c6a3dc12833c7ea2" { + t.Fatalf("FileSHA1 = %q", plan.FileSHA1) + } + if plan.FileSum != "fefeffa5b6ae9f39851050b44cacfcb1" { + t.Fatalf("FileSum = %q", plan.FileSum) + } +} diff --git a/drivers/yunpan360/util.go b/drivers/yunpan360/util.go new file mode 100644 index 00000000000..2718f9c99cc --- /dev/null +++ b/drivers/yunpan360/util.go @@ -0,0 +1,909 @@ +package yunpan360 + +import ( + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "html" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + baseURL = "https://www.yunpan.com" + indexPath = "/file/index" + listPath = "/file/list" + downloadPath = "/file/download" + openAPIProdURL = "https://openapi.eyun.360.cn/intf.php" + openAPITestURL = "https://qaopen.eyun.360.cn/intf.php" + openAPIHGTestURL = "https://hg-openapi.eyun.360.cn/intf.php" +) + +func (d *Yunpan360) listPage(ctx context.Context, dirPath string, page, pageSize int) (ListResp, error) { + if d.authMode() == authTypeAPIKey { + return d.listOpenPage(ctx, dirPath, page, pageSize) + } + return d.listCookiePage(ctx, dirPath, page, pageSize) +} + +func (d *Yunpan360) listCookiePage(ctx context.Context, dirPath string, page, pageSize int) (*CookieListResp, error) { + var resp CookieListResp + err := d.cookieRequestForm(ctx, listPath, map[string]string{ + "path": requestPath(dirPath), + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(pageSize), + "order": requestOrder(d.OrderDirection), + "field": "file_name", + "focus_nid": "0", + }, &resp) + if err != nil { + return nil, err + } + d.cacheCookieDownloadSession(resp.GetOwnerQID(), resp.Token) + return &resp, nil +} + +func (d *Yunpan360) cookieRequestForm(ctx context.Context, apiPath string, form map[string]string, out interface{}) error { + req := base.RestyClient.R(). + SetContext(ctx). + SetHeaders(map[string]string{ + "Accept": "text/javascript, text/html, application/xml, text/xml, */*", + "Content-Type": "application/x-www-form-urlencoded", + "Cookie": d.Cookie, + "Origin": baseURL, + "Referer": baseURL + "/file/index", + "X-Requested-With": "XMLHttpRequest", + }). + SetFormData(form) + + res, err := req.Execute(http.MethodPost, baseURL+apiPath) + if err != nil { + return err + } + + var baseResp BaseResp + if err := utils.Json.Unmarshal(res.Body(), &baseResp); err != nil { + return err + } + if baseResp.Errno != 0 { + if baseResp.Errmsg == "" { + return fmt.Errorf("yunpan request failed: errno=%d", baseResp.Errno) + } + return errors.New(baseResp.Errmsg) + } + if out == nil { + return nil + } + return utils.Json.Unmarshal(res.Body(), out) +} + +func requestPath(dirPath string) string { + path := normalizeRemotePath(dirPath) + if path == "" { + return "/" + } + return path +} + +func requestOrder(order string) string { + if strings.EqualFold(order, "desc") { + return "desc" + } + return "asc" +} + +func (d *Yunpan360) cookiePage(ctx context.Context, pagePath string) ([]byte, error) { + req := base.RestyClient.R(). + SetContext(ctx). + SetHeaders(map[string]string{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Cookie": d.Cookie, + "Referer": baseURL + indexPath, + }) + res, err := req.Get(baseURL + pagePath) + if err != nil { + return nil, err + } + return res.Body(), nil +} + +func openAPIURL(env string) string { + switch env { + case "test": + return openAPITestURL + case "hgtest": + return openAPIHGTestURL + default: + return openAPIProdURL + } +} + +func openClientSecretForEnv(env string) string { + if env == "test" { + return openClientSecretQA + } + return openClientSecret +} + +func phpQueryEscape(raw string) string { + escaped := url.QueryEscape(raw) + return strings.ReplaceAll(escaped, "~", "%7E") +} + +func openSign(accessToken, qid, method string, extra map[string]string) string { + params := map[string]string{ + "access_token": accessToken, + "method": method, + "qid": qid, + } + for key, value := range extra { + if value != "" { + params[key] = value + } + } + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sortStrings(keys) + pairs := make([]string, 0, len(keys)) + for _, key := range keys { + pairs = append(pairs, key+"="+phpQueryEscape(params[key])) + } + sum := md5.Sum([]byte(strings.Join(pairs, "&") + openSignSecret)) + return hex.EncodeToString(sum[:]) +} + +func sortStrings(values []string) { + for i := 0; i < len(values); i++ { + for j := i + 1; j < len(values); j++ { + if values[j] < values[i] { + values[i], values[j] = values[j], values[i] + } + } + } +} + +func (d *Yunpan360) getOpenAuth(ctx context.Context) (*OpenAuthInfo, error) { + d.authMu.Lock() + defer d.authMu.Unlock() + + if d.cachedOpenAuth != nil && time.Now().Before(d.openAuthExpire) { + auth := *d.cachedOpenAuth + return &auth, nil + } + + reqURL := openAPIURL(d.EcsEnv) + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Accept", "application/json"). + SetHeader("api_key", d.APIKey). + SetQueryParams(map[string]string{ + "method": "Oauth.getAccessTokenByApiKey", + "client_id": openClientID, + "client_secret": openClientSecretForEnv(d.EcsEnv), + "grant_type": "authorization_code", + "sub_channel": d.SubChannel, + "api_key": d.APIKey, + }) + + res, err := req.Get(reqURL) + if err != nil { + return nil, err + } + + var resp OpenAuthResp + if err := utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + if resp.Errno != 0 { + if resp.Errmsg == "" { + return nil, fmt.Errorf("yunpan auth failed: errno=%d", resp.Errno) + } + return nil, errors.New(resp.Errmsg) + } + + auth := &OpenAuthInfo{ + AccessToken: resp.Data.AccessToken, + Qid: resp.Data.Qid, + Token: resp.Data.Token, + SubChannel: d.SubChannel, + } + d.cachedOpenAuth = auth + d.openAuthExpire = time.Now().Add(50 * time.Minute) + + copied := *auth + return &copied, nil +} + +func (d *Yunpan360) openBaseParams(auth *OpenAuthInfo, method string, signParams map[string]string, withSign bool) map[string]string { + params := map[string]string{ + "method": method, + "access_token": auth.AccessToken, + "qid": auth.Qid, + "sub_channel": auth.SubChannel, + } + if withSign { + params["sign"] = openSign(auth.AccessToken, auth.Qid, method, signParams) + } else { + params["sign"] = "" + } + return params +} + +func (d *Yunpan360) openGET(ctx context.Context, method string, signParams map[string]string, query map[string]string, out interface{}, withSign bool) error { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return err + } + params := d.openBaseParams(auth, method, signParams, withSign) + for key, value := range query { + params[key] = value + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Access-Token", auth.AccessToken). + SetQueryParams(params) + res, err := req.Get(openAPIURL(d.EcsEnv)) + if err != nil { + return err + } + return decodeBaseResp(res.Body(), out) +} + +func (d *Yunpan360) openPOST(ctx context.Context, method string, signParams map[string]string, query, body map[string]string, out interface{}, withSign bool) error { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return err + } + queryParams := map[string]string{} + for key, value := range query { + queryParams[key] = value + } + bodyParams := map[string]string{} + for key, value := range body { + bodyParams[key] = value + } + + baseParams := d.openBaseParams(auth, method, signParams, withSign) + if len(queryParams) == 0 { + bodyParams = mergeStringMaps(baseParams, bodyParams) + } else { + queryParams = mergeStringMaps(baseParams, queryParams) + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Access-Token", auth.AccessToken). + SetHeader("Content-Type", "application/x-www-form-urlencoded") + if len(queryParams) > 0 { + req.SetQueryParams(queryParams) + } + if len(bodyParams) > 0 { + req.SetFormData(bodyParams) + } + res, err := req.Post(openAPIURL(d.EcsEnv)) + if err != nil { + return err + } + return decodeBaseResp(res.Body(), out) +} + +func decodeBaseResp(body []byte, out interface{}) error { + var baseResp BaseResp + if err := utils.Json.Unmarshal(body, &baseResp); err != nil { + return err + } + if baseResp.Errno != 0 { + if baseResp.Errmsg == "" { + return fmt.Errorf("yunpan request failed: errno=%d", baseResp.Errno) + } + return errors.New(baseResp.Errmsg) + } + if out == nil { + return nil + } + return utils.Json.Unmarshal(body, out) +} + +func mergeStringMaps(baseMap, extra map[string]string) map[string]string { + merged := map[string]string{} + for key, value := range baseMap { + merged[key] = value + } + for key, value := range extra { + merged[key] = value + } + return merged +} + +func (d *Yunpan360) cookieDownloadURL(ctx context.Context, file model.Obj) (*CookieDownloadResp, error) { + resp, err := d.cookieDownloadURLOnce(ctx, file, false) + if err == nil { + return resp, nil + } + + d.invalidateCookieDownloadSession() + return d.cookieDownloadURLOnce(ctx, file, true) +} + +func (d *Yunpan360) cookieDownloadURLOnce(ctx context.Context, file model.Obj, refresh bool) (*CookieDownloadResp, error) { + nid := strings.TrimSpace(file.GetID()) + if nid == "" { + return nil, errors.New("missing file id") + } + + fname := normalizeRemotePath(file.GetPath()) + if fname == "" { + return nil, errors.New("missing file path") + } + + ownerQID, token, err := d.resolveCookieDownloadParams(ctx, file, refresh) + if err != nil { + return nil, err + } + + var resp CookieDownloadResp + err = d.cookieRequestForm(ctx, downloadPath, map[string]string{ + "nid": nid, + "fname": fname, + "owner_qid": ownerQID, + "token": token, + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) cookieRename(ctx context.Context, srcObj model.Obj, newName string) error { + path := normalizeRemotePath(srcObj.GetPath()) + if path == "" { + return errors.New("missing object path") + } + nid := strings.TrimSpace(srcObj.GetID()) + if nid == "" { + return errors.New("missing object id") + } + + ownerQID, err := d.resolveCookieOwnerQID(ctx, srcObj, false) + if err != nil { + return err + } + + return d.cookieRequestForm(ctx, "/file/rename", map[string]string{ + "path": path, + "nid": nid, + "newpath": strings.TrimSuffix(strings.TrimSpace(newName), "/"), + "owner_qid": ownerQID, + }, nil) +} + +func (d *Yunpan360) resolveCookieDownloadParams(ctx context.Context, file model.Obj, refresh bool) (string, string, error) { + ownerQID := sanitizeOwnerQID(d.OwnerQID) + token := strings.TrimSpace(d.DownloadToken) + + if obj, ok := file.(*YunpanObject); ok { + ownerQID = firstNonEmpty(sanitizeOwnerQID(obj.OwnerQID), ownerQID) + token = firstNonEmpty(strings.TrimSpace(obj.DownloadToken), token) + } + + ownerQID = firstNonEmpty(ownerQID, + sanitizeOwnerQID(getCookieValue(d.Cookie, "owner_qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "ownerQid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "QID")), + ) + token = firstNonEmpty(token, + getCookieValue(d.Cookie, "download_token"), + getCookieValue(d.Cookie, "token"), + getCookieValue(d.Cookie, "Token"), + ) + + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + return ownerQID, token, nil + } + + if !refresh { + if cached := d.getCachedCookieDownloadSession(); cached != nil { + ownerQID = firstNonEmpty(ownerQID, cached.OwnerQID) + token = firstNonEmpty(token, cached.Token) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + return ownerQID, token, nil + } + } + + resp, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + if err == nil && resp != nil { + ownerQID = firstNonEmpty(ownerQID, resp.GetOwnerQID()) + token = firstNonEmpty(token, strings.TrimSpace(resp.Token)) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + d.cacheCookieDownloadSession(ownerQID, token) + return ownerQID, token, nil + } + + pageSession, err := d.getCookieDownloadSessionFromPage(ctx) + if err == nil && pageSession != nil { + ownerQID = firstNonEmpty(ownerQID, pageSession.OwnerQID) + token = firstNonEmpty(token, pageSession.Token) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID == "" || token == "" { + return "", "", errors.New("missing owner_qid or download_token for cookie mode") + } + + d.cacheCookieDownloadSession(ownerQID, token) + return ownerQID, token, nil +} + +func (d *Yunpan360) resolveCookieOwnerQID(ctx context.Context, file model.Obj, refresh bool) (string, error) { + ownerQID := sanitizeOwnerQID(d.OwnerQID) + + if obj, ok := file.(*YunpanObject); ok { + ownerQID = firstNonEmpty(sanitizeOwnerQID(obj.OwnerQID), ownerQID) + } + + ownerQID = firstNonEmpty(ownerQID, + sanitizeOwnerQID(getCookieValue(d.Cookie, "owner_qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "ownerQid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "QID")), + ) + if ownerQID != "" { + return ownerQID, nil + } + + if !refresh { + if cached := d.getCachedCookieDownloadSession(); cached != nil { + ownerQID = firstNonEmpty(ownerQID, cached.OwnerQID) + } + if ownerQID != "" { + return ownerQID, nil + } + } + + resp, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + if err == nil && resp != nil { + ownerQID = firstNonEmpty(ownerQID, resp.GetOwnerQID()) + } + if ownerQID != "" { + return ownerQID, nil + } + + pageSession, err := d.getCookieDownloadSessionFromPage(ctx) + if err == nil && pageSession != nil { + ownerQID = firstNonEmpty(ownerQID, pageSession.OwnerQID) + } + if ownerQID == "" { + return "", errors.New("missing owner_qid for cookie mode") + } + return ownerQID, nil +} + +func (d *Yunpan360) getCookieDownloadSessionFromPage(ctx context.Context) (*CookieDownloadSession, error) { + body, err := d.cookiePage(ctx, indexPath) + if err != nil { + return nil, err + } + session := parseCookieDownloadSessionFromText(string(body)) + if session == nil { + return nil, errors.New("failed to parse cookie download session from page") + } + d.cacheCookieSession(session) + return session, nil +} + +func (d *Yunpan360) getCachedCookieDownloadSession() *CookieDownloadSession { + d.authMu.Lock() + defer d.authMu.Unlock() + + if d.cachedCookieSession == nil || time.Now().After(d.cookieSessionExpire) { + return nil + } + session := *d.cachedCookieSession + return &session +} + +func (d *Yunpan360) cacheCookieDownloadSession(ownerQID, token string) { + d.cacheCookieSession(&CookieDownloadSession{ + OwnerQID: ownerQID, + Token: token, + }) +} + +func (d *Yunpan360) cacheCookieSession(session *CookieDownloadSession) { + if session == nil { + return + } + + cached := &CookieDownloadSession{ + OwnerQID: sanitizeOwnerQID(session.OwnerQID), + Token: strings.TrimSpace(session.Token), + } + if cached.OwnerQID == "" && cached.Token != "" { + cached.OwnerQID = ownerQIDFromToken(cached.Token) + } + if cached.OwnerQID == "" || cached.Token == "" { + return + } + + d.authMu.Lock() + defer d.authMu.Unlock() + + d.cachedCookieSession = cached + d.cookieSessionExpire = time.Now().Add(10 * time.Minute) +} + +func (d *Yunpan360) invalidateCookieDownloadSession() { + d.authMu.Lock() + defer d.authMu.Unlock() + + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} +} + +func getCookieValue(rawCookie, name string) string { + for _, item := range strings.Split(rawCookie, ";") { + part := strings.TrimSpace(item) + if part == "" { + continue + } + key, value, ok := strings.Cut(part, "=") + if !ok || key != name { + continue + } + value = strings.TrimSpace(value) + value = strings.Trim(value, "\"") + unescaped, err := url.QueryUnescape(value) + if err == nil { + return strings.TrimSpace(unescaped) + } + return value + } + return "" +} + +func ownerQIDFromToken(token string) string { + parts := strings.Split(strings.TrimSpace(token), ".") + if len(parts) < 4 { + return "" + } + qid := strings.TrimSpace(parts[3]) + for _, ch := range qid { + if ch < '0' || ch > '9' { + return "" + } + } + return qid +} + +func sanitizeOwnerQID(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "0" { + return "" + } + return raw +} + +func parseCookieDownloadSessionFromText(text string) *CookieDownloadSession { + token := extractFirstMatch(text, + `(?i)["']download_token["']\s*[:=]\s*["']([^"'<>]+)["']`, + `(?i)["']token["']\s*[:=]\s*["']([^"'<>]+)["']`, + `(?i)\btoken\s*[:=]\s*["']([^"'<>]+)["']`, + ) + ownerQID := extractFirstMatch(text, + `(?i)["']owner_qid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)["']ownerQid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)["']qid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)\bowner_qid\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)\bqid\s*[:=]\s*["']?([0-9]+)["']?`, + ) + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID == "" || token == "" { + return nil + } + return &CookieDownloadSession{ + OwnerQID: ownerQID, + Token: token, + } +} + +func extractFirstMatch(text string, patterns ...string) string { + return extractFirstValidatedMatch(nil, text, patterns...) +} + +func extractFirstValidatedMatch(validate func(string) bool, text string, patterns ...string) string { + for _, pattern := range patterns { + re := regexp.MustCompile(pattern) + for _, matches := range re.FindAllStringSubmatch(text, -1) { + if len(matches) < 2 { + continue + } + value := html.UnescapeString(strings.TrimSpace(matches[1])) + value = strings.Trim(value, "\"'") + if value == "" { + continue + } + if validate == nil || validate(value) { + return value + } + } + } + return "" +} + +func (d *Yunpan360) listOpenPage(ctx context.Context, dirPath string, page, pageSize int) (*OpenListResp, error) { + var resp OpenListResp + path := ensureDirAPIPath(dirPath) + params := map[string]string{ + "path": path, + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(pageSize), + } + err := d.openGET(ctx, "File.getList", params, params, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openUserInfo(ctx context.Context) (*OpenUserInfoResp, error) { + var resp OpenUserInfoResp + err := d.openGET(ctx, "User.getUserDetail", nil, nil, &resp, false) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openDownloadURL(ctx context.Context, file model.Obj) (*OpenDownloadResp, error) { + var resp OpenDownloadResp + signParams := map[string]string{} + body := map[string]string{} + + if file.GetPath() != "" { + signParams["fpath"] = normalizeRemotePath(file.GetPath()) + body["fpath"] = signParams["fpath"] + } else if file.GetID() != "" { + signParams["nid"] = file.GetID() + body["nid"] = file.GetID() + } else { + return nil, errors.New("missing file path and id") + } + + err := d.openPOST(ctx, "MCP.getDownLoadUrl", signParams, nil, body, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) cookieMakeDir(ctx context.Context, fullPath string) (*CookieMkdirResp, error) { + var resp CookieMkdirResp + body := map[string]string{ + "path": ensureDirAPIPath(fullPath), + "owner_qid": "0", + } + err := d.cookieRequestForm(ctx, "/file/mkdir", body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openMakeDir(ctx context.Context, fullPath string) (*OpenMkdirResp, error) { + var resp OpenMkdirResp + body := map[string]string{"fname": ensureDirAPIPath(fullPath)} + err := d.openPOST(ctx, "File.mkdir", body, nil, body, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openRename(ctx context.Context, srcName, newName string) error { + signParams := map[string]string{ + "src_name": srcName, + "new_name": newName, + } + return d.openPOST(ctx, "File.rename", signParams, nil, signParams, nil, true) +} + +func (d *Yunpan360) cookieMove(ctx context.Context, srcPath, dstPath string) error { + var resp CookieMoveResp + body := map[string]string{ + "path[]": srcPath, + "newpath": ensureDirAPIPath(dstPath), + } + if err := d.cookieRequestForm(ctx, "/file/move", body, &resp); err != nil { + return err + } + if !resp.Data.IsAsync { + return nil + } + return d.waitCookieAsyncTask(ctx, resp.Data.TaskID) +} + +func (d *Yunpan360) cookieRecycle(ctx context.Context, obj model.Obj) error { + path := apiPathForObj(obj) + if path == "" { + return errors.New("missing object path") + } + ownerQID, err := d.resolveCookieOwnerQID(ctx, obj, false) + if err != nil { + return err + } + + var resp CookieRecycleResp + if err := d.cookieRequestForm(ctx, "/file/recycle", map[string]string{ + "path[]": path, + "owner_qid": ownerQID, + }, &resp); err != nil { + return err + } + if !resp.Data.IsAsync { + return nil + } + return d.waitCookieAsyncTask(ctx, resp.Data.TaskID, 3008) +} + +func (d *Yunpan360) cookieAsyncQuery(ctx context.Context, taskID string) (*CookieAsyncQueryResp, error) { + var resp CookieAsyncQueryResp + err := d.cookieRequestForm(ctx, "/async/query", map[string]string{ + "task_id": strings.TrimSpace(taskID), + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) waitCookieAsyncTask(ctx context.Context, taskID string, toleratedErrnos ...int) error { + taskID = strings.TrimSpace(taskID) + if taskID == "" { + return nil + } + + tolerated := map[int]struct{}{} + for _, errno := range toleratedErrnos { + tolerated[errno] = struct{}{} + } + + for attempt := 0; attempt < 15; attempt++ { + resp, err := d.cookieAsyncQuery(ctx, taskID) + if err == nil && resp != nil { + if task, ok := resp.Data[taskID]; ok { + done, taskErr := checkCookieAsyncTask(task, tolerated) + if done { + return taskErr + } + } + } + if attempt == 14 { + break + } + if err := sleepWithContext(ctx, 300*time.Millisecond); err != nil { + return err + } + } + + // Keep prior behavior when the async task is still pending after the probe window. + return nil +} + +func checkCookieAsyncTask(task CookieAsyncTask, toleratedErrnos map[int]struct{}) (bool, error) { + if task.Status != 10 { + return false, nil + } + if task.Errno == 0 { + return true, nil + } + if _, ok := toleratedErrnos[task.Errno]; ok { + return true, nil + } + if strings.TrimSpace(task.Errstr) != "" { + return true, errors.New(task.Errstr) + } + if strings.TrimSpace(task.Action) != "" { + return true, fmt.Errorf("yunpan async task %s failed: errno=%d", task.Action, task.Errno) + } + return true, fmt.Errorf("yunpan async task failed: errno=%d", task.Errno) +} + +func (d *Yunpan360) openMove(ctx context.Context, srcName, dstPath string) error { + signParams := map[string]string{ + "src_name": srcName, + "new_name": dstPath, + } + return d.openPOST(ctx, "File.move", signParams, nil, signParams, nil, true) +} + +func (d *Yunpan360) openDelete(ctx context.Context, targetPath string) error { + return d.openPOST(ctx, "File.delete", nil, nil, map[string]string{ + "fname": targetPath, + }, nil, true) +} + +func apiPathForObj(obj model.Obj) string { + if obj.IsDir() { + return ensureDirAPIPath(obj.GetPath()) + } + return normalizeRemotePath(obj.GetPath()) +} + +func ensureDirSuffix(name string) string { + name = strings.TrimSpace(name) + if name == "" || strings.HasSuffix(name, "/") { + return name + } + return name + "/" +} + +func ensureDirAPIPath(p string) string { + p = normalizeRemotePath(p) + if p == "" || p == "/" { + return "/" + } + return p + "/" +} + +func cloneObj(src model.Obj, newPath, newName string) model.Obj { + obj := model.Object{ + ID: src.GetID(), + Path: normalizeRemotePath(newPath), + Name: newName, + Size: src.GetSize(), + Modified: src.ModTime(), + Ctime: src.CreateTime(), + IsFolder: src.IsDir(), + HashInfo: src.GetHash(), + } + if raw, ok := src.(*YunpanObject); ok { + return &YunpanObject{ + Object: obj, + Thumbnail: raw.Thumbnail, + OwnerQID: raw.OwnerQID, + DownloadToken: raw.DownloadToken, + } + } + return &obj +} + +func absoluteURL(raw string) string { + if raw == "" { + return "" + } + if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + return raw + } + if strings.HasPrefix(raw, "/") { + return baseURL + raw + } + return baseURL + "/" + raw +} diff --git a/drivers/yunpan360/util_test.go b/drivers/yunpan360/util_test.go new file mode 100644 index 00000000000..987ca7f928c --- /dev/null +++ b/drivers/yunpan360/util_test.go @@ -0,0 +1,109 @@ +package yunpan360 + +import "testing" + +func TestOwnerQIDFromToken(t *testing.T) { + token := "3061246061.9.cb696851.679627082.16700351802400171.1774866942" + if got := ownerQIDFromToken(token); got != "679627082" { + t.Fatalf("ownerQIDFromToken() = %q, want %q", got, "679627082") + } +} + +func TestParseCookieDownloadSessionFromText(t *testing.T) { + page := `window.__NUXT__={"token":"3061246061.9.cb696851.679627082.16700351802400171.1774866942","owner_qid":"679627082"}` + session := parseCookieDownloadSessionFromText(page) + if session == nil { + t.Fatal("parseCookieDownloadSessionFromText() = nil") + } + if session.OwnerQID != "679627082" { + t.Fatalf("OwnerQID = %q, want %q", session.OwnerQID, "679627082") + } + if session.Token == "" { + t.Fatal("Token should not be empty") + } +} + +func TestCookieListRespObjectsCarrySession(t *testing.T) { + resp := CookieListResp{ + Token: "3061246061.9.cb696851.679627082.16700351802400171.1774866942", + OwnerQid: "679627082", + Data: []ListItem{{ + NID: "17748755101917705", + FileName: "统计4.mp4", + FilePath: "/统计4.mp4", + FileSize: "1024", + }}, + } + + objs := resp.Objects("/") + if len(objs) != 1 { + t.Fatalf("len(objs) = %d, want 1", len(objs)) + } + + obj, ok := objs[0].(*YunpanObject) + if !ok { + t.Fatalf("object type = %T, want *YunpanObject", objs[0]) + } + if obj.OwnerQID != "679627082" { + t.Fatalf("OwnerQID = %q, want %q", obj.OwnerQID, "679627082") + } + if obj.DownloadToken == "" { + t.Fatal("DownloadToken should not be empty") + } +} + +func TestResolveCookieOwnerQIDFromObject(t *testing.T) { + d := &Yunpan360{} + obj := &YunpanObject{ + OwnerQID: "679627082", + } + got, err := d.resolveCookieOwnerQID(t.Context(), obj, false) + if err != nil { + t.Fatalf("resolveCookieOwnerQID() error = %v", err) + } + if got != "679627082" { + t.Fatalf("resolveCookieOwnerQID() = %q, want %q", got, "679627082") + } +} + +func TestCheckCookieAsyncTaskSuccess(t *testing.T) { + done, err := checkCookieAsyncTask(CookieAsyncTask{ + Status: 10, + Action: "File.move", + Errno: 0, + }, nil) + if !done { + t.Fatal("checkCookieAsyncTask() should be done") + } + if err != nil { + t.Fatalf("checkCookieAsyncTask() error = %v", err) + } +} + +func TestCheckCookieAsyncTaskPending(t *testing.T) { + done, err := checkCookieAsyncTask(CookieAsyncTask{ + Status: 1, + Action: "File.move", + }, nil) + if done { + t.Fatal("checkCookieAsyncTask() should be pending") + } + if err != nil { + t.Fatalf("checkCookieAsyncTask() error = %v", err) + } +} + +func TestCheckCookieAsyncTaskIgnoredErrno(t *testing.T) { + done, err := checkCookieAsyncTask(CookieAsyncTask{ + Status: 10, + Action: "File.recycle", + Errno: 3008, + Errstr: "文件（夹）已移动或删除！", + }, map[int]struct{}{3008: {}}) + if !done { + t.Fatal("checkCookieAsyncTask() should be done") + } + if err != nil { + t.Fatalf("checkCookieAsyncTask() error = %v", err) + } +} From 0fe31e042f701146734f572c32aa98fdb40e482a Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Tue, 31 Mar 2026 16:28:19 +0800 Subject: [PATCH 627/659] chore: remove yunpan360 driver tests --- drivers/yunpan360/upload_test.go | 25 ------- drivers/yunpan360/util_test.go | 109 ------------------------------- 2 files changed, 134 deletions(-) delete mode 100644 drivers/yunpan360/upload_test.go delete mode 100644 drivers/yunpan360/util_test.go diff --git a/drivers/yunpan360/upload_test.go b/drivers/yunpan360/upload_test.go deleted file mode 100644 index 092570ff5e8..00000000000 --- a/drivers/yunpan360/upload_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package yunpan360 - -import ( - "bytes" - "testing" - "time" - - "github.com/alist-org/alist/v3/internal/model" -) - -func TestBuildUploadPlanComputesFileSHA1AndMD5(t *testing.T) { - d := &Yunpan360{} - data := []byte("hello yunpan") - file := model.NewNopMFile(bytes.NewReader(data)) - plan, err := d.buildUploadPlan(t.Context(), file, "/hello.txt", int64(len(data)), time.Unix(1700000000, 0), nil) - if err != nil { - t.Fatalf("buildUploadPlan() error = %v", err) - } - if plan.FileSHA1 != "254ec33af17332a3964145f8c6a3dc12833c7ea2" { - t.Fatalf("FileSHA1 = %q", plan.FileSHA1) - } - if plan.FileSum != "fefeffa5b6ae9f39851050b44cacfcb1" { - t.Fatalf("FileSum = %q", plan.FileSum) - } -} diff --git a/drivers/yunpan360/util_test.go b/drivers/yunpan360/util_test.go deleted file mode 100644 index 987ca7f928c..00000000000 --- a/drivers/yunpan360/util_test.go +++ /dev/null @@ -1,109 +0,0 @@ -package yunpan360 - -import "testing" - -func TestOwnerQIDFromToken(t *testing.T) { - token := "3061246061.9.cb696851.679627082.16700351802400171.1774866942" - if got := ownerQIDFromToken(token); got != "679627082" { - t.Fatalf("ownerQIDFromToken() = %q, want %q", got, "679627082") - } -} - -func TestParseCookieDownloadSessionFromText(t *testing.T) { - page := `window.__NUXT__={"token":"3061246061.9.cb696851.679627082.16700351802400171.1774866942","owner_qid":"679627082"}` - session := parseCookieDownloadSessionFromText(page) - if session == nil { - t.Fatal("parseCookieDownloadSessionFromText() = nil") - } - if session.OwnerQID != "679627082" { - t.Fatalf("OwnerQID = %q, want %q", session.OwnerQID, "679627082") - } - if session.Token == "" { - t.Fatal("Token should not be empty") - } -} - -func TestCookieListRespObjectsCarrySession(t *testing.T) { - resp := CookieListResp{ - Token: "3061246061.9.cb696851.679627082.16700351802400171.1774866942", - OwnerQid: "679627082", - Data: []ListItem{{ - NID: "17748755101917705", - FileName: "统计4.mp4", - FilePath: "/统计4.mp4", - FileSize: "1024", - }}, - } - - objs := resp.Objects("/") - if len(objs) != 1 { - t.Fatalf("len(objs) = %d, want 1", len(objs)) - } - - obj, ok := objs[0].(*YunpanObject) - if !ok { - t.Fatalf("object type = %T, want *YunpanObject", objs[0]) - } - if obj.OwnerQID != "679627082" { - t.Fatalf("OwnerQID = %q, want %q", obj.OwnerQID, "679627082") - } - if obj.DownloadToken == "" { - t.Fatal("DownloadToken should not be empty") - } -} - -func TestResolveCookieOwnerQIDFromObject(t *testing.T) { - d := &Yunpan360{} - obj := &YunpanObject{ - OwnerQID: "679627082", - } - got, err := d.resolveCookieOwnerQID(t.Context(), obj, false) - if err != nil { - t.Fatalf("resolveCookieOwnerQID() error = %v", err) - } - if got != "679627082" { - t.Fatalf("resolveCookieOwnerQID() = %q, want %q", got, "679627082") - } -} - -func TestCheckCookieAsyncTaskSuccess(t *testing.T) { - done, err := checkCookieAsyncTask(CookieAsyncTask{ - Status: 10, - Action: "File.move", - Errno: 0, - }, nil) - if !done { - t.Fatal("checkCookieAsyncTask() should be done") - } - if err != nil { - t.Fatalf("checkCookieAsyncTask() error = %v", err) - } -} - -func TestCheckCookieAsyncTaskPending(t *testing.T) { - done, err := checkCookieAsyncTask(CookieAsyncTask{ - Status: 1, - Action: "File.move", - }, nil) - if done { - t.Fatal("checkCookieAsyncTask() should be pending") - } - if err != nil { - t.Fatalf("checkCookieAsyncTask() error = %v", err) - } -} - -func TestCheckCookieAsyncTaskIgnoredErrno(t *testing.T) { - done, err := checkCookieAsyncTask(CookieAsyncTask{ - Status: 10, - Action: "File.recycle", - Errno: 3008, - Errstr: "文件（夹）已移动或删除！", - }, map[int]struct{}{3008: {}}) - if !done { - t.Fatal("checkCookieAsyncTask() should be done") - } - if err != nil { - t.Fatalf("checkCookieAsyncTask() error = %v", err) - } -} From 97a4bb39a059cdc0ee147b6f4159644730322e39 Mon Sep 17 00:00:00 2001 From: icebear0828 Date: Sun, 5 Apr 2026 12:17:41 -0500 Subject: [PATCH 628/659] fix: enable GFM extension for markdown rendering in proxy mode When `filter_readme_scripts` is enabled, markdown files served through the proxy are converted to HTML using goldmark. However, the default goldmark converter does not include GFM (GitHub Flavored Markdown) extensions, causing tables, strikethrough, and other GFM syntax to be rendered as plain text instead of proper HTML elements. This adds `extension.GFM` to the goldmark converter so that GFM tables, strikethrough, autolinks, and task lists are correctly converted to HTML. --- server/handles/down.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/handles/down.go b/server/handles/down.go index 37439f00bb3..680c33f54b1 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -18,6 +18,7 @@ import ( "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" ) func Down(c *gin.Context) { @@ -151,7 +152,8 @@ func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange boo } var html bytes.Buffer - if err = goldmark.Convert(buf.Bytes(), &html); err != nil { + md := goldmark.New(goldmark.WithExtensions(extension.GFM)) + if err = md.Convert(buf.Bytes(), &html); err != nil { err = fmt.Errorf("markdown conversion failed: %w", err) } else { buf.Reset() From 9b697e452581b4da75aca0f1385f56511d0717d8 Mon Sep 17 00:00:00 2001 From: darkizone Date: Sun, 12 Apr 2026 20:05:31 +0100 Subject: [PATCH 629/659] feat(driver): add Darkibox driver Add driver for Darkibox (https://darkibox.com/), an XFileSharing-based video hosting platform. Supported operations: - List files and folders (with pagination) - Upload files (via upload server) - Download files (via direct link API with quality selection) - Create and delete folders - Delete and move files - Account validation on init Closes #8079 Co-Authored-By: Claude Opus 4.6 (1M context) --- drivers/all.go | 1 + drivers/darkibox/driver.go | 299 +++++++++++++++++++++++++++++++++++++ drivers/darkibox/meta.go | 27 ++++ drivers/darkibox/types.go | 79 ++++++++++ drivers/darkibox/util.go | 88 +++++++++++ 5 files changed, 494 insertions(+) create mode 100644 drivers/darkibox/driver.go create mode 100644 drivers/darkibox/meta.go create mode 100644 drivers/darkibox/types.go create mode 100644 drivers/darkibox/util.go diff --git a/drivers/all.go b/drivers/all.go index 1f096a86224..e62f67fbd85 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -27,6 +27,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/cloudreve" _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" + _ "github.com/alist-org/alist/v3/drivers/darkibox" _ "github.com/alist-org/alist/v3/drivers/doubao" _ "github.com/alist-org/alist/v3/drivers/doubao_new" _ "github.com/alist-org/alist/v3/drivers/doubao_share" diff --git a/drivers/darkibox/driver.go b/drivers/darkibox/driver.go new file mode 100644 index 00000000000..0e3f2e5065b --- /dev/null +++ b/drivers/darkibox/driver.go @@ -0,0 +1,299 @@ +package darkibox + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" +) + +type Darkibox struct { + model.Storage + Addition +} + +func (d *Darkibox) Config() driver.Config { + return config +} + +func (d *Darkibox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Darkibox) Init(ctx context.Context) error { + if d.APIKey == "" { + return fmt.Errorf("API key is required") + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + + // Verify API key by calling account/info + var account accountInfoResult + if err := d.callAPI(ctx, "/account/info", nil, &account); err != nil { + return fmt.Errorf("failed to verify API key: %w", err) + } + + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Darkibox) Drop(ctx context.Context) error { + return nil +} + +func (d *Darkibox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = folderIDFromObjID(dir.GetID()) + } + + var objects []model.Obj + + // List sub-folders via /api/folder/list + var folders folderListResult + if err := d.callAPI(ctx, "/folder/list", map[string]string{ + "fld_id": fldIDStr(folderID), + }, &folders); err != nil { + return nil, fmt.Errorf("list folders failed: %w", err) + } + for _, f := range folders.Folders { + objects = append(objects, &model.Object{ + ID: encodeFolderID(f.FldID), + Name: f.Name, + IsFolder: true, + }) + } + + // List files via /api/file/list (paginated) + page := 1 + for { + var files fileListResult + if err := d.callAPI(ctx, "/file/list", map[string]string{ + "fld_id": fldIDStr(folderID), + "per_page": "200", + "page": strconv.Itoa(page), + }, &files); err != nil { + return nil, fmt.Errorf("list files failed: %w", err) + } + + for _, f := range files.Files { + modified := time.Now() + if f.Uploaded != "" { + if t, err := time.Parse("2006-01-02 15:04:05", f.Uploaded); err == nil { + modified = t + } + } + name := f.Name + if name == "" { + name = f.Title + } + objects = append(objects, &model.Object{ + ID: encodeFileID(f.FileCode), + Name: name, + Size: f.Size, + Modified: modified, + IsFolder: false, + }) + } + + // Check if there are more pages + if len(files.Files) < 200 { + break + } + page++ + } + + return objects, nil +} + +func (d *Darkibox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + + fileCode := fileCodeFromObjID(file.GetID()) + if fileCode == "" { + return nil, fmt.Errorf("empty file code") + } + + var result directLinkResult + if err := d.callAPI(ctx, "/file/direct_link", map[string]string{ + "file_code": fileCode, + }, &result); err != nil { + return nil, fmt.Errorf("failed to get direct link: %w", err) + } + + // Find the original quality version, fall back to first available + var dlURL string + for _, v := range result.Versions { + if v.Name == "o" { + dlURL = v.URL + break + } + } + if dlURL == "" && len(result.Versions) > 0 { + dlURL = result.Versions[0].URL + } + if dlURL == "" { + return nil, fmt.Errorf("no download URL available for file %s", fileCode) + } + + return &model.Link{ + URL: dlURL, + }, nil +} + +func (d *Darkibox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID := d.RootFolderID + if parentDir.GetID() != "" { + parentID = folderIDFromObjID(parentDir.GetID()) + } + + var result folderCreateResult + if err := d.callAPI(ctx, "/folder/create", map[string]string{ + "name": dirName, + "parent_id": fldIDStr(parentID), + }, &result); err != nil { + return nil, fmt.Errorf("create folder failed: %w", err) + } + + return &model.Object{ + ID: encodeFolderID(result.FldID), + Name: dirName, + IsFolder: true, + }, nil +} + +func (d *Darkibox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, errs.NotImplement + } + + fileCode := fileCodeFromObjID(srcObj.GetID()) + if fileCode == "" { + return nil, fmt.Errorf("empty file code") + } + + dstFolderID := d.RootFolderID + if dstDir.GetID() != "" { + dstFolderID = folderIDFromObjID(dstDir.GetID()) + } + + if err := d.callAPI(ctx, "/file/move", map[string]string{ + "file_code": fileCode, + "to_folder": fldIDStr(dstFolderID), + }, nil); err != nil { + return nil, fmt.Errorf("move file failed: %w", err) + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: false, + }, nil +} + +func (d *Darkibox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + folderID := folderIDFromObjID(obj.GetID()) + return d.callAPI(ctx, "/folder/delete", map[string]string{ + "fld_id": folderID, + }, nil) + } + + fileCode := fileCodeFromObjID(obj.GetID()) + return d.callAPI(ctx, "/file/delete", map[string]string{ + "file_code": fileCode, + }, nil) +} + +func (d *Darkibox) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + // Step 1: Get the upload server URL + var server uploadServerResult + if err := d.callAPI(ctx, "/upload/server", nil, &server); err != nil { + return nil, fmt.Errorf("get upload server failed: %w", err) + } + if server.URL == "" { + return nil, fmt.Errorf("no upload server URL returned") + } + + // Step 2: Upload the file to the upload server + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) + + res, err := base.RestyClient.R(). + SetContext(ctx). + SetMultipartField("file", file.GetName(), "", reader). + SetMultipartFormData(map[string]string{ + "key": d.APIKey, + "fld_id": fldIDStr(folderID), + }). + Post(server.URL) + if err != nil { + return nil, fmt.Errorf("upload failed: %w", err) + } + if res.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("upload failed: http %d", res.StatusCode()) + } + + // Try to parse upload response to get the file code + var uploadResp uploadResult + if err := base.RestyClient.JSONUnmarshal(res.Body(), &uploadResp); err == nil && len(uploadResp.Files) > 0 { + uf := uploadResp.Files[0] + return &model.Object{ + ID: encodeFileID(uf.FileCode), + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil + } + + return &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Darkibox) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Darkibox)(nil) diff --git a/drivers/darkibox/meta.go b/drivers/darkibox/meta.go new file mode 100644 index 00000000000..a09707707fd --- /dev/null +++ b/drivers/darkibox/meta.go @@ -0,0 +1,27 @@ +package darkibox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APIKey string `json:"api_key" required:"true" help:"API key from your Darkibox account"` +} + +var config = driver.Config{ + Name: "Darkibox", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Darkibox{} + }) +} diff --git a/drivers/darkibox/types.go b/drivers/darkibox/types.go new file mode 100644 index 00000000000..9f193f97e4f --- /dev/null +++ b/drivers/darkibox/types.go @@ -0,0 +1,79 @@ +package darkibox + +import "encoding/json" + +// apiResponse is the common wrapper for all Darkibox API responses. +type apiResponse struct { + Msg string `json:"msg"` + Result json.RawMessage `json:"result"` + ServerTime string `json:"server_time"` + Status int `json:"status"` +} + +// accountInfoResult represents the result of /api/account/info +type accountInfoResult struct { + Email string `json:"email"` + Balance string `json:"balance"` + StorageUsed string `json:"storage_used"` +} + +// fileListResult represents the result of /api/file/list +type fileListResult struct { + Results int `json:"results"` + ResultsTotal int `json:"results_total"` + Files []fileItem `json:"files"` +} + +type fileItem struct { + FileCode string `json:"file_code"` + Name string `json:"name"` + Title string `json:"title"` + Size int64 `json:"size"` + Uploaded string `json:"uploaded"` + FldID int64 `json:"fld_id"` +} + +// folderListResult represents the result of /api/folder/list +type folderListResult struct { + Folders []folderItem `json:"folders"` +} + +type folderItem struct { + FldID int64 `json:"fld_id"` + Name string `json:"name"` + Code string `json:"code"` +} + +// directLinkResult represents the result of /api/file/direct_link +type directLinkResult struct { + Versions []directLinkVersion `json:"versions"` +} + +type directLinkVersion struct { + Name string `json:"name"` + URL string `json:"url"` +} + +// uploadServerResult represents the result of /api/upload/server +type uploadServerResult struct { + URL string `json:"url"` +} + +// uploadResult represents the response from the upload endpoint +type uploadResult struct { + Files []uploadedFile `json:"files"` +} + +type uploadedFile struct { + FileCode string `json:"filecode"` + URL string `json:"url"` + Name string `json:"name"` + Size int64 `json:"size"` + Status int `json:"status"` +} + +// folderCreateResult represents the result of /api/folder/create +type folderCreateResult struct { + FldID int64 `json:"fld_id"` + Name string `json:"name"` +} diff --git a/drivers/darkibox/util.go b/drivers/darkibox/util.go new file mode 100644 index 00000000000..0032abab106 --- /dev/null +++ b/drivers/darkibox/util.go @@ -0,0 +1,88 @@ +package darkibox + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" +) + +const apiBase = "https://darkibox.com/api" + +// callAPI makes a GET request to the Darkibox API with the given endpoint and params. +// It automatically injects the API key. The result JSON is unmarshalled into out if non-nil. +func (d *Darkibox) callAPI(ctx context.Context, endpoint string, params map[string]string, out any) error { + query := map[string]string{ + "key": d.APIKey, + } + for k, v := range params { + if strings.TrimSpace(v) == "" { + continue + } + query[k] = v + } + + var resp apiResponse + r, err := base.RestyClient.R(). + SetContext(ctx). + SetQueryParams(query). + SetResult(&resp). + Get(apiBase + endpoint) + if err != nil { + return err + } + if r.StatusCode() != http.StatusOK { + return fmt.Errorf("darkibox http error: %d", r.StatusCode()) + } + if resp.Status != 200 { + return fmt.Errorf("darkibox api error: status=%d msg=%s", resp.Status, resp.Msg) + } + if out == nil || len(resp.Result) == 0 || string(resp.Result) == "null" { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("decode darkibox result failed: %w", err) + } + return nil +} + +// fldIDStr converts a folder ID (which may be the root "0") to a string suitable for API params. +func fldIDStr(id string) string { + if id == "" { + return "0" + } + return id +} + +// encodeFolderID prefixes a folder ID so we can distinguish folders from files. +func encodeFolderID(id int64) string { + return "d:" + strconv.FormatInt(id, 10) +} + +// encodeFileID prefixes a file code so we can distinguish files from folders. +func encodeFileID(code string) string { + return "f:" + code +} + +// folderIDFromObjID extracts the numeric folder ID string from an object ID. +func folderIDFromObjID(id string) string { + if strings.HasPrefix(id, "d:") { + return strings.TrimPrefix(id, "d:") + } + if id == "" || id == "/" { + return "0" + } + return id +} + +// fileCodeFromObjID extracts the file code from an object ID. +func fileCodeFromObjID(id string) string { + if strings.HasPrefix(id, "f:") { + return strings.TrimPrefix(id, "f:") + } + return id +} From 06cb5ee555e6c0936d7b0780bf9a563e9aba8ea8 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 15 Apr 2026 19:19:49 +0800 Subject: [PATCH 630/659] feat(guangyapan): add full GuangYaPan driver integration Implement GuangYaPan storage adapter and register driver. Includes: - SMS/captcha login flow with token refresh - list/link operations - mkdir/rename/remove/move/copy - upload via res_center token + OSS multipart + task polling - compatibility fixes for provider type, endpoint normalization, and upload status codes --- drivers/all.go | 1 + drivers/guangyapan/driver.go | 950 +++++++++++++++++++++++++++++++++++ drivers/guangyapan/meta.go | 42 ++ drivers/guangyapan/types.go | 141 ++++++ 4 files changed, 1134 insertions(+) create mode 100644 drivers/guangyapan/driver.go create mode 100644 drivers/guangyapan/meta.go create mode 100644 drivers/guangyapan/types.go diff --git a/drivers/all.go b/drivers/all.go index fe375586a2d..3dc90424cbe 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -42,6 +42,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/gofile" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" + _ "github.com/alist-org/alist/v3/drivers/guangyapan" _ "github.com/alist-org/alist/v3/drivers/halalcloud" _ "github.com/alist-org/alist/v3/drivers/ilanzou" _ "github.com/alist-org/alist/v3/drivers/ipfs_api" diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go new file mode 100644 index 00000000000..ff83c9a88fa --- /dev/null +++ b/drivers/guangyapan/driver.go @@ -0,0 +1,950 @@ +package guangyapan + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +const ( + accountBaseURL = "https://account.guangyapan.com" + apiBaseURL = "https://api.guangyapan.com" + defaultClient = "aMe-8VSlkrbQXpUR" +) + +type GuangYaPan struct { + model.Storage + Addition + + accountClient *resty.Client + apiClient *resty.Client +} + +func (d *GuangYaPan) Config() driver.Config { + return config +} + +func (d *GuangYaPan) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GuangYaPan) Init(ctx context.Context) error { + d.ClientID = strings.TrimSpace(d.ClientID) + if d.ClientID == "" { + d.ClientID = defaultClient + } + d.DeviceID = normalizeDeviceID(d.DeviceID) + if d.DeviceID == "" { + d.DeviceID = randomDeviceID() + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + if d.OrderBy < 0 { + d.OrderBy = 3 + } + if d.SortType != 0 && d.SortType != 1 { + d.SortType = 1 + } + + d.AccessToken = strings.TrimSpace(d.AccessToken) + d.RefreshToken = strings.TrimSpace(d.RefreshToken) + d.PhoneNumber = strings.TrimSpace(d.PhoneNumber) + d.VerifyCode = strings.TrimSpace(d.VerifyCode) + d.CaptchaToken = strings.TrimSpace(d.CaptchaToken) + d.VerificationID = strings.TrimSpace(d.VerificationID) + + d.accountClient = base.NewRestyClient(). + SetBaseURL(accountBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("X-Device-Model", "chrome%2F147.0.0.0"). + SetHeader("X-Device-Name", "PC-Chrome"). + SetHeader("X-Device-Sign", "wdi10."+d.DeviceID+"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"). + SetHeader("X-Net-Work-Type", "NONE"). + SetHeader("X-OS-Version", "MacIntel"). + SetHeader("X-Platform-Version", "1"). + SetHeader("X-Protocol-Version", "301"). + SetHeader("X-Provider-Name", "NONE"). + SetHeader("X-SDK-Version", "9.0.2"). + SetHeader("X-Client-Id", d.ClientID). + SetHeader("X-Client-Version", "0.0.1"). + SetHeader("X-Device-Id", d.DeviceID) + if d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + d.apiClient = base.NewRestyClient(). + SetBaseURL(apiBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("Did", d.DeviceID). + SetHeader("Dt", "4") + + // Priority: access_token -> refresh_token -> sms login. + if d.AccessToken != "" { + if err := d.validateToken(ctx); err == nil { + return nil + } + d.AccessToken = "" + } + if d.RefreshToken != "" { + if err := d.refreshToken(ctx); err == nil { + if err2 := d.validateToken(ctx); err2 == nil { + return nil + } + } + } + // Two-stage SMS flow: + // 1) phone only + send_code=true: send code and cache verification_id (do not fail init). + // 2) phone + verify_code: complete login and save tokens. + if d.PhoneNumber != "" { + if d.canSMSLogin() { + if err := d.loginBySMSCode(ctx); err != nil { + return err + } + return d.validateToken(ctx) + } + if d.SendCode { + d.setTempStatus("SMS sending in progress...") + if err := d.prepareSMSCode(ctx); err != nil { + d.setTempStatus(fmt.Sprintf("SMS send failed: %v. Please check captcha/meta and set send_code=true to retry.", err)) + log.Warnf("guangyapan: prepare sms code failed: %v", err) + } else { + d.setTempStatus("SMS sent successfully. Please fill verify_code and save to complete login.") + } + } + return nil + } + return errors.New("login failed: provide a valid access_token, or refresh_token, or phone_number + verify_code + captcha_token") +} + +func (d *GuangYaPan) Drop(ctx context.Context) error { + return nil +} + +func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + parentID := dir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + res := make([]model.Obj, 0, d.PageSize) + for page := 0; ; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": d.PageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + "fileTypes": []int{}, + } + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + return nil, err + } + for _, item := range resp.Data.List { + res = append(res, &model.Object{ + ID: item.FileID, + Path: parentID, + Name: item.FileName, + Size: item.FileSize, + Modified: unixOrZero(item.UTime), + Ctime: unixOrZero(item.CTime), + IsFolder: item.ResType == 2, + }) + } + if len(resp.Data.List) < d.PageSize { + break + } + if resp.Data.Total > 0 && len(res) >= resp.Data.Total { + break + } + } + return res, nil +} + +func (d *GuangYaPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + var resp downloadResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_download_url", map[string]any{ + "fileId": file.GetID(), + }, &resp); err != nil { + return nil, err + } + + url := strings.TrimSpace(resp.Data.SignedURL) + if url == "" { + url = strings.TrimSpace(resp.Data.DownloadURL) + } + if url == "" { + return nil, errors.New("empty download url") + } + return &model.Link{URL: url}, nil +} + +func (d *GuangYaPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + name := strings.TrimSpace(dirName) + if name == "" { + return errors.New("dir name is empty") + } + + parentID := parentDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + var out createDirResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/create_dir", map[string]any{ + "parentId": parentID, + "dirName": name, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("make dir failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + name := strings.TrimSpace(newName) + if name == "" { + return errors.New("new name is empty") + } + + var out commonResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/rename", map[string]any{ + "fileId": fileID, + "newName": name, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("rename failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Remove(ctx context.Context, obj model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(obj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + + var del deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/delete_file", map[string]any{ + "fileIds": []string{fileID}, + }, &del); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(del.Msg), "success") { + return fmt.Errorf("delete failed: %s", strings.TrimSpace(del.Msg)) + } + + taskID := strings.TrimSpace(del.Data.TaskID) + if taskID == "" { + // Some backends may apply deletion synchronously. + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + var out deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/move_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("move failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + var out deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/copy_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("copy failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if file == nil { + return errors.New("file is nil") + } + if file.GetSize() < 0 { + return errors.New("invalid file size") + } + name := strings.TrimSpace(file.GetName()) + if name == "" { + return errors.New("file name is empty") + } + + parentID := dstDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + token, code, err := d.getUploadToken(ctx, parentID, name, file.GetSize()) + if err != nil { + return err + } + taskID := strings.TrimSpace(token.TaskID) + if code == 156 { + if taskID == "" { + return errors.New("instant upload returns empty task id") + } + return d.waitUploadTaskInfo(ctx, taskID) + } + + if token.ObjectPath == "" || token.BucketName == "" || token.EndPoint == "" || token.AccessKeyID == "" || token.SecretAccessKey == "" { + return errors.New("upload token is incomplete") + } + + ossEndpoint := normalizeOSSEndpoint(token.EndPoint, token.BucketName) + client, err := oss.New(ossEndpoint, token.AccessKeyID, token.SecretAccessKey, oss.SecurityToken(token.SessionToken)) + if err != nil { + return fmt.Errorf("create oss client failed: %w", err) + } + bucket, err := client.Bucket(token.BucketName) + if err != nil { + return fmt.Errorf("create oss bucket failed: %w", err) + } + + if file.GetSize() == 0 { + if err := bucket.PutObject(token.ObjectPath, strings.NewReader("")); err != nil { + return err + } + } else { + if err := d.multipartUploadToOSS(ctx, bucket, token.ObjectPath, file, up); err != nil { + return err + } + } + + if taskID == "" { + return nil + } + return d.waitUploadTaskInfo(ctx, taskID) +} + +func (d *GuangYaPan) ensureAccessToken(ctx context.Context) error { + if strings.TrimSpace(d.AccessToken) != "" { + return nil + } + if strings.TrimSpace(d.RefreshToken) == "" { + if d.canSMSLogin() { + return d.loginBySMSCode(ctx) + } + if d.PhoneNumber != "" { + return errors.New("not logged in yet: please fill verify_code and save storage to finish SMS login") + } + return errors.New("access token is empty") + } + return d.refreshToken(ctx) +} + +func (d *GuangYaPan) validateToken(ctx context.Context) error { + var me userMeResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetResult(&me). + Get("/v1/user/me") + if err != nil { + return err + } + if resp.IsError() { + return fmt.Errorf("validate token failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if strings.TrimSpace(me.Sub) == "" { + return errors.New("validate token failed: empty user sub") + } + return nil +} + +func (d *GuangYaPan) refreshToken(ctx context.Context) error { + if strings.TrimSpace(d.RefreshToken) == "" { + return errors.New("refresh_token is empty") + } + + var out tokenResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&out). + Post("/v1/auth/token") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + errMsg := strings.TrimSpace(out.ErrorDesc) + if errMsg == "" { + errMsg = strings.TrimSpace(out.Error) + } + if errMsg == "" { + errMsg = strings.TrimSpace(resp.String()) + } + if errMsg == "" { + errMsg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + return fmt.Errorf("refresh token failed: %s", errMsg) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + if strings.TrimSpace(out.RefreshToken) != "" { + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + } + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) canSMSLogin() bool { + return d.PhoneNumber != "" && d.VerifyCode != "" +} + +func (d *GuangYaPan) loginBySMSCode(ctx context.Context) error { + verificationID := strings.TrimSpace(d.VerificationID) + if verificationID == "" { + var err error + verificationID, err = d.requestVerificationID(ctx) + if err != nil { + return err + } + } + + var step2 verifyResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_id": verificationID, + "verification_code": d.VerifyCode, + "client_id": d.ClientID, + }). + SetResult(&step2). + Post("/v1/auth/verification/verify") + if err != nil { + return err + } + if resp.IsError() || step2.Error != "" || strings.TrimSpace(step2.VerificationToken) == "" { + return fmt.Errorf("verify code failed: %s", d.accountErr(step2.ErrorDesc, step2.Error, resp)) + } + + var out tokenResp + resp, err = d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_code": d.VerifyCode, + "verification_token": step2.VerificationToken, + "username": normalizePhoneE164(d.PhoneNumber), + "client_id": d.ClientID, + }). + SetResult(&out). + Post("/v1/auth/signin") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + return fmt.Errorf("signin failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + d.VerificationID = "" + // One-time SMS code should not be reused after successful login. + d.VerifyCode = "" + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) prepareSMSCode(ctx context.Context) error { + // Explicit send action should always refresh verification_id. + d.VerificationID = "" + if err := d.ensureCaptchaToken(ctx, false); err != nil { + return err + } + verificationID, err := d.requestVerificationID(ctx) + if err != nil { + return err + } + d.VerificationID = verificationID + d.SendCode = false + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) requestVerificationID(ctx context.Context) (string, error) { + if d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + var step1 verificationResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "phone_number": normalizePhoneE164(d.PhoneNumber), + "target": "ANY", + "client_id": d.ClientID, + }). + SetResult(&step1). + Post("/v1/auth/verification") + if err != nil { + return "", err + } + if resp.IsError() || step1.Error != "" || strings.TrimSpace(step1.VerificationID) == "" { + // If captcha token is expired/invalid, refresh it once and retry. + if strings.Contains(step1.Error, "captcha_invalid") || strings.Contains(step1.ErrorDesc, "captcha_token expired") { + if err := d.ensureCaptchaToken(ctx, true); err == nil { + return d.requestVerificationID(ctx) + } + } + return "", fmt.Errorf("request verification failed: %s", d.accountErr(step1.ErrorDesc, step1.Error, resp)) + } + return strings.TrimSpace(step1.VerificationID), nil +} + +func (d *GuangYaPan) ensureCaptchaToken(ctx context.Context, force bool) error { + if !force && d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + return nil + } + + var out captchaInitResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "action": "POST:/v1/auth/verification", + "device_id": d.DeviceID, + "meta": map[string]any{ + "username": normalizePhoneE164(d.PhoneNumber), + "phone_number": normalizePhoneE164(d.PhoneNumber), + "VERIFICATION_PHONE": normalizePhoneE164(d.PhoneNumber), + }, + }). + SetResult(&out). + Post("/v1/shield/captcha/init") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.CaptchaToken) == "" { + return fmt.Errorf("init captcha token failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + d.CaptchaToken = strings.TrimSpace(out.CaptchaToken) + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + op.MustSaveDriverStorage(d) + return nil +} + +func normalizeCaptchaUsername(phone string) string { + p := strings.TrimSpace(phone) + p = strings.ReplaceAll(p, " ", "") + p = strings.TrimPrefix(p, "+") + // Keep only digits. + b := make([]rune, 0, len(p)) + for _, ch := range p { + if ch >= '0' && ch <= '9' { + b = append(b, ch) + } + } + digits := string(b) + // Mainland number normalization: +86xxxxxxxxxxx -> xxxxxxxxxxx + if strings.HasPrefix(digits, "86") && len(digits) > 11 { + digits = digits[2:] + } + return digits +} + +func normalizePhoneE164(phone string) string { + p := strings.TrimSpace(phone) + if p == "" { + return "" + } + p = strings.ReplaceAll(p, " ", "") + if strings.HasPrefix(p, "+") { + // Format as "+86 1xxxxxxxxxx" to match browser payload expectations. + if strings.HasPrefix(p, "+86") && len(p) > 3 { + rest := strings.TrimPrefix(p, "+86") + return "+86 " + rest + } + return p + } + // If raw mainland number is provided, normalize with +86 prefix. + digits := normalizeCaptchaUsername(p) + if len(digits) == 11 { + return "+86 " + digits + } + return p +} + +func (d *GuangYaPan) setTempStatus(status string) { + // initStorage sets status to WORK after Init returns, so we update it shortly after. + time.AfterFunc(200*time.Millisecond, func() { + d.GetStorage().SetStatus(status) + op.MustSaveDriverStorage(d) + }) +} + +func (d *GuangYaPan) accountErr(desc, short string, resp *resty.Response) string { + msg := strings.TrimSpace(desc) + if msg == "" { + msg = strings.TrimSpace(short) + } + if msg == "" && resp != nil { + msg = strings.TrimSpace(resp.String()) + } + if msg == "" && resp != nil { + msg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + if msg == "" { + msg = "unknown error" + } + return msg +} + +func (d *GuangYaPan) postAPI(ctx context.Context, path string, body any, out any) error { + if strings.TrimSpace(d.AccessToken) == "" { + return errors.New("access token is empty") + } + resp, err := d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + if resp.StatusCode() == 401 || resp.StatusCode() == 403 { + if strings.TrimSpace(d.RefreshToken) == "" { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if err := d.refreshToken(ctx); err != nil { + return err + } + resp, err = d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + } + if resp.IsError() { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + return nil +} + +func (d *GuangYaPan) waitTaskDone(ctx context.Context, taskID string) error { + const ( + maxTry = 30 + interval = 300 * time.Millisecond + ) + for i := 0; i < maxTry; i++ { + var out taskStatusResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_task_status", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("get task status failed: %s", strings.TrimSpace(out.Msg)) + } + switch out.Data.Status { + case 2: + return nil + case -1, 3: + return fmt.Errorf("task %s failed with status=%d", taskID, out.Data.Status) + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("task %s timeout", taskID) +} + +func (d *GuangYaPan) getUploadToken(ctx context.Context, parentID, name string, size int64) (*uploadTokenData, int, error) { + var out uploadTokenResp + err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_center_token", map[string]any{ + "capacity": 2, + "name": name, + "parentId": parentID, + "res": map[string]any{ + "fileSize": size, + }, + }, &out) + if err != nil { + return nil, 0, err + } + msg := strings.TrimSpace(out.Msg) + if msg != "" && !strings.EqualFold(msg, "success") { + return nil, out.Code, fmt.Errorf("get upload token failed: %s", msg) + } + if out.Data.TaskID == "" { + return nil, out.Code, errors.New("get upload token failed: empty task id") + } + if out.Data.AccessKeyID == "" { + out.Data.AccessKeyID = out.Data.Creds.AccessKeyID + } + if out.Data.SecretAccessKey == "" { + out.Data.SecretAccessKey = out.Data.Creds.SecretAccessKey + } + if out.Data.SessionToken == "" { + out.Data.SessionToken = out.Data.Creds.SessionToken + } + if strings.TrimSpace(out.Data.EndPoint) == "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } + if strings.TrimSpace(out.Data.EndPoint) != "" && !strings.HasPrefix(out.Data.EndPoint, "http://") && !strings.HasPrefix(out.Data.EndPoint, "https://") { + if strings.TrimSpace(out.Data.FullEndPoint) != "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } else if strings.TrimSpace(out.Data.BucketName) != "" { + host := strings.TrimSpace(out.Data.EndPoint) + prefix := strings.TrimSpace(out.Data.BucketName) + "." + if strings.HasPrefix(host, prefix) { + out.Data.EndPoint = "https://" + host + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.BucketName) + "." + host + } + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.EndPoint) + } + } + return &out.Data, out.Code, nil +} + +func (d *GuangYaPan) waitUploadTaskInfo(ctx context.Context, taskID string) error { + const ( + maxTry = 300 + interval = 1 * time.Second + ) + for i := 0; i < maxTry; i++ { + var out taskInfoResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_info_by_task_id", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if out.Data.FileID != "" { + return nil + } + switch out.Code { + case 145, 146, 147, 155, 163, 0: + // uploading/verifying/processing + default: + if strings.TrimSpace(out.Msg) != "" { + return fmt.Errorf("upload task failed: code=%d msg=%s", out.Code, strings.TrimSpace(out.Msg)) + } + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("upload task %s timeout", taskID) +} + +func (d *GuangYaPan) multipartUploadToOSS(ctx context.Context, bucket *oss.Bucket, objectPath string, file model.FileStreamer, up driver.UpdateProgress) error { + partSize := calcUploadPartSize(file.GetSize()) + imur, err := bucket.InitiateMultipartUpload(objectPath, oss.Sequential()) + if err != nil { + return err + } + + total := file.GetSize() + partCount := int((total + partSize - 1) / partSize) + parts := make([]oss.UploadPart, 0, partCount) + var uploaded int64 + partNumber := 1 + + for uploaded < total { + if err := ctx.Err(); err != nil { + return err + } + curPartSize := partSize + left := total - uploaded + if left < curPartSize { + curPartSize = left + } + + reader := io.LimitReader(file, curPartSize) + part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, reader), curPartSize, partNumber) + if err != nil { + return err + } + parts = append(parts, part) + uploaded += curPartSize + partNumber++ + if total > 0 { + up(100 * float64(uploaded) / float64(total)) + } + } + + _, err = bucket.CompleteMultipartUpload(imur, parts) + return err +} + +func calcUploadPartSize(size int64) int64 { + const ( + mb = int64(1024 * 1024) + gb = int64(1024 * 1024 * 1024) + ) + switch { + case size <= 100*mb: + return 1 * mb + case size <= 16*gb: + return 2 * mb + case size <= 160*gb: + return 4 * mb + default: + return 8 * mb + } +} + +func normalizeOSSEndpoint(endpoint, bucket string) string { + ep := strings.TrimSpace(endpoint) + if ep == "" { + return ep + } + if !strings.HasPrefix(ep, "http://") && !strings.HasPrefix(ep, "https://") { + ep = "https://" + ep + } + u, err := url.Parse(ep) + if err != nil || u.Host == "" { + return ep + } + host := u.Host + prefix := strings.TrimSpace(bucket) + if prefix != "" && strings.HasPrefix(host, prefix+".") { + host = strings.TrimPrefix(host, prefix+".") + } + u.Host = host + return u.String() +} + +func normalizeDeviceID(v string) string { + v = strings.ToLower(strings.TrimSpace(v)) + v = strings.ReplaceAll(v, "-", "") + if len(v) != 32 { + return "" + } + for _, ch := range v { + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + return "" + } + } + return v +} + +func randomDeviceID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "0123456789abcdef0123456789abcdef" + } + return hex.EncodeToString(b) +} + +var _ driver.Driver = (*GuangYaPan)(nil) diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go new file mode 100644 index 00000000000..606d6aec8ee --- /dev/null +++ b/drivers/guangyapan/meta.go @@ -0,0 +1,42 @@ +package guangyapan + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + PhoneNumber string `json:"phone_number" type:"text" help:"Phone number for SMS login, e.g. +86 13800000000"` + CaptchaToken string `json:"captcha_token" type:"text" help:"Captcha token required by /v1/auth/verification"` + SendCode bool `json:"send_code" type:"bool" help:"Set true and save to send SMS code, it auto-resets to false after sending"` + VerifyCode string `json:"verify_code" type:"text" help:"SMS verification code used with phone_number; fill then save to finish login"` + VerificationID string `json:"verification_id" type:"text" help:"Auto-generated after sending SMS code; do not edit manually"` + AccessToken string `json:"access_token" type:"text" help:"Bearer access token (optional if refresh_token is provided)"` + RefreshToken string `json:"refresh_token" type:"text" help:"Refresh token for auto-login/auto-refresh"` + ClientID string `json:"client_id" default:"aMe-8VSlkrbQXpUR"` + DeviceID string `json:"device_id" help:"Optional custom device id (32 hex chars), auto-generated when empty"` + PageSize int `json:"page_size" type:"number" default:"100"` + OrderBy int `json:"order_by" type:"number" default:"3" help:"0:name,1:size,2:create_time,3:update_time"` + SortType int `json:"sort_type" type:"number" default:"1" help:"0:asc,1:desc"` +} + +var config = driver.Config{ + Name: "GuangYaPan", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: true, + Alert: "info|Two-stage SMS login: (1) fill phone_number (+ captcha_token if needed), set send_code=true and save; (2) fill verify_code and save to finish login and auto-save access_token/refresh_token.", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GuangYaPan{} + }) +} diff --git a/drivers/guangyapan/types.go b/drivers/guangyapan/types.go new file mode 100644 index 00000000000..bd0094f3070 --- /dev/null +++ b/drivers/guangyapan/types.go @@ -0,0 +1,141 @@ +package guangyapan + +import "time" + +type tokenResp struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + TokenType string `json:"token_type"` + ExpiresIn int64 `json:"expires_in"` + Sub string `json:"sub"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verificationResp struct { + VerificationID string `json:"verification_id"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type captchaInitResp struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verifyResp struct { + VerificationToken string `json:"verification_token"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type userMeResp struct { + Sub string `json:"sub"` +} + +type listResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Total int `json:"total"` + List []fileItem `json:"list"` + } `json:"data"` +} + +type fileItem struct { + FileID string `json:"fileId"` + ParentID string `json:"parentId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` +} + +type downloadResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + SignedURL string `json:"signedURL"` + DownloadURL string `json:"downloadUrl"` + } `json:"data"` +} + +type createDirResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + FileName string `json:"fileName"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` + } `json:"data"` +} + +type commonResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +type deleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + } `json:"data"` +} + +type taskStatusResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Status int `json:"status"` + } `json:"data"` +} + +type uploadTokenResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data uploadTokenData `json:"data"` +} + +type uploadTokenData struct { + TaskID string `json:"taskId"` + ObjectPath string `json:"objectPath"` + Provider any `json:"provider"` + Region string `json:"region"` + BucketName string `json:"bucketName"` + EndPoint string `json:"endPoint"` + FullEndPoint string `json:"fullEndPoint"` + CallbackVar string `json:"callbackVar"` + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + Creds struct { + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + } `json:"creds"` +} + +type taskInfoResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + } `json:"data"` +} + +func unixOrZero(v int64) time.Time { + if v <= 0 { + return time.Time{} + } + return time.Unix(v, 0) +} From ee27244094a24b7fbff12837594ff68baeafb048 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 15 Apr 2026 20:32:42 +0800 Subject: [PATCH 631/659] feat(frp): add runtime log API and stop endpoint --- internal/bootstrap/frp.go | 19 +++ internal/frp/frp.go | 335 ++++++++++++++++++++++++++++++++++++++ server/handles/setting.go | 37 +++++ server/router.go | 3 + 4 files changed, 394 insertions(+) create mode 100644 internal/bootstrap/frp.go create mode 100644 internal/frp/frp.go diff --git a/internal/bootstrap/frp.go b/internal/bootstrap/frp.go new file mode 100644 index 00000000000..b8417843c30 --- /dev/null +++ b/internal/bootstrap/frp.go @@ -0,0 +1,19 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func InitFRP() { + frp.Instance = frp.Init() + if setting.GetBool(conf.FRPEnabled) { + if err := frp.Instance.Start(); err != nil { + utils.Log.Warnf("failed to start frp client: %v", err) + } else { + utils.Log.Info("frp client started") + } + } +} diff --git a/internal/frp/frp.go b/internal/frp/frp.go new file mode 100644 index 00000000000..c244e18b870 --- /dev/null +++ b/internal/frp/frp.go @@ -0,0 +1,335 @@ +package frp + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/cmd/flags" + frpclient "github.com/fatedier/frp/client" + "github.com/fatedier/frp/pkg/config/source" + v1 "github.com/fatedier/frp/pkg/config/v1" + frplog "github.com/fatedier/frp/pkg/util/log" + log "github.com/sirupsen/logrus" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" +) + +// Instance is the global FRP manager. +var Instance *Manager + +// Manager controls the lifecycle of the embedded FRP client. +type Manager struct { + mu sync.Mutex + cancel context.CancelFunc + wg sync.WaitGroup + status string + logs []string + logPath string +} + +const maxLogEntries = 300 +const maxTailBytes = 512 * 1024 + +// RuntimeInfo contains FRP runtime status and recent logs. +type RuntimeInfo struct { + Status string `json:"status"` + Logs []string `json:"logs"` +} + +// Init creates and returns a new Manager. +func Init() *Manager { + m := &Manager{ + status: "stopped", + logPath: filepath.Join(flags.DataDir, "log", "frp.log"), + } + m.logs = append(m.logs, fmt.Sprintf("[%s] initialized", time.Now().Format(time.RFC3339))) + return m +} + +// Start builds the FRP config from settings and starts the client. +func (m *Manager) Start() error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.cancel != nil { + m.appendLogLocked("start skipped: already running") + return nil // already running + } + + cfg, proxyCfgs, err := buildConfig() + if err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("start failed: %s", err.Error()) + return err + } + cfg.Log.To = m.logPath + cfg.Log.Level = "info" + cfg.Log.MaxDays = 7 + if err := os.MkdirAll(filepath.Dir(m.logPath), 0o755); err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("init log dir failed: %s", err.Error()) + return err + } + frplog.InitLogger(cfg.Log.To, cfg.Log.Level, int(cfg.Log.MaxDays), true) + + configSource := source.NewConfigSource() + if err := configSource.ReplaceAll(proxyCfgs, nil); err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("replace config failed: %s", err.Error()) + return err + } + aggregator := source.NewAggregator(configSource) + + svr, err := frpclient.NewService(frpclient.ServiceOptions{ + Common: cfg, + ConfigSourceAggregator: aggregator, + }) + if err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("create service failed: %s", err.Error()) + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + m.cancel = cancel + m.status = "running" + m.appendLogLocked("service started") + m.wg.Add(1) + + go func() { + defer m.wg.Done() + if err := svr.Run(ctx); err != nil && ctx.Err() == nil { + // Context was not cancelled, so this is an unexpected error. + log.Warnf("frp client stopped unexpectedly: %v", err) + m.mu.Lock() + m.status = "error: " + err.Error() + m.appendLogLocked("service stopped unexpectedly: %s", err.Error()) + m.cancel = nil + m.mu.Unlock() + } + }() + + return nil +} + +// Stop gracefully shuts down the FRP client. +func (m *Manager) Stop() { + m.mu.Lock() + cancel := m.cancel + m.cancel = nil + m.mu.Unlock() + + if cancel != nil { + m.appendLog("stopping service") + cancel() + m.wg.Wait() + } + + m.mu.Lock() + m.status = "stopped" + m.appendLogLocked("service stopped") + m.mu.Unlock() +} + +// Restart stops any running client and starts a fresh one with current settings. +func (m *Manager) Restart() error { + m.appendLog("restarting service") + m.Stop() + return m.Start() +} + +// Status returns the current status string: "running", "stopped", or "error: ". +func (m *Manager) Status() string { + m.mu.Lock() + defer m.mu.Unlock() + return m.status +} + +// Runtime returns status and latest logs. +func (m *Manager) Runtime(limit int) RuntimeInfo { + m.mu.Lock() + status := m.status + logPath := m.logPath + m.mu.Unlock() + + logs, err := readLogTail(logPath, limit) + if err != nil { + m.mu.Lock() + logs = m.copyLogsLocked(limit) + m.mu.Unlock() + } + + return RuntimeInfo{ + Status: status, + Logs: logs, + } +} + +func (m *Manager) appendLog(format string, args ...interface{}) { + m.mu.Lock() + defer m.mu.Unlock() + m.appendLogLocked(format, args...) +} + +func (m *Manager) appendLogLocked(format string, args ...interface{}) { + line := fmt.Sprintf(format, args...) + entry := fmt.Sprintf("[%s] %s", time.Now().Format(time.RFC3339), line) + m.logs = append(m.logs, entry) + if len(m.logs) > maxLogEntries { + m.logs = m.logs[len(m.logs)-maxLogEntries:] + } +} + +func (m *Manager) copyLogsLocked(limit int) []string { + if limit <= 0 || limit > maxLogEntries { + limit = maxLogEntries + } + total := len(m.logs) + if total <= limit { + return append([]string(nil), m.logs...) + } + return append([]string(nil), m.logs[total-limit:]...) +} + +func readLogTail(path string, limit int) ([]string, error) { + if limit <= 0 || limit > maxLogEntries { + limit = maxLogEntries + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + size := info.Size() + start := int64(0) + if size > maxTailBytes { + start = size - maxTailBytes + } + if _, err = f.Seek(start, io.SeekStart); err != nil { + return nil, err + } + buf, err := io.ReadAll(f) + if err != nil { + return nil, err + } + + if start > 0 { + if idx := bytes.IndexByte(buf, '\n'); idx >= 0 && idx+1 < len(buf) { + buf = buf[idx+1:] + } + } + text := strings.TrimRight(string(buf), "\n") + if text == "" { + return []string{}, nil + } + lines := strings.Split(text, "\n") + if len(lines) <= limit { + return lines, nil + } + return lines[len(lines)-limit:], nil +} + +func buildConfig() (*v1.ClientCommonConfig, []v1.ProxyConfigurer, error) { + serverAddr := setting.GetStr(conf.FRPServerAddr) + if serverAddr == "" { + return nil, nil, fmt.Errorf("frp server address is required") + } + + serverPort := setting.GetInt(conf.FRPServerPort, 7000) + authToken := setting.GetStr(conf.FRPAuthToken) + proxyName := setting.GetStr(conf.FRPProxyName, "alist") + proxyType := setting.GetStr(conf.FRPProxyType, "http") + customDomain := setting.GetStr(conf.FRPCustomDomain) + subdomain := setting.GetStr(conf.FRPSubdomain) + remotePort := setting.GetInt(conf.FRPRemotePort, 0) + localPort := setting.GetInt(conf.FRPLocalPort, 5244) + tlsEnable := setting.GetBool(conf.FRPTLSEnable) + stcpSecretKey := setting.GetStr(conf.FRPSTCPSecretKey) + + cfg := &v1.ClientCommonConfig{ + ServerAddr: serverAddr, + ServerPort: serverPort, + Auth: v1.AuthClientConfig{ + Method: v1.AuthMethodToken, + Token: authToken, + }, + } + if tlsEnable { + enabled := true + cfg.Transport.TLS.Enable = &enabled + } + + backend := v1.ProxyBackend{ + LocalIP: "127.0.0.1", + LocalPort: localPort, + } + + var proxyCfgs []v1.ProxyConfigurer + + switch proxyType { + case "http": + p := &v1.HTTPProxyConfig{} + p.Name = proxyName + p.Type = "http" + p.ProxyBackend = backend + if customDomain != "" { + p.CustomDomains = []string{customDomain} + } + if subdomain != "" { + p.SubDomain = subdomain + } + proxyCfgs = append(proxyCfgs, p) + + case "https": + p := &v1.HTTPSProxyConfig{} + p.Name = proxyName + p.Type = "https" + p.ProxyBackend = backend + if customDomain != "" { + p.CustomDomains = []string{customDomain} + } + if subdomain != "" { + p.SubDomain = subdomain + } + proxyCfgs = append(proxyCfgs, p) + + case "tcp": + if remotePort <= 0 { + return nil, nil, fmt.Errorf("remote_port is required for tcp proxy type") + } + p := &v1.TCPProxyConfig{} + p.Name = proxyName + p.Type = "tcp" + p.ProxyBackend = backend + p.RemotePort = remotePort + proxyCfgs = append(proxyCfgs, p) + + case "stcp": + p := &v1.STCPProxyConfig{} + p.Name = proxyName + p.Type = "stcp" + p.ProxyBackend = backend + p.Secretkey = stcpSecretKey + p.AllowUsers = []string{"*"} + proxyCfgs = append(proxyCfgs, p) + + default: + return nil, nil, fmt.Errorf("unsupported proxy type: %s", proxyType) + } + + return cfg, proxyCfgs, nil +} diff --git a/server/handles/setting.go b/server/handles/setting.go index f209b7c5dc5..81f7dc61c24 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" @@ -183,6 +184,42 @@ func DeleteSetting(c *gin.Context) { common.SuccessResp(c) } +// SetFRP saves FRP settings and restarts the FRP client. +// Returns the current FRP connection status. +func SetFRP(c *gin.Context) { + var req []model.SettingItem + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.SaveSettingItems(req); err != nil { + common.ErrorResp(c, err, 500) + return + } + if err := frp.Instance.Restart(); err != nil { + common.SuccessResp(c, frp.Instance.Status()) + return + } + common.SuccessResp(c, frp.Instance.Status()) +} + +// StopFRP stops the FRP client and returns current status. +func StopFRP(c *gin.Context) { + frp.Instance.Stop() + common.SuccessResp(c, frp.Instance.Status()) +} + +// GetFRPRuntime returns current FRP status and recent runtime logs. +func GetFRPRuntime(c *gin.Context) { + limit := 200 + if limitStr := c.Query("limit"); limitStr != "" { + if parsed, err := strconv.Atoi(limitStr); err == nil { + limit = parsed + } + } + common.SuccessResp(c, frp.Instance.Runtime(limit)) +} + func PublicSettings(c *gin.Context) { common.SuccessResp(c, op.GetPublicSettingsMap()) } diff --git a/server/router.go b/server/router.go index 63503838af7..03a52841ed3 100644 --- a/server/router.go +++ b/server/router.go @@ -161,6 +161,9 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_115", handles.Set115) setting.POST("/set_pikpak", handles.SetPikPak) setting.POST("/set_thunder", handles.SetThunder) + setting.POST("/set_frp", handles.SetFRP) + setting.POST("/stop_frp", handles.StopFRP) + setting.GET("/frp_runtime", handles.GetFRPRuntime) // retain /admin/task API to ensure compatibility with legacy automation scripts _task(g.Group("/task")) From 51114e4943f75f0c1dbd99fc6a3758b84eb5540b Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 15 Apr 2026 20:48:38 +0800 Subject: [PATCH 632/659] feat(frp): wire bootstrap settings and runtime dependencies --- cmd/server.go | 3 + go.mod | 51 +++++++++-- go.sum | 140 ++++++++++++++++++++++++----- internal/bootstrap/data/setting.go | 15 ++++ internal/conf/const.go | 15 ++++ internal/model/setting.go | 1 + 6 files changed, 195 insertions(+), 30 deletions(-) diff --git a/cmd/server.go b/cmd/server.go index 4263f02021d..abfbcb2c4c9 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -18,6 +18,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server" @@ -43,6 +44,7 @@ the address is defined in config file`, bootstrap.InitOfflineDownloadTools() bootstrap.LoadStorages() bootstrap.InitTaskManager() + bootstrap.InitFRP() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) } @@ -167,6 +169,7 @@ the address is defined in config file`, <-quit utils.Log.Println("Shutdown server...") fs.ArchiveContentUploadTaskManager.RemoveAll() + frp.Instance.Stop() Release() ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() diff --git a/go.mod b/go.mod index bc2475c548d..f148b235562 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/alist-org/alist/v3 -go 1.23.4 +go 1.25.0 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 @@ -29,6 +29,7 @@ require ( github.com/disintegration/imaging v1.6.2 github.com/dlclark/regexp2 v1.11.4 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 + github.com/fatedier/frp v0.68.0 github.com/foxxorcat/mopan-sdk-go v0.1.6 github.com/foxxorcat/weiyun-sdk-go v0.1.3 github.com/gin-contrib/cors v1.7.2 @@ -70,10 +71,10 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.36.0 + golang.org/x/crypto v0.41.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 - golang.org/x/net v0.38.0 + golang.org/x/net v0.43.0 golang.org/x/oauth2 v0.30.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 @@ -86,18 +87,50 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect github.com/ProtonMail/go-srp v0.0.7 // indirect github.com/PuerkitoBio/goquery v1.8.1 // indirect github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect github.com/bradenaw/juniper v0.15.2 // indirect + github.com/coreos/go-oidc/v3 v3.14.1 // indirect github.com/cronokirby/saferith v0.33.0 // indirect github.com/emersion/go-message v0.18.0 // indirect github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect + github.com/fatedier/golib v0.5.1 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/klauspost/reedsolomon v1.12.0 // indirect + github.com/pion/dtls/v2 v2.2.7 // indirect + github.com/pion/logging v0.2.2 // indirect + github.com/pion/stun/v2 v2.0.0 // indirect + github.com/pion/transport/v2 v2.2.1 // indirect + github.com/pion/transport/v3 v3.0.1 // indirect + github.com/pires/go-proxyproto v0.7.0 // indirect + github.com/quic-go/quic-go v0.55.0 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect + github.com/samber/lo v1.47.0 // indirect + github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect + github.com/templexxx/cpu v0.1.1 // indirect + github.com/templexxx/xorsimd v0.4.3 // indirect + github.com/tjfoc/gmsm v1.4.1 // indirect + github.com/vishvananda/netlink v1.3.0 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + github.com/xtaci/kcp-go/v5 v5.6.13 // indirect + golang.org/x/mod v0.27.0 // indirect + golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect + golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apimachinery v0.28.8 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) require ( @@ -266,15 +299,15 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.12.0 - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 - golang.org/x/tools v0.24.0 // indirect + golang.org/x/sync v0.16.0 + golang.org/x/sys v0.35.0 // indirect + golang.org/x/term v0.34.0 // indirect + golang.org/x/text v0.28.0 + golang.org/x/tools v0.36.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect google.golang.org/grpc v1.66.0 - google.golang.org/protobuf v1.34.2 // indirect + google.golang.org/protobuf v1.36.5 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index e6b04779ede..5ad4346b9fb 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -89,6 +91,8 @@ github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOL github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -195,8 +199,11 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= +github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -229,10 +236,16 @@ github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwo github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA= github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fatedier/frp v0.68.0 h1:woKC31EpgCLQDirRAOAUgHP3IRxLu2mBHS6zGpcw7tw= +github.com/fatedier/frp v0.68.0/go.mod h1:qFdez6Z+RDqoqF1xJhh48+IW91SVQ+pNWnrmcl43Wjs= +github.com/fatedier/golib v0.5.1 h1:hcKAnaw5mdI/1KWRGejxR+i1Hn/NvbY5UsMKDr7o13M= +github.com/fatedier/golib v0.5.1/go.mod h1:W6kIYkIFxHsTzbgqg5piCxIiDo4LzwgTY6R5W8l9NFQ= github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= github.com/fclairamb/go-log v0.5.0/go.mod h1:XoRO1dYezpsGmLLkZE9I+sHqpqY65p8JA+Vqblb7k40= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= @@ -258,13 +271,15 @@ github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -310,6 +325,12 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -318,6 +339,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -325,8 +348,9 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -348,6 +372,8 @@ github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUh github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -365,6 +391,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI0= github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= @@ -427,6 +455,8 @@ github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuV github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.12.0 h1:I5FEp3xSwVCcEh3F5A7dofEfhXdF/bWhQWPH+XwBFno= +github.com/klauspost/reedsolomon v1.12.0/go.mod h1:EPLZJeh4l27pUGC3aXOjheaoh1I9yut7xTURiW3LQ9Y= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -535,6 +565,18 @@ github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= +github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs= +github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -563,6 +605,8 @@ github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSz github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= +github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U= github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= @@ -581,25 +625,28 @@ github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/go-m1cpu v0.2.0 h1:t4GNqvPZ84Vjtpboo/kT3pIkbaK3vc+JIlD/Wz1zSFY= github.com/shoenig/go-m1cpu v0.2.0/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= @@ -624,6 +671,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -632,8 +680,14 @@ github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= +github.com/templexxx/cpu v0.1.1 h1:isxHaxBXpYFWnk2DReuKkigaZyrjs2+9ypIdGP4h+HI= +github.com/templexxx/cpu v0.1.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= +github.com/templexxx/xorsimd v0.4.3 h1:9AQTFHd7Bhk3dIT7Al2XeBX5DWOvsUPZCuhyAtNbHjU= +github.com/templexxx/xorsimd v0.4.3/go.mod h1:oZQcD6RFDisW2Am58dSAGwwL6rHjbzrlu25VDqfWkQg= github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= +github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= +github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= @@ -658,6 +712,10 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco= github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= +github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= @@ -670,6 +728,10 @@ github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= github.com/xhofe/tache v0.1.5/go.mod h1:PYt6I/XUKliSg1uHlgsk6ha+le/f6PAvjUtFZAVl3a8= github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= +github.com/xtaci/kcp-go/v5 v5.6.13 h1:FEjtz9+D4p8t2x4WjciGt/jsIuhlWjjgPCCWjrVR4Hk= +github.com/xtaci/kcp-go/v5 v5.6.13/go.mod h1:75S1AKYYzNUSXIv30h+jPKJYZUwqpfvLshu63nCNSOM= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37 h1:EWU6Pktpas0n8lLQwDsRyZfmkPeRbdgPtW609es+/9E= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37/go.mod h1:HpMP7DB2CyokmAh4lp0EQnnWhmycP/TvwBGzvuie+H0= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= @@ -701,6 +763,8 @@ go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= @@ -714,17 +778,20 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -761,6 +828,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= +golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -775,6 +844,7 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -789,12 +859,13 @@ golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -814,8 +885,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -851,14 +922,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -868,12 +940,13 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -886,12 +959,13 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -929,12 +1003,16 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= -golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -971,15 +1049,23 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -987,6 +1073,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= @@ -1010,11 +1098,17 @@ gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDa gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/apimachinery v0.28.8 h1:hi/nrxHwk4QLV+W/SHve1bypTE59HCDorLY1stBIxKQ= +k8s.io/apimachinery v0.28.8/go.mod h1:cBnwIM3fXoRo28SqbV/Ihxf/iviw85KyXOrzxvZQ83U= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= @@ -1024,4 +1118,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 61ebe1fa207..265a6502b60 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -220,6 +220,21 @@ func InitialSettings() []model.SettingItem { {Key: conf.FTPTLSPrivateKeyPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, {Key: conf.FTPTLSPublicCertPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + // frp settings + {Key: conf.FRPEnabled, Value: "false", Type: conf.TypeBool, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPServerAddr, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPServerPort, Value: "7000", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPAuthToken, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPProxyName, Value: "alist", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPProxyType, Value: "http", Type: conf.TypeSelect, Options: "http,https,tcp,stcp", Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPCustomDomain, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPSubdomain, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPRemotePort, Value: "0", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPLocalPort, Value: "5244", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPTLSEnable, Value: "false", Type: conf.TypeBool, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPSTCPSecretKey, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE, Help: "Required for stcp proxy type"}, + {Key: conf.FRPStatus, Value: "stopped", Type: conf.TypeString, Group: model.FRP, Flag: model.READONLY}, + // traffic settings {Key: conf.TaskOfflineDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Download.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, {Key: conf.TaskOfflineDownloadTransferThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Transfer.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index db2c84dc4ae..5bcb6eb5f91 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -125,6 +125,21 @@ const ( FTPTLSPrivateKeyPath = "ftp_tls_private_key_path" FTPTLSPublicCertPath = "ftp_tls_public_cert_path" + // frp + FRPEnabled = "frp_enabled" + FRPServerAddr = "frp_server_addr" + FRPServerPort = "frp_server_port" + FRPAuthToken = "frp_auth_token" + FRPProxyName = "frp_proxy_name" + FRPProxyType = "frp_proxy_type" + FRPCustomDomain = "frp_custom_domain" + FRPSubdomain = "frp_subdomain" + FRPRemotePort = "frp_remote_port" + FRPLocalPort = "frp_local_port" + FRPTLSEnable = "frp_tls_enable" + FRPSTCPSecretKey = "frp_stcp_secret_key" + FRPStatus = "frp_status" + // traffic TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" TaskOfflineDownloadTransferThreadsNum = "offline_download_transfer_task_threads_num" diff --git a/internal/model/setting.go b/internal/model/setting.go index 93b81fe5941..9e23f9509e6 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -13,6 +13,7 @@ const ( S3 FTP TRAFFIC + FRP ) const ( From 2edfcaee7be9e57d10b9a8009e122586dde904be Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Wed, 15 Apr 2026 20:50:27 +0800 Subject: [PATCH 633/659] add mcp support --- cmd/mcp.go | 27 +++++ cmd/server.go | 23 ++++ go.mod | 4 + go.sum | 18 ++- internal/conf/config.go | 10 ++ internal/model/user.go | 10 ++ server/common/role_perm.go | 2 + server/mcp/auth.go | 182 +++++++++++++++++++++++++++++ server/mcp/convert.go | 56 +++++++++ server/mcp/errors.go | 40 +++++++ server/mcp/server.go | 64 +++++++++++ server/mcp/tools_manage.go | 228 +++++++++++++++++++++++++++++++++++++ server/mcp/tools_read.go | 207 +++++++++++++++++++++++++++++++++ server/mcp/tools_upload.go | 131 +++++++++++++++++++++ 14 files changed, 997 insertions(+), 5 deletions(-) create mode 100644 cmd/mcp.go create mode 100644 server/mcp/auth.go create mode 100644 server/mcp/convert.go create mode 100644 server/mcp/errors.go create mode 100644 server/mcp/server.go create mode 100644 server/mcp/tools_manage.go create mode 100644 server/mcp/tools_read.go create mode 100644 server/mcp/tools_upload.go diff --git a/cmd/mcp.go b/cmd/mcp.go new file mode 100644 index 00000000000..356b8bfc1b8 --- /dev/null +++ b/cmd/mcp.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "github.com/alist-org/alist/v3/internal/bootstrap" + mcpserver "github.com/alist-org/alist/v3/server/mcp" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/spf13/cobra" +) + +var MCPCmd = &cobra.Command{ + Use: "mcp", + Short: "Start MCP server in STDIO mode", + Long: `Start an MCP (Model Context Protocol) server that communicates via STDIO, suitable for integration with AI assistants like Claude Desktop.`, + Run: func(cmd *cobra.Command, args []string) { + Init() + bootstrap.LoadStorages() + username, _ := cmd.Flags().GetString("user") + if err := mcpserver.ServeStdio(username); err != nil { + utils.Log.Fatalf("MCP STDIO server error: %v", err) + } + }, +} + +func init() { + MCPCmd.Flags().String("user", "admin", "Username for MCP operations") + RootCmd.AddCommand(MCPCmd) +} diff --git a/cmd/server.go b/cmd/server.go index 4263f02021d..238b89296e0 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -21,6 +21,7 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server" + mcpserver "github.com/alist-org/alist/v3/server/mcp" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -157,6 +158,19 @@ the address is defined in config file`, }() } } + var mcpHttpSrv *http.Server + if conf.Conf.MCP.Port != -1 && conf.Conf.MCP.Enable { + mcpHandler := mcpserver.NewHTTPHandler() + mcpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.MCP.Port) + utils.Log.Infof("start MCP server @ %s", mcpBase) + mcpHttpSrv = &http.Server{Addr: mcpBase, Handler: mcpHandler} + go func() { + err := mcpHttpSrv.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + utils.Log.Fatalf("failed to start MCP server: %s", err.Error()) + } + }() + } // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -217,6 +231,15 @@ the address is defined in config file`, } }() } + if conf.Conf.MCP.Port != -1 && conf.Conf.MCP.Enable && mcpHttpSrv != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := mcpHttpSrv.Shutdown(ctx); err != nil { + utils.Log.Fatal("MCP server shutdown err: ", err) + } + }() + } wg.Wait() utils.Log.Println("Server exit") }, diff --git a/go.mod b/go.mod index 17c09201ab0..0807c49e1f1 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/kdomanski/iso9660 v0.4.0 github.com/larksuite/oapi-sdk-go/v3 v3.3.1 + github.com/mark3labs/mcp-go v0.48.0 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.2 github.com/mholt/archives v0.1.0 @@ -97,7 +98,10 @@ require ( github.com/emersion/go-message v0.18.0 // indirect github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect github.com/relvacode/iso8601 v1.3.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect ) require ( diff --git a/go.sum b/go.sum index 370ac6d206f..3baf8885944 100644 --- a/go.sum +++ b/go.sum @@ -241,6 +241,8 @@ github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/sr github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= @@ -325,11 +327,14 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -456,6 +461,8 @@ github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIg github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mark3labs/mcp-go v0.48.0 h1:o+MXuGW/HCeR2ny5LcAcZQn2bo6I2xaZMEHnpRG+dtw= +github.com/mark3labs/mcp-go v0.48.0/go.mod h1:JKTC7R2LLVagkEWK7Kwu7DbmA6iIvnNAod6yrHiQMag= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= @@ -587,15 +594,12 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIG github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/go-m1cpu v0.2.0 h1:t4GNqvPZ84Vjtpboo/kT3pIkbaK3vc+JIlD/Wz1zSFY= -github.com/shoenig/go-m1cpu v0.2.0/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= @@ -609,6 +613,8 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -676,6 +682,8 @@ github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZ github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9/go.mod h1:9BnoKCcgJ/+SLhfAXj15352hTOuVmG5Gzo8xNRINfqI= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= diff --git a/internal/conf/config.go b/internal/conf/config.go index 15bd7feba0a..383a5a4e52f 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -94,6 +94,11 @@ type SFTP struct { Listen string `json:"listen" env:"LISTEN"` } +type MCP struct { + Enable bool `json:"enable" env:"ENABLE"` + Port int `json:"port" env:"PORT"` +} + type Config struct { Force bool `json:"force" env:"FORCE"` SiteURL string `json:"site_url" env:"SITE_URL"` @@ -116,6 +121,7 @@ type Config struct { S3 S3 `json:"s3" envPrefix:"S3_"` FTP FTP `json:"ftp" envPrefix:"FTP_"` SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` + MCP MCP `json:"mcp" envPrefix:"MCP_"` LastLaunchedVersion string `json:"last_launched_version"` } @@ -218,6 +224,10 @@ func DefaultConfig() *Config { Enable: false, Listen: ":5222", }, + MCP: MCP{ + Enable: false, + Port: 5248, + }, LastLaunchedVersion: "", } } diff --git a/internal/model/user.go b/internal/model/user.go index 8ea1ef1aaff..f55b6a5a2a2 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -49,6 +49,8 @@ type User struct { // 12: can read archives // 13: can decompress archives // 14: check path limit + // 15: mcp read + // 16: mcp write Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -144,6 +146,14 @@ func (u *User) CheckPathLimit() bool { return (u.Permission>>14)&1 == 1 } +func (u *User) CanMCPAccess() bool { + return (u.Permission>>15)&1 == 1 +} + +func (u *User) CanMCPManage() bool { + return (u.Permission>>16)&1 == 1 +} + func (u *User) JoinPath(reqPath string) (string, error) { if reqPath == "/" { return utils.FixAndCleanPath(u.BasePath), nil diff --git a/server/common/role_perm.go b/server/common/role_perm.go index 36dedf98c5e..ec82d4d91a0 100644 --- a/server/common/role_perm.go +++ b/server/common/role_perm.go @@ -27,6 +27,8 @@ const ( PermReadArchives PermDecompress PermPathLimit + PermMCPAccess + PermMCPManage ) func HasPermission(perm int32, bit uint) bool { diff --git a/server/mcp/auth.go b/server/mcp/auth.go new file mode 100644 index 00000000000..6d1e670d14a --- /dev/null +++ b/server/mcp/auth.go @@ -0,0 +1,182 @@ +package mcp + +import ( + "context" + "crypto/subtle" + "fmt" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/server/common" + log "github.com/sirupsen/logrus" +) + +type ctxKey string + +const userKey ctxKey = "user" + +// HTTPContextFunc extracts JWT/admin token from HTTP request and injects user into context. +// Used as WithHTTPContextFunc callback for Streamable HTTP transport. +func HTTPContextFunc(ctx context.Context, r *http.Request) context.Context { + token := r.Header.Get("Authorization") + if token == "" { + token = r.URL.Query().Get("token") + } + + user, err := authenticateToken(token) + if err != nil { + log.Debugf("MCP auth failed: %v", err) + return ctx + } + + return context.WithValue(ctx, userKey, user) +} + +func authenticateToken(token string) (*model.User, error) { + // Check admin static token + if token != "" && subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { + admin, err := op.GetAdmin() + if err != nil { + return nil, fmt.Errorf("failed to get admin: %w", err) + } + if err := loadRoles(admin); err != nil { + return nil, err + } + return admin, nil + } + + // No token: guest + if token == "" { + guest, err := op.GetGuest() + if err != nil { + return nil, fmt.Errorf("failed to get guest: %w", err) + } + if guest.Disabled { + return nil, fmt.Errorf("guest user is disabled") + } + if err := loadRoles(guest); err != nil { + return nil, err + } + return guest, nil + } + + // JWT token + claims, err := common.ParseToken(token) + if err != nil { + return nil, fmt.Errorf("invalid token: %w", err) + } + + user, err := op.GetUserByName(claims.Username) + if err != nil { + return nil, fmt.Errorf("user not found: %w", err) + } + + if claims.PwdTS != user.PwdTS { + return nil, fmt.Errorf("password has been changed") + } + if user.Disabled { + return nil, fmt.Errorf("user is disabled") + } + + if err := loadRoles(user); err != nil { + return nil, err + } + return user, nil +} + +func loadRoles(user *model.User) error { + if len(user.Role) > 0 { + roles, err := op.GetRolesByUserID(user.ID) + if err != nil { + return fmt.Errorf("failed to load roles: %w", err) + } + user.RolesDetail = roles + } + return nil +} + +// resolveUser extracts the authenticated user from context. +func resolveUser(ctx context.Context) (*model.User, error) { + user, ok := ctx.Value(userKey).(*model.User) + if !ok || user == nil { + return nil, fmt.Errorf("authentication required") + } + return user, nil +} + +// buildFsContext resolves path and sets meta in context for fs operations. +func buildFsContext(ctx context.Context, user *model.User, path string) (context.Context, string, error) { + reqPath, err := user.JoinPath(path) + if err != nil { + return ctx, "", err + } + meta, _ := op.GetNearestMeta(reqPath) + ctx = context.WithValue(ctx, "meta", meta) + ctx = context.WithValue(ctx, "user", user) + return ctx, reqPath, nil +} + +// checkAccess checks if user can access the path (read). +func checkAccess(user *model.User, reqPath string) error { + meta, _ := op.GetNearestMeta(reqPath) + if !common.CanAccessWithRoles(user, meta, reqPath, "") { + return fmt.Errorf("permission denied") + } + perm := common.MergeRolePermissions(user, reqPath) + if !user.IsAdmin() && !common.HasPermission(perm, common.PermMCPAccess) { + return fmt.Errorf("MCP access not permitted") + } + return nil +} + +// checkManage checks if user can perform write operations via MCP. +func checkManage(user *model.User, reqPath string, permBit uint) error { + if err := checkAccess(user, reqPath); err != nil { + return err + } + perm := common.MergeRolePermissions(user, reqPath) + if !user.IsAdmin() && !common.HasPermission(perm, common.PermMCPManage) { + return fmt.Errorf("MCP manage not permitted") + } + if !user.IsAdmin() && !common.HasPermission(perm, permBit) { + return fmt.Errorf("permission denied for this operation") + } + return nil +} + +// UserContextFunc returns an HTTPContextFunc that injects a specific user (for STDIO mode). +func userContextMiddleware(user *model.User) func(ctx context.Context) context.Context { + return func(ctx context.Context) context.Context { + return context.WithValue(ctx, userKey, user) + } +} + +// resolveUserForStdio resolves a user by username for STDIO mode. +func resolveUserForStdio(username string) (*model.User, error) { + username = strings.TrimSpace(username) + if username == "" || username == "admin" { + admin, err := op.GetAdmin() + if err != nil { + return nil, fmt.Errorf("failed to get admin user: %w", err) + } + if err := loadRoles(admin); err != nil { + return nil, err + } + return admin, nil + } + user, err := op.GetUserByName(username) + if err != nil { + return nil, fmt.Errorf("user %q not found: %w", username, err) + } + if user.Disabled { + return nil, fmt.Errorf("user %q is disabled", username) + } + if err := loadRoles(user); err != nil { + return nil, err + } + return user, nil +} diff --git a/server/mcp/convert.go b/server/mcp/convert.go new file mode 100644 index 00000000000..7eede8cbc87 --- /dev/null +++ b/server/mcp/convert.go @@ -0,0 +1,56 @@ +package mcp + +import ( + "encoding/json" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/mark3labs/mcp-go/mcp" +) + +type objJSON struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + HashInfo map[string]string `json:"hash_info,omitempty"` +} + +func objToJSON(obj model.Obj) objJSON { + j := objJSON{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + } + hi := obj.GetHash() + if hm := hashInfoToMap(hi); len(hm) > 0 { + j.HashInfo = hm + } + return j +} + +func hashInfoToMap(hi utils.HashInfo) map[string]string { + m := make(map[string]string) + for ht, v := range hi.All() { + if v != "" { + m[ht.Name] = v + } + } + return m +} + +func jsonResult(v interface{}) (*mcp.CallToolResult, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + return mcp.NewToolResultText(string(data)), nil +} + +func textResult(msg string) (*mcp.CallToolResult, error) { + return mcp.NewToolResultText(msg), nil +} diff --git a/server/mcp/errors.go b/server/mcp/errors.go new file mode 100644 index 00000000000..c107633739f --- /dev/null +++ b/server/mcp/errors.go @@ -0,0 +1,40 @@ +package mcp + +import ( + "fmt" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/mark3labs/mcp-go/mcp" + pkgerr "github.com/pkg/errors" +) + +func toolError(msg string) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(msg), nil +} + +func toolErrorf(format string, args ...interface{}) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(fmt.Sprintf(format, args...)), nil +} + +func wrapError(err error) (*mcp.CallToolResult, error) { + if err == nil { + return nil, nil + } + cause := pkgerr.Cause(err) + switch { + case errs.IsObjectNotFound(err) || errs.IsNotFoundError(err): + return toolErrorf("not found: %s", err.Error()) + case cause == errs.PermissionDenied: + return toolError("permission denied") + case cause == errs.NotImplement: + return toolError("not supported by storage driver") + case cause == errs.NotSupport: + return toolError("operation not supported") + case cause == errs.UploadNotSupported: + return toolError("upload not supported by storage") + case cause == errs.MoveBetweenTwoStorages: + return toolError("can't move between two storages, use copy instead") + default: + return toolErrorf("error: %s", err.Error()) + } +} diff --git a/server/mcp/server.go b/server/mcp/server.go new file mode 100644 index 00000000000..b45a0113eb3 --- /dev/null +++ b/server/mcp/server.go @@ -0,0 +1,64 @@ +package mcp + +import ( + "context" + "net/http" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +// NewServer creates an MCP server with all alist tools registered. +func NewServer() *mcpserver.MCPServer { + s := mcpserver.NewMCPServer( + "alist", + conf.Version, + mcpserver.WithToolCapabilities(false), + mcpserver.WithRecovery(), + ) + registerReadTools(s) + registerManageTools(s) + registerUploadTools(s) + return s +} + +// NewHTTPHandler creates a Streamable HTTP handler for the MCP server. +func NewHTTPHandler() http.Handler { + s := NewServer() + return mcpserver.NewStreamableHTTPServer(s, + mcpserver.WithHTTPContextFunc(HTTPContextFunc), + ) +} + +// NewStdioServer creates an MCP server configured for STDIO mode with a fixed user. +func NewStdioServer(username string) (*mcpserver.MCPServer, *model.User, error) { + user, err := resolveUserForStdio(username) + if err != nil { + return nil, nil, err + } + s := NewServer() + return s, user, nil +} + +// ServeStdio starts the MCP server in STDIO mode. +func ServeStdio(username string) error { + s, user, err := NewStdioServer(username) + if err != nil { + return err + } + ctxFunc := userContextMiddleware(user) + return mcpserver.ServeStdio(s, mcpserver.WithStdioContextFunc(ctxFunc)) +} + +// toolHandlerWithAuth wraps a tool handler to require authentication. +func toolHandlerWithAuth(fn func(ctx context.Context, user *model.User, request mcp.CallToolRequest) (*mcp.CallToolResult, error)) mcpserver.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + user, err := resolveUser(ctx) + if err != nil { + return toolError("authentication required") + } + return fn(ctx, user, request) + } +} diff --git a/server/mcp/tools_manage.go b/server/mcp/tools_manage.go new file mode 100644 index 00000000000..d287bb6bed0 --- /dev/null +++ b/server/mcp/tools_manage.go @@ -0,0 +1,228 @@ +package mcp + +import ( + "context" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +func registerManageTools(s *mcpserver.MCPServer) { + // fs_mkdir + s.AddTool(mcp.NewTool("fs_mkdir", + mcp.WithDescription("Create a new directory"), + mcp.WithString("path", mcp.Required(), mcp.Description("Full path of directory to create")), + ), toolHandlerWithAuth(handleFsMkdir)) + + // fs_rename + s.AddTool(mcp.NewTool("fs_rename", + mcp.WithDescription("Rename a file or directory"), + mcp.WithString("path", mcp.Required(), mcp.Description("Current path of the file/directory")), + mcp.WithString("name", mcp.Required(), mcp.Description("New name (filename only, not a path)")), + ), toolHandlerWithAuth(handleFsRename)) + + // fs_move + s.AddTool(mcp.NewTool("fs_move", + mcp.WithDescription("Move files/directories to another location"), + mcp.WithString("src_dir", mcp.Required(), mcp.Description("Source directory")), + mcp.WithString("dst_dir", mcp.Required(), mcp.Description("Destination directory")), + mcp.WithArray("names", mcp.Description("Names of files/directories to move")), + ), toolHandlerWithAuth(handleFsMove)) + + // fs_copy + s.AddTool(mcp.NewTool("fs_copy", + mcp.WithDescription("Copy files/directories to another location"), + mcp.WithString("src_dir", mcp.Required(), mcp.Description("Source directory")), + mcp.WithString("dst_dir", mcp.Required(), mcp.Description("Destination directory")), + mcp.WithArray("names", mcp.Description("Names of files/directories to copy")), + ), toolHandlerWithAuth(handleFsCopy)) + + // fs_remove + s.AddTool(mcp.NewTool("fs_remove", + mcp.WithDescription("Delete files/directories"), + mcp.WithString("dir", mcp.Required(), mcp.Description("Directory containing items to remove")), + mcp.WithArray("names", mcp.Description("Names of files/directories to remove")), + ), toolHandlerWithAuth(handleFsRemove)) +} + +func handleFsMkdir(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqPath, common.PermWrite); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.MakeDir(ctx, reqPath); err != nil { + return wrapError(err) + } + return textResult("directory created successfully") +} + +func handleFsRename(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + name, err := req.RequireString("name") + if err != nil { + return toolError("name is required") + } + if err := utils.ValidateNameComponent(name); err != nil { + return toolErrorf("invalid name: %s", err.Error()) + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqPath, common.PermRename); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.Rename(ctx, reqPath, name); err != nil { + return wrapError(err) + } + return textResult("renamed successfully") +} + +func handleFsMove(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + srcDirStr, err := req.RequireString("src_dir") + if err != nil { + return toolError("src_dir is required") + } + dstDirStr, err := req.RequireString("dst_dir") + if err != nil { + return toolError("dst_dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + srcDir, err := user.JoinPath(srcDirStr) + if err != nil { + return wrapError(err) + } + dstDir, err := user.JoinPath(dstDirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, srcDir, common.PermMove); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for i, name := range names { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if err := fs.Move(ctx, srcPath, dstDir, len(names) > i+1); err != nil { + return wrapError(err) + } + } + return textResult("moved successfully") +} + +func handleFsCopy(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + srcDirStr, err := req.RequireString("src_dir") + if err != nil { + return toolError("src_dir is required") + } + dstDirStr, err := req.RequireString("dst_dir") + if err != nil { + return toolError("dst_dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + srcDir, err := user.JoinPath(srcDirStr) + if err != nil { + return wrapError(err) + } + dstDir, err := user.JoinPath(dstDirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, srcDir, common.PermCopy); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for i, name := range names { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if _, err := fs.Copy(ctx, srcPath, dstDir, len(names) > i+1); err != nil { + return wrapError(err) + } + } + return textResult("copied successfully") +} + +func handleFsRemove(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + dirStr, err := req.RequireString("dir") + if err != nil { + return toolError("dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + reqDir, err := user.JoinPath(dirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqDir, common.PermRemove); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for _, name := range names { + removePath, err := utils.JoinUnderBase(reqDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if err := fs.Remove(ctx, removePath); err != nil { + return wrapError(err) + } + } + return textResult("removed successfully") +} + +// getStringArray extracts a string array from tool request arguments. +func getStringArray(req mcp.CallToolRequest, name string) []string { + args := req.GetArguments() + val, ok := args[name] + if !ok { + return nil + } + arr, ok := val.([]interface{}) + if !ok { + return nil + } + result := make([]string, 0, len(arr)) + for _, v := range arr { + if s, ok := v.(string); ok { + result = append(result, s) + } + } + return result +} diff --git a/server/mcp/tools_read.go b/server/mcp/tools_read.go new file mode 100644 index 00000000000..d8af570b233 --- /dev/null +++ b/server/mcp/tools_read.go @@ -0,0 +1,207 @@ +package mcp + +import ( + "context" + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/search" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" + "github.com/pkg/errors" +) + +func registerReadTools(s *mcpserver.MCPServer) { + // fs_list + s.AddTool(mcp.NewTool("fs_list", + mcp.WithDescription("List files and directories at the given path"), + mcp.WithString("path", mcp.Required(), mcp.Description("Directory path to list")), + mcp.WithNumber("page", mcp.Description("Page number (default: 1)")), + mcp.WithNumber("per_page", mcp.Description("Items per page (default: 30, max: 500)")), + mcp.WithBoolean("refresh", mcp.Description("Force refresh from storage (default: false)")), + ), toolHandlerWithAuth(handleFsList)) + + // fs_get + s.AddTool(mcp.NewTool("fs_get", + mcp.WithDescription("Get file or directory metadata"), + mcp.WithString("path", mcp.Required(), mcp.Description("Path to file or directory")), + ), toolHandlerWithAuth(handleFsGet)) + + // fs_search + s.AddTool(mcp.NewTool("fs_search", + mcp.WithDescription("Search for files by keywords"), + mcp.WithString("path", mcp.Required(), mcp.Description("Parent directory to search within")), + mcp.WithString("keywords", mcp.Required(), mcp.Description("Search keywords")), + mcp.WithNumber("scope", mcp.Description("0=all, 1=dir only, 2=file only (default: 0)")), + mcp.WithNumber("page", mcp.Description("Page number (default: 1)")), + mcp.WithNumber("per_page", mcp.Description("Items per page (default: 20)")), + ), toolHandlerWithAuth(handleFsSearch)) + + // fs_download_url + s.AddTool(mcp.NewTool("fs_download_url", + mcp.WithDescription("Get download URL for a file"), + mcp.WithString("path", mcp.Required(), mcp.Description("Path to the file")), + ), toolHandlerWithAuth(handleFsDownloadURL)) +} + +func handleFsList(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + page := intParam(req, "page", 1) + perPage := intParam(req, "per_page", 30) + if perPage > 500 { + perPage = 500 + } + refresh := req.GetBool("refresh", false) + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{Refresh: refresh}) + if err != nil { + return wrapError(err) + } + + // Paginate + total := len(objs) + start := (page - 1) * perPage + if start > total { + start = total + } + end := start + perPage + if end > total { + end = total + } + pageObjs := objs[start:end] + + items := make([]objJSON, 0, len(pageObjs)) + for _, obj := range pageObjs { + items = append(items, objToJSON(obj)) + } + + return jsonResult(map[string]interface{}{ + "content": items, + "total": total, + "page": page, + "per_page": perPage, + }) +} + +func handleFsGet(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + if err != nil { + return wrapError(err) + } + + return jsonResult(objToJSON(obj)) +} + +func handleFsSearch(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + keywords, err := req.RequireString("keywords") + if err != nil { + return toolError("keywords is required") + } + scope := intParam(req, "scope", 0) + page := intParam(req, "page", 1) + perPage := intParam(req, "per_page", 20) + + parent, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + + searchReq := model.SearchReq{ + Parent: parent, + Keywords: keywords, + Scope: scope, + PageReq: model.PageReq{Page: page, PerPage: perPage}, + } + if err := searchReq.Validate(); err != nil { + return toolErrorf("invalid search request: %s", err.Error()) + } + + nodes, total, err := search.Search(ctx, searchReq) + if err != nil { + return wrapError(err) + } + + // Filter by permission + filtered := make([]model.SearchNode, 0, len(nodes)) + for _, node := range nodes { + if !strings.HasPrefix(node.Parent, user.BasePath) { + continue + } + meta, err := op.GetNearestMeta(node.Parent) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + continue + } + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), "") { + continue + } + filtered = append(filtered, node) + } + + return jsonResult(map[string]interface{}{ + "content": filtered, + "total": total, + }) +} + +func handleFsDownloadURL(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{}) + if err != nil { + return wrapError(err) + } + + return jsonResult(map[string]interface{}{ + "raw_url": link.URL, + }) +} + +// intParam extracts an integer parameter with a default value. +func intParam(req mcp.CallToolRequest, name string, defaultVal int) int { + v := req.GetFloat(name, float64(defaultVal)) + return int(v) +} diff --git a/server/mcp/tools_upload.go b/server/mcp/tools_upload.go new file mode 100644 index 00000000000..c84eb9623a9 --- /dev/null +++ b/server/mcp/tools_upload.go @@ -0,0 +1,131 @@ +package mcp + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +func registerUploadTools(s *mcpserver.MCPServer) { + s.AddTool(mcp.NewTool("fs_upload", + mcp.WithDescription("Upload a local file to alist. Automatically uses direct internal upload (local deployment) or HTTP API upload (remote deployment)."), + mcp.WithString("path", mcp.Required(), mcp.Description("Destination path in alist including filename")), + mcp.WithString("local_path", mcp.Required(), mcp.Description("Absolute local file path to upload")), + ), toolHandlerWithAuth(handleFsUpload)) +} + +func handleFsUpload(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + localPath, err := req.RequireString("local_path") + if err != nil { + return toolError("local_path is required") + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + dir := stdpath.Dir(reqPath) + + if err := checkManage(user, dir, common.PermWrite); err != nil { + return toolError(err.Error()) + } + + if strings.Contains(conf.Conf.SiteURL, "://") { + return uploadViaHTTP(reqPath, localPath) + } + return uploadDirectly(ctx, user, reqPath, localPath) +} + +func uploadDirectly(ctx context.Context, user *model.User, reqPath, localPath string) (*mcp.CallToolResult, error) { + file, err := os.Open(localPath) + if err != nil { + return toolErrorf("failed to open local file: %s", err.Error()) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return toolErrorf("failed to stat local file: %s", err.Error()) + } + + dir := stdpath.Dir(reqPath) + name := stdpath.Base(reqPath) + + fileStream := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{ + Name: name, + Size: info.Size(), + Modified: time.Now(), + IsFolder: false, + }, + Reader: io.NopCloser(file), + Closers: utils.EmptyClosers(), + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.PutDirectly(ctx, dir, fileStream); err != nil { + return wrapError(err) + } + return textResult("uploaded successfully") +} + +func uploadViaHTTP(reqPath, localPath string) (*mcp.CallToolResult, error) { + file, err := os.Open(localPath) + if err != nil { + return toolErrorf("failed to open local file: %s", err.Error()) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return toolErrorf("failed to stat local file: %s", err.Error()) + } + + name := stdpath.Base(reqPath) + apiURL := fmt.Sprintf("%s/api/fs/put", conf.Conf.SiteURL) + + httpReq, err := http.NewRequest(http.MethodPut, apiURL, file) + if err != nil { + return toolErrorf("failed to create request: %s", err.Error()) + } + + httpReq.Header.Set("File-Path", url.PathEscape(reqPath)) + httpReq.Header.Set("Content-Length", strconv.FormatInt(info.Size(), 10)) + httpReq.Header.Set("Content-Type", utils.GetMimeType(name)) + httpReq.Header.Set("Authorization", setting.GetStr(conf.Token)) + httpReq.ContentLength = info.Size() + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return toolErrorf("upload request failed: %s", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return toolErrorf("upload failed (HTTP %d): %s", resp.StatusCode, string(body)) + } + return textResult("uploaded successfully") +} From e35abf51b622d37ed55e87643804e1a7b6e9dda2 Mon Sep 17 00:00:00 2001 From: orbisai0security Date: Wed, 22 Apr 2026 14:22:55 +0530 Subject: [PATCH 634/659] fix: V-002 security vulnerability Automated security fix generated by Orbis Security AI --- drivers/local/util.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/drivers/local/util.go b/drivers/local/util.go index fa95ddd24a0..bbd652bd7e5 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -41,8 +41,32 @@ func isLinkedDir(f fs.FileInfo, path string) bool { return false } +// sanitizeFilePath validates and sanitizes a file path before passing it to external commands. +// It ensures the path is absolute, clean, and refers to an existing regular file, +// preventing path traversal and command injection via shell metacharacters. +func sanitizeFilePath(path string) (string, error) { + cleaned := filepath.Clean(path) + if !filepath.IsAbs(cleaned) { + return "", fmt.Errorf("file path must be absolute: %s", path) + } + info, err := os.Stat(cleaned) + if err != nil { + return "", fmt.Errorf("file path is not accessible: %w", err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("path is not a regular file: %s", cleaned) + } + return cleaned, nil +} + // resizeImageToBufferWithFFmpegGo 使用 ffmpeg-go 调整图片大小并输出到内存缓冲区 func resizeImageToBufferWithFFmpegGo(inputFile string, width int, outputFormat string /* e.g., "image2pipe", "png_pipe", "mjpeg" */) (*bytes.Buffer, error) { + sanitized, err := sanitizeFilePath(inputFile) + if err != nil { + return nil, fmt.Errorf("invalid input file path: %w", err) + } + inputFile = sanitized + outBuffer := bytes.NewBuffer(nil) // Determine codec based on desired output format for piping @@ -124,6 +148,12 @@ func generateThumbnailWithImagingOptimized(imagePath string, targetWidth int, qu // Get the snapshot of the video func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) { + sanitized, err := sanitizeFilePath(videoPath) + if err != nil { + return nil, fmt.Errorf("invalid video path: %w", err) + } + videoPath = sanitized + // Run ffprobe to get the video duration jsonOutput, err := ffmpeg.Probe(videoPath) if err != nil { From 135a0b433e54d4c7ea0e7202e710b7c6901ad769 Mon Sep 17 00:00:00 2001 From: kyle-meng <2078739489@qq.com> Date: Sun, 26 Apr 2026 14:41:44 +0800 Subject: [PATCH 635/659] =?UTF-8?q?feat(139-share):=20support=20mounting?= =?UTF-8?q?=20and=20HLS=20playback=20|=20=E6=94=AF=E6=8C=81=E5=88=86?= =?UTF-8?q?=E4=BA=AB=E9=93=BE=E6=8E=A5=E6=8C=82=E8=BD=BD=E4=B8=8E=E6=92=AD?= =?UTF-8?q?=E6=94=BE=20Root=20cause:=20139=20Cloud=20share=20links=20use?= =?UTF-8?q?=20relative=20TS=20paths=20in=20M3U8=20playlists=20which=20cann?= =?UTF-8?q?ot=20be=20resolved=20by=20proxied=20clients.=20Additionally,=20?= =?UTF-8?q?AList's=20downloader=20enforces=20strict=20metadata-to-stream?= =?UTF-8?q?=20size=20validation,=20leading=20to=20416=20(Range)=20or=20EOF?= =?UTF-8?q?=20errors=20when=20serving=20dynamic=20M3U8=20content.=20We=20i?= =?UTF-8?q?mplemented=20a=201MB=20padding=20technique=20to=20ensure=20comp?= =?UTF-8?q?atibility=20with=20AList's=20strict=20size=20checks;=201MB=20is?= =?UTF-8?q?=20sufficient=20for=20almost=20all=20M3U8=20files=20without=20i?= =?UTF-8?q?mpacting=20performance.=20|=20139=E4=BA=91=E7=9B=98=E5=88=86?= =?UTF-8?q?=E4=BA=AB=E9=93=BE=E6=8E=A5=E5=9C=A8M3U8=E4=B8=AD=E4=BD=BF?= =?UTF-8?q?=E7=94=A8=E7=9B=B8=E5=AF=B9TS=E8=B7=AF=E5=BE=84=EF=BC=8C?= =?UTF-8?q?=E5=AF=BC=E8=87=B4=E4=BB=A3=E7=90=86=E8=AF=B7=E6=B1=82=E6=97=A0?= =?UTF-8?q?=E6=B3=95=E6=AD=A3=E5=B8=B8=E8=A7=A3=E6=9E=90=E3=80=82=E6=AD=A4?= =?UTF-8?q?=E5=A4=96=EF=BC=8CAList=E4=B8=8B=E8=BD=BD=E5=99=A8=E4=BC=9A?= =?UTF-8?q?=E4=B8=A5=E6=A0=BC=E6=A0=A1=E9=AA=8C=E6=96=87=E4=BB=B6=E5=85=83?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E4=B8=8E=E5=AE=9E=E9=99=85=E6=B5=81=E7=9A=84?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E4=B8=80=E8=87=B4=E6=80=A7=EF=BC=8C=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=8A=A8=E6=80=81=E7=94=9F=E6=88=90=E7=9A=84M3U8?= =?UTF-8?q?=E5=9B=A0=E9=95=BF=E5=BA=A6=E4=B8=8D=E5=8C=B9=E9=85=8D=E8=A7=A6?= =?UTF-8?q?=E5=8F=91416=E6=88=96EOF=E9=94=99=E8=AF=AF=E3=80=82=E6=88=91?= =?UTF-8?q?=E4=BB=AC=E9=87=87=E7=94=A8=E4=BA=861MB=E5=A1=AB=E5=85=85?= =?UTF-8?q?=E6=8A=80=E6=9C=AF=E4=BB=A5=E5=85=BC=E5=AE=B9AList=E7=9A=84?= =?UTF-8?q?=E4=B8=A5=E6=A0=BC=E6=A0=A1=E9=AA=8C=EF=BC=8C=E4=B8=941MB?= =?UTF-8?q?=E8=B6=B3=E4=BB=A5=E5=AE=B9=E7=BA=B3=E7=BB=9D=E5=A4=A7=E5=A4=9A?= =?UTF-8?q?=E6=95=B0M3U8=E6=96=87=E4=BB=B6=E8=80=8C=E4=B8=8D=E5=BD=B1?= =?UTF-8?q?=E5=93=8D=E6=80=A7=E8=83=BD=E3=80=82=20Changes:=20alist/drivers?= =?UTF-8?q?/139/types.go=20=20=20-=20Added=20ShareCatalog=20and=20ShareCon?= =?UTF-8?q?tent=20structs=20for=20API=20response=20mapping=20|=20=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=88=86=E4=BA=AB=E7=9B=AE=E5=BD=95=E4=B8=8E=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E7=9A=84API=E5=93=8D=E5=BA=94=E6=98=A0=E5=B0=84?= =?UTF-8?q?=E7=BB=93=E6=9E=84=E4=BD=93=20alist/drivers/139/meta.go=20=20?= =?UTF-8?q?=20-=20Integrated=20'share'=20storage=20type=20and=20simplified?= =?UTF-8?q?=20struct=20tags=20for=20UI=20cleanliness=20|=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E5=88=86=E4=BA=AB=E5=AD=98=E5=82=A8=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=B9=B6=E7=B2=BE=E7=AE=80=E7=BB=93=E6=9E=84=E4=BD=93=E6=A0=87?= =?UTF-8?q?=E7=AD=BE=E4=BB=A5=E7=A1=AE=E4=BF=9D=E7=95=8C=E9=9D=A2=E6=95=B4?= =?UTF-8?q?=E6=B4=81=20alist/drivers/139/driver.go=20=20=20-=20Implemented?= =?UTF-8?q?=20share=20mode=20handling=20and=20forced=201MB=20file=20size?= =?UTF-8?q?=20for=20video=20listings=20|=20=E5=AE=9E=E7=8E=B0=E5=88=86?= =?UTF-8?q?=E4=BA=AB=E6=A8=A1=E5=BC=8F=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= =?UTF-8?q?=E5=B9=B6=E5=9C=A8=E5=88=97=E8=A1=A8=E6=97=B6=E5=B0=86=E8=A7=86?= =?UTF-8?q?=E9=A2=91=E5=A4=A7=E5=B0=8F=E5=BC=BA=E5=88=B6=E5=A3=B0=E6=98=8E?= =?UTF-8?q?=E4=B8=BA1MB=20alist/drivers/139/util.go=20=20=20-=20Implemente?= =?UTF-8?q?d=20M3U8=20absolute=20URL=20rewriter=20and=20a=20padded=20Range?= =?UTF-8?q?ReadCloser=20to=20ensure=20proxy=20compatibility=20|=20?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0M3U8=E7=BB=9D=E5=AF=B9=E8=B7=AF=E5=BE=84?= =?UTF-8?q?=E9=87=8D=E5=86=99=E5=99=A8=E5=8F=8A=E5=B8=A6=E5=A1=AB=E5=85=85?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E7=9A=84=E8=AF=BB=E5=8F=96=E5=99=A8=E4=BB=A5?= =?UTF-8?q?=E9=80=82=E9=85=8D=E4=BB=A3=E7=90=86=E6=A0=A1=E9=AA=8C=20=20=20?= =?UTF-8?q?-=20Cleaned=20up=20all=20debug=20logging=20and=20temporary=20co?= =?UTF-8?q?de=20for=20production=20readiness=20|=20=E6=B8=85=E7=90=86?= =?UTF-8?q?=E4=BA=86=E6=89=80=E6=9C=89=E8=B0=83=E8=AF=95=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=92=8C=E4=B8=B4=E6=97=B6=E4=BB=A3=E7=A0=81=E4=BB=A5=E8=BE=BE?= =?UTF-8?q?=E5=88=B0=E5=8F=91=E5=B8=83=E6=A0=87=E5=87=86=20Verified:=20Suc?= =?UTF-8?q?cessfully=20mounted=20share=20links;=20shared=20videos=20play?= =?UTF-8?q?=20via=20HLS=20without=20416=20errors;=20padded=20content=20siz?= =?UTF-8?q?e=20matches=20the=201MB=20metadata.=20|=20=E6=88=90=E5=8A=9F?= =?UTF-8?q?=E6=8C=82=E8=BD=BD=E5=88=86=E4=BA=AB=E9=93=BE=E6=8E=A5=EF=BC=9B?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E5=8F=AF=E9=80=9A=E8=BF=87HLS=E6=AD=A3?= =?UTF-8?q?=E5=B8=B8=E6=92=AD=E6=94=BE=E4=B8=94=E6=97=A0416=E9=94=99?= =?UTF-8?q?=E8=AF=AF=EF=BC=9B=E5=A1=AB=E5=85=85=E5=90=8E=E7=9A=84=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E5=A4=A7=E5=B0=8F=E4=B8=8E=E5=A3=B0=E6=98=8E=E7=9A=84?= =?UTF-8?q?1MB=E5=85=83=E6=95=B0=E6=8D=AE=E5=AE=8C=E7=BE=8E=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/139/driver.go | 9 ++ drivers/139/meta.go | 10 +- drivers/139/types.go | 36 +++++ drivers/139/util.go | 340 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 390 insertions(+), 5 deletions(-) diff --git a/drivers/139/driver.go b/drivers/139/driver.go index c182dd24e93..10d9d3e9e03 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -76,6 +76,10 @@ func (d *Yun139) Init(ctx context.Context) error { d.RootFolderID = d.CloudID } case MetaFamily: + case "share": + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = "root" + } default: return errs.NotImplement } @@ -100,6 +104,7 @@ func (d *Yun139) Drop(ctx context.Context) error { } func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + log.Infof("[139Share-Debug] List called! Type: %s, DirID: %s", d.Addition.Type, dir.GetID()) switch d.Addition.Type { case MetaPersonalNew: return d.personalGetFiles(dir.GetID()) @@ -109,6 +114,8 @@ func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( return d.familyGetFiles(dir.GetID()) case MetaGroup: return d.groupGetFiles(dir.GetID()) + case "share": + return d.shareGetFiles(dir.GetID()) default: return nil, errs.NotImplement } @@ -126,6 +133,8 @@ func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) url, err = d.familyGetLink(file.GetID(), file.GetPath()) case MetaGroup: url, err = d.groupGetLink(file.GetID(), file.GetPath()) + case "share": + return d.shareGetLink(file.GetID()) default: return nil, errs.NotImplement } diff --git a/drivers/139/meta.go b/drivers/139/meta.go index c02b1347587..ea90df86cf9 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -6,14 +6,14 @@ import ( ) type Addition struct { - //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` driver.RootID - Type string `json:"type" type:"select" options:"personal_new,family,group,personal" default:"personal_new"` + Type string `json:"type" type:"select" options:"personal_new,family,group,personal,share" default:"personal_new"` CloudID string `json:"cloud_id"` - CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` - ReportRealSize bool `json:"report_real_size" type:"bool" default:"true" help:"Enable to report the real file size during upload"` - UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false" help:"Enable to use large thumbnail for images"` + LinkID string `json:"link_id"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0"` + ReportRealSize bool `json:"report_real_size" type:"bool" default:"true"` + UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false"` } var config = driver.Config{ diff --git a/drivers/139/types.go b/drivers/139/types.go index d5f025a1672..e3a2adc68b7 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -312,3 +312,39 @@ type RefreshTokenResp struct { AccessToken string `xml:"accessToken"` Desc string `xml:"desc"` } + +type ShareCatalog struct { + CaID string `json:"caID"` + CaName string `json:"caName"` +} + +type ShareContent struct { + CoID string `json:"coID"` + CoName string `json:"coName"` + CoSize int64 `json:"coSize"` + CoType int `json:"coType"` + CoSuffix string `json:"coSuffix"` +} + +type ShareListResp struct { + Data struct { + CaLst []ShareCatalog `json:"caLst"` + CoLst []ShareContent `json:"coLst"` + } `json:"data"` +} + +type ShareLinkResp struct { + DownloadURL string `json:"downloadURL"` +} +type ShareContentInfo struct { + ContentName string `json:"contentName"` + ContentSize int64 `json:"contentSize"` + PresentURL string `json:"presentURL"` // HLS 播放地址 + DownloadURL string `json:"cdnDownLoadUrl"` // 真正的下载直链 +} + +type ShareContentInfoResp struct { + Data struct { + ContentInfo ShareContentInfo `json:"contentInfo"` + } `json:"data"` +} \ No newline at end of file diff --git a/drivers/139/util.go b/drivers/139/util.go index 79c78842470..2bece66ba4a 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1,9 +1,16 @@ package _139 import ( + "bytes" + "compress/gzip" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" "encoding/base64" "errors" "fmt" + "io" "net/http" "net/url" "path" @@ -12,11 +19,14 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" @@ -666,3 +676,333 @@ func (d *Yun139) getPersonalCloudHost() string { } return d.PersonalCloudHost } + +func (d *Yun139) sharePost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + crypto := NewYunCrypto() + encryptedBody, err := crypto.Encrypt(data) + if err != nil { + return nil, err + } + + url := "https://share-kd-njs.yun.139.com" + pathname + req := base.RestyClient.R() + + auth := d.getAuthorization() + if !strings.HasPrefix(auth, "Basic ") { + auth = "Basic " + auth + } + // randStr := random.String(16) + // ts := time.Now().Format("2006-01-02 15:04:05") + // body, err := utils.Json.Marshal(req.Body) + // if err != nil { + // return nil, err + // } + // sign := calSign(string(body), ts, randStr) + // svcType := "1" + // if d.isFamily() { + // svcType = "2" + // } + req.SetHeaders(map[string]string{ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0", + "Accept": "application/json, text/plain, */*", + "Content-Type": "application/json;charset=UTF-8", + "Authorization": auth, + "X-Deviceinfo": "||9|12.27.0|firefox|140.0|12b780037221ab547c682223327dc9cd||linux unknow|1920X526|zh-CN|||", + "hcy-cool-flag": "1", + "CMS-DEVICE": "default", + "x-m4c-caller": "PC", + "X-Yun-Api-Version": "v1", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/", + }) + req.SetBody(encryptedBody) + + res, err := req.Post(url) + if err != nil { + return nil, err + } + + decryptedText, err := crypto.Decrypt(res.String()) + if err != nil { + log.Errorf("[139Share] Decryption failed, raw response: %s", res.String()) + return nil, fmt.Errorf("decryption failed: %v, raw: %s", err, res.String()) + } + + if resp != nil { + err = utils.Json.Unmarshal([]byte(decryptedText), resp) + if err != nil { + return nil, err + } + } + return []byte(decryptedText), nil +} + +func (d *Yun139) shareGetFiles(pCaID string) ([]model.Obj, error) { + if pCaID == "" { + pCaID = "root" + } + data := base.Json{ + "getOutLinkInfoReq": base.Json{ + "account": d.getAccount(), + "linkID": d.LinkID, + "pCaID": pCaID, + }, + } + var resp ShareListResp + _, err := d.sharePost("/yun-share/richlifeApp/devapp/IOutLink/getOutLinkInfoV6", data, &resp) + if err != nil { + return nil, err + } + files := make([]model.Obj, 0) + // 直接从 Data 中读取 CaLst + for _, catalog := range resp.Data.CaLst { + f := model.Object{ + ID: catalog.CaID, + Name: catalog.CaName, + IsFolder: true, + } + files = append(files, &f) + } + for _, content := range resp.Data.CoLst { + name := content.CoName + size := content.CoSize + // 如果是视频，强行加 .m3u8 后缀，并声明为 1MB 以匹配 Padding 逻辑 + if content.CoType == 3 || strings.HasSuffix(strings.ToLower(name), ".mp4") { + if !strings.HasSuffix(name, ".m3u8") { + name += ".m3u8" + } + size = 1024 * 1024 // 关键：声明为 1MB + } + f := model.Object{ + ID: content.CoID, + Name: name, + Size: size, + } + files = append(files, &f) + } + + return files, nil +} + + + +type YunCrypto struct { + Key []byte + BlockSize int +} + +func NewYunCrypto() *YunCrypto { + return &YunCrypto{ + Key: []byte("PVGDwmcvfs1uV3d1"), + BlockSize: aes.BlockSize, + } +} + +func (y *YunCrypto) PKCS7Padding(ciphertext []byte, blockSize int) []byte { + padding := blockSize - len(ciphertext)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(ciphertext, padtext...) +} + +func (y *YunCrypto) PKCS7UnPadding(origData []byte) ([]byte, error) { + length := len(origData) + if length == 0 { + return nil, errors.New("data is empty") + } + unpadding := int(origData[length-1]) + if length < unpadding { + return nil, errors.New("unpadding error") + } + return origData[:(length - unpadding)], nil +} + +func (y *YunCrypto) Encrypt(data interface{}) (string, error) { + jsonData, err := utils.Json.Marshal(data) + if err != nil { + return "", err + } + iv := make([]byte, y.BlockSize) + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return "", err + } + block, err := aes.NewCipher(y.Key) + if err != nil { + return "", err + } + content := y.PKCS7Padding(jsonData, y.BlockSize) + ciphertext := make([]byte, len(content)) + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(ciphertext, content) + result := append(iv, ciphertext...) + return base64.StdEncoding.EncodeToString(result), nil +} + +func (y *YunCrypto) Decrypt(b64Data string) (string, error) { + b64Data = strings.Join(strings.Fields(b64Data), "") + raw, err := base64.StdEncoding.DecodeString(b64Data) + if err != nil { + return "", err + } + if len(raw) < y.BlockSize { + return "", errors.New("data too short") + } + iv := raw[:y.BlockSize] + ciphertext := raw[y.BlockSize:] + block, err := aes.NewCipher(y.Key) + if err != nil { + return "", err + } + decrypted := make([]byte, len(ciphertext)) + mode := cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(decrypted, ciphertext) + if len(decrypted) > 2 && decrypted[0] == 0x1f && decrypted[1] == 0x8b { + reader, err := gzip.NewReader(bytes.NewReader(decrypted)) + if err == nil { + defer reader.Close() + unzipped, err := io.ReadAll(reader) + if err == nil { + return string(unzipped), nil + } + } + } + unpadded, err := y.PKCS7UnPadding(decrypted) + if err != nil { + return strings.TrimSpace(string(decrypted)), nil + } + return string(unpadded), nil +} + +func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { + client := resty.New().SetTimeout(10 * time.Second) + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Referer": "https://yun.139.com/", + } + + // 1. 获取 Master M3U8 + resp, err := client.R().SetHeaders(headers).Get(masterURL) + if err != nil { + return "", err + } + masterContent := resp.String() + + // 2. 找到子播放列表路径 + var subRelPath string + lines := strings.Split(masterContent, "\n") + for i, line := range lines { + if strings.Contains(line, "RESOLUTION=") { + if i+1 < len(lines) { + subRelPath = strings.TrimSpace(lines[i+1]) + if strings.Contains(line, "1920x1080") { + break + } + } + } + } + if subRelPath == "" { + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line != "" && !strings.HasPrefix(line, "#") { + subRelPath = line + break + } + } + } + if subRelPath == "" { + return "", fmt.Errorf("could not find sub-playlist") + } + + // 3. 获取子播放列表内容 + base, _ := url.Parse(masterURL) + if subRelPath == "" { + return "", fmt.Errorf("sub playlist not found in master m3u8") + } + ref, _ := url.Parse(subRelPath) + subURL := base.ResolveReference(ref).String() + + resp, err = client.R().SetHeaders(headers).Get(subURL) + if err != nil { + return "", err + } + subContent := resp.String() + + subBase, _ := url.Parse(subURL) + subLines := strings.Split(subContent, "\n") + var finalLines []string + for _, line := range subLines { + cleanLine := strings.TrimSpace(line) + if cleanLine != "" && !strings.HasPrefix(cleanLine, "#") { + if !strings.HasPrefix(cleanLine, "http") { + tsRef, _ := url.Parse(cleanLine) + finalLines = append(finalLines, subBase.ResolveReference(tsRef).String()) + } else { + finalLines = append(finalLines, cleanLine) + } + } else { + finalLines = append(finalLines, line) + } + } + + finalM3U8 := strings.Join(finalLines, "\n") + + return finalM3U8, nil +} + +func (d *Yun139) Proxy(c *gin.Context, obj model.Obj) error { + return nil +} + +func (d *Yun139) shareGetLink(coID string) (*model.Link, error) { + data := base.Json{ + "getContentInfoFromOutLinkReq": base.Json{ + "contentId": coID, + "linkID": d.LinkID, + "account": d.getAccount(), + }, + } + var resp ShareContentInfoResp + _, err := d.sharePost("/yun-share/richlifeApp/devapp/IOutLink/getContentInfoFromOutLink", data, &resp) + if err != nil { + return nil, err + } + + res := resp.Data.ContentInfo + if res.PresentURL != "" { + m3u8Content, err := d.rewriteM3U8(res.PresentURL) + if err != nil { + return nil, err + } + + // 核心逻辑：填充到 1MB，确保 AList 不报大小错误 + targetSize := int64(1024 * 1024) + contentBytes := []byte(m3u8Content) + if int64(len(contentBytes)) < targetSize { + padding := bytes.Repeat([]byte(" "), int(targetSize-int64(len(contentBytes)))) + contentBytes = append(contentBytes, padding...) + } else { + // 如果 M3U8 竟然超过了 1MB（极罕见），则按实际大小截断（或报错） + contentBytes = contentBytes[:targetSize] + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, range_ http_range.Range) (io.ReadCloser, error) { + reader := bytes.NewReader(contentBytes) + // 处理 AList 的 Range 请求 + _, _ = reader.Seek(range_.Start, io.SeekStart) + // 包装成 ReadCloser + return io.NopCloser(reader), nil + }, + }, + Header: http.Header{ + "Content-Type": []string{"application/vnd.apple.mpegurl"}, + }, + }, nil + } + + if res.DownloadURL != "" { + return &model.Link{URL: res.DownloadURL}, nil + } + + return nil, fmt.Errorf("failed to get link") +} From 45daac9e04e4e3471758468b7d17e02516a553f5 Mon Sep 17 00:00:00 2001 From: kyle-meng <2078739489@qq.com> Date: Sun, 26 Apr 2026 16:02:23 +0800 Subject: [PATCH 636/659] =?UTF-8?q?fix(139-share):=20fix=20modification=20?= =?UTF-8?q?time=20parsing=20|=20=E4=BF=AE=E5=A4=8D=E5=88=86=E4=BA=AB?= =?UTF-8?q?=E6=A8=A1=E5=BC=8F=E4=B8=8B=E7=9A=84=E4=BF=AE=E6=94=B9=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- drivers/139/types.go | 2 ++ drivers/139/util.go | 34 ++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/drivers/139/types.go b/drivers/139/types.go index e3a2adc68b7..03f88aa7aba 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -316,6 +316,7 @@ type RefreshTokenResp struct { type ShareCatalog struct { CaID string `json:"caID"` CaName string `json:"caName"` + UdTime string `json:"udTime"` } type ShareContent struct { @@ -323,6 +324,7 @@ type ShareContent struct { CoName string `json:"coName"` CoSize int64 `json:"coSize"` CoType int `json:"coType"` + UdTime string `json:"udTime"` CoSuffix string `json:"coSuffix"` } diff --git a/drivers/139/util.go b/drivers/139/util.go index 2bece66ba4a..3823569ad70 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -756,9 +756,11 @@ func (d *Yun139) shareGetFiles(pCaID string) ([]model.Obj, error) { files := make([]model.Obj, 0) // 直接从 Data 中读取 CaLst for _, catalog := range resp.Data.CaLst { + modTime, _ := time.ParseInLocation("20060102150405", catalog.UdTime, utils.CNLoc) f := model.Object{ ID: catalog.CaID, Name: catalog.CaName, + Modified: modTime, IsFolder: true, } files = append(files, &f) @@ -766,17 +768,19 @@ func (d *Yun139) shareGetFiles(pCaID string) ([]model.Obj, error) { for _, content := range resp.Data.CoLst { name := content.CoName size := content.CoSize - // 如果是视频，强行加 .m3u8 后缀，并声明为 1MB 以匹配 Padding 逻辑 + // Force .m3u8 suffix for videos and declare 1MB size for padding logic if content.CoType == 3 || strings.HasSuffix(strings.ToLower(name), ".mp4") { if !strings.HasSuffix(name, ".m3u8") { name += ".m3u8" } - size = 1024 * 1024 // 关键：声明为 1MB + size = 1024 * 1024 // Key: declare 1MB to match RangeReadCloser padding } + modTime, _ := time.ParseInLocation("20060102150405", content.UdTime, utils.CNLoc) f := model.Object{ - ID: content.CoID, - Name: name, - Size: size, + ID: content.CoID, + Name: name, + Size: size, + Modified: modTime, } files = append(files, &f) } @@ -879,14 +883,14 @@ func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { "Referer": "https://yun.139.com/", } - // 1. 获取 Master M3U8 + // 1. Get Master M3U8 resp, err := client.R().SetHeaders(headers).Get(masterURL) if err != nil { return "", err } masterContent := resp.String() - // 2. 找到子播放列表路径 + // 2. Find sub-playlist path var subRelPath string lines := strings.Split(masterContent, "\n") for i, line := range lines { @@ -909,14 +913,11 @@ func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { } } if subRelPath == "" { - return "", fmt.Errorf("could not find sub-playlist") + return "", fmt.Errorf("sub playlist not found in master m3u8") } - // 3. 获取子播放列表内容 + // 3. Get sub-playlist content base, _ := url.Parse(masterURL) - if subRelPath == "" { - return "", fmt.Errorf("sub playlist not found in master m3u8") - } ref, _ := url.Parse(subRelPath) subURL := base.ResolveReference(ref).String() @@ -926,6 +927,7 @@ func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { } subContent := resp.String() + // 4. Resolve relative TS paths to absolute URLs subBase, _ := url.Parse(subURL) subLines := strings.Split(subContent, "\n") var finalLines []string @@ -973,14 +975,14 @@ func (d *Yun139) shareGetLink(coID string) (*model.Link, error) { return nil, err } - // 核心逻辑：填充到 1MB，确保 AList 不报大小错误 + // Core logic: pad to 1MB to ensure compatibility with AList's size validation targetSize := int64(1024 * 1024) contentBytes := []byte(m3u8Content) if int64(len(contentBytes)) < targetSize { padding := bytes.Repeat([]byte(" "), int(targetSize-int64(len(contentBytes)))) contentBytes = append(contentBytes, padding...) } else { - // 如果 M3U8 竟然超过了 1MB（极罕见），则按实际大小截断（或报错） + // Truncate if M3U8 exceeds 1MB (extremely rare) contentBytes = contentBytes[:targetSize] } @@ -988,9 +990,9 @@ func (d *Yun139) shareGetLink(coID string) (*model.Link, error) { RangeReadCloser: &model.RangeReadCloser{ RangeReader: func(ctx context.Context, range_ http_range.Range) (io.ReadCloser, error) { reader := bytes.NewReader(contentBytes) - // 处理 AList 的 Range 请求 + // Handle AList Range requests _, _ = reader.Seek(range_.Start, io.SeekStart) - // 包装成 ReadCloser + // Wrap as ReadCloser return io.NopCloser(reader), nil }, }, From dba5c279ca72faa322cde125cc0b62566a39153b Mon Sep 17 00:00:00 2001 From: abandonstudy <94209629+abandonstudy@users.noreply.github.com> Date: Thu, 30 Apr 2026 22:43:14 +0800 Subject: [PATCH 637/659] fix(guangyapan): allow user input folder path in driver root path fix #9493 allow users to mount a guangyapan subfolder --- drivers/guangyapan/driver.go | 137 ++++++++++++++++++++++++++++++----- drivers/guangyapan/meta.go | 4 +- 2 files changed, 121 insertions(+), 20 deletions(-) diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go index ff83c9a88fa..8c80ebf3565 100644 --- a/drivers/guangyapan/driver.go +++ b/drivers/guangyapan/driver.go @@ -33,6 +33,9 @@ type GuangYaPan struct { accountClient *resty.Client apiClient *resty.Client + + resolvedRootFolderID string + rootFolderResolved bool } func (d *GuangYaPan) Config() driver.Config { @@ -62,12 +65,15 @@ func (d *GuangYaPan) Init(ctx context.Context) error { d.SortType = 1 } + d.RootPath = strings.TrimSpace(d.RootPath) d.AccessToken = strings.TrimSpace(d.AccessToken) d.RefreshToken = strings.TrimSpace(d.RefreshToken) d.PhoneNumber = strings.TrimSpace(d.PhoneNumber) d.VerifyCode = strings.TrimSpace(d.VerifyCode) d.CaptchaToken = strings.TrimSpace(d.CaptchaToken) d.VerificationID = strings.TrimSpace(d.VerificationID) + d.resolvedRootFolderID = "" + d.rootFolderResolved = false d.accountClient = base.NewRestyClient(). SetBaseURL(accountBaseURL). @@ -99,14 +105,14 @@ func (d *GuangYaPan) Init(ctx context.Context) error { // Priority: access_token -> refresh_token -> sms login. if d.AccessToken != "" { if err := d.validateToken(ctx); err == nil { - return nil + return d.prepareRootFolder(ctx) } d.AccessToken = "" } if d.RefreshToken != "" { if err := d.refreshToken(ctx); err == nil { if err2 := d.validateToken(ctx); err2 == nil { - return nil + return d.prepareRootFolder(ctx) } } } @@ -118,7 +124,10 @@ func (d *GuangYaPan) Init(ctx context.Context) error { if err := d.loginBySMSCode(ctx); err != nil { return err } - return d.validateToken(ctx) + if err := d.validateToken(ctx); err != nil { + return err + } + return d.prepareRootFolder(ctx) } if d.SendCode { d.setTempStatus("SMS sending in progress...") @@ -138,15 +147,27 @@ func (d *GuangYaPan) Drop(ctx context.Context) error { return nil } +func (d *GuangYaPan) GetRoot(ctx context.Context) (model.Obj, error) { + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + return &model.Object{ + ID: rootID, + Path: "/", + Name: "root", + Size: 0, + Modified: d.Modified, + IsFolder: true, + }, nil +} + func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { if err := d.ensureAccessToken(ctx); err != nil { return nil, err } parentID := dir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } res := make([]model.Obj, 0, d.PageSize) for page := 0; ; page++ { @@ -219,9 +240,6 @@ func (d *GuangYaPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName s } parentID := parentDir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } var out createDirResp if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/create_dir", map[string]any{ @@ -301,9 +319,6 @@ func (d *GuangYaPan) Move(ctx context.Context, srcObj, dstDir model.Obj) error { return errors.New("file id is empty") } parentID := dstDir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } var out deleteResp if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/move_file", map[string]any{ @@ -332,9 +347,6 @@ func (d *GuangYaPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return errors.New("file id is empty") } parentID := dstDir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } var out deleteResp if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/copy_file", map[string]any{ @@ -369,9 +381,6 @@ func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileS } parentID := dstDir.GetID() - if parentID == d.RootFolderID { - parentID = "" - } token, code, err := d.getUploadToken(ctx, parentID, name, file.GetSize()) if err != nil { @@ -415,6 +424,97 @@ func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileS return d.waitUploadTaskInfo(ctx, taskID) } +func (d *GuangYaPan) getRootFolderID(ctx context.Context) (string, error) { + if d.rootFolderResolved { + return d.resolvedRootFolderID, nil + } + if err := d.ensureAccessToken(ctx); err != nil { + return "", err + } + if err := d.prepareRootFolder(ctx); err != nil { + return "", err + } + return d.resolvedRootFolderID, nil +} + +func (d *GuangYaPan) prepareRootFolder(ctx context.Context) error { + rootID, err := d.resolveConfiguredRootFolderID(ctx) + if err != nil { + return err + } + d.resolvedRootFolderID = rootID + d.rootFolderResolved = true + return nil +} + +func (d *GuangYaPan) resolveConfiguredRootFolderID(ctx context.Context) (string, error) { + root := strings.TrimSpace(d.RootPath) + if root == "" { + return "", nil + } + return d.resolveFolderPath(ctx, root) +} + +func (d *GuangYaPan) resolveFolderPath(ctx context.Context, rootPath string) (string, error) { + cleanPath := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rootPath), "\\", "/"), "/") + if cleanPath == "" { + return "", nil + } + + parentID := "" + for _, name := range strings.Split(cleanPath, "/") { + if name == "" { + continue + } + childID, err := d.findChildFolderID(ctx, parentID, name) + if err != nil { + return "", err + } + parentID = childID + } + return parentID, nil +} + +func (d *GuangYaPan) findChildFolderID(ctx context.Context, parentID, name string) (string, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 100 + } + + seen := 0 + for page := 0; ; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": pageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + "fileTypes": []int{}, + } + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + return "", err + } + for _, item := range resp.Data.List { + seen++ + if item.ResType == 2 && item.FileName == name { + return item.FileID, nil + } + } + if len(resp.Data.List) < pageSize { + break + } + if resp.Data.Total > 0 && seen >= resp.Data.Total { + break + } + } + + if parentID == "" { + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under /", name) + } + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under parent %s", name, parentID) +} + func (d *GuangYaPan) ensureAccessToken(ctx context.Context) error { if strings.TrimSpace(d.AccessToken) != "" { return nil @@ -948,3 +1048,4 @@ func randomDeviceID() string { } var _ driver.Driver = (*GuangYaPan)(nil) +var _ driver.GetRooter = (*GuangYaPan)(nil) \ No newline at end of file diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go index 606d6aec8ee..434a151655e 100644 --- a/drivers/guangyapan/meta.go +++ b/drivers/guangyapan/meta.go @@ -6,7 +6,7 @@ import ( ) type Addition struct { - driver.RootID + RootPath string `json:"root_path" help:"光鸭云盘中的完整路径"` PhoneNumber string `json:"phone_number" type:"text" help:"Phone number for SMS login, e.g. +86 13800000000"` CaptchaToken string `json:"captcha_token" type:"text" help:"Captcha token required by /v1/auth/verification"` SendCode bool `json:"send_code" type:"bool" help:"Set true and save to send SMS code, it auto-resets to false after sending"` @@ -39,4 +39,4 @@ func init() { op.RegisterDriver(func() driver.Driver { return &GuangYaPan{} }) -} +} \ No newline at end of file From ffbbe7a96fbe175f04f44b00f84acf24aff29224 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 7 May 2026 14:45:31 +0800 Subject: [PATCH 638/659] fix(storage): clear list cache after storage updates --- internal/op/storage.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/op/storage.go b/internal/op/storage.go index dfb305aaa43..3961e32ed9a 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -229,10 +229,13 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { if err != nil { return errors.WithMessage(err, "failed update storage in database") } + storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) + if err == nil { + ClearCache(storageDriver, "/") + } if storage.Disabled { return nil } - storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) From cbeb088d40fac6949f3e74871969ac1eb94f7f90 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 7 May 2026 14:45:38 +0800 Subject: [PATCH 639/659] feat(settings): add frontend sort memory switch --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 43 +++++++++++++++--------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 265a6502b60..0d39a377915 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -108,6 +108,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Group: model.SITE}, // newui settings {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, + {Key: conf.FrontendRememberSort, Value: "false", Type: conf.TypeBool, Group: model.SITE, Help: "Persist frontend list sorting in the browser. When disabled, backend/driver order is used until the user sorts manually in the current session."}, // style settings {Key: conf.Logo, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE}, {Key: conf.Favicon, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 5bcb6eb5f91..aaa2fac8349 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -10,15 +10,16 @@ const ( const ( // site - VERSION = "version" - SiteTitle = "site_title" - Announcement = "announcement" - AllowIndexed = "allow_indexed" - AllowMounted = "allow_mounted" - RobotsTxt = "robots_txt" - AllowRegister = "allow_register" - DefaultRole = "default_role" - UseNewui = "use_newui" + VERSION = "version" + SiteTitle = "site_title" + Announcement = "announcement" + AllowIndexed = "allow_indexed" + AllowMounted = "allow_mounted" + RobotsTxt = "robots_txt" + AllowRegister = "allow_register" + DefaultRole = "default_role" + UseNewui = "use_newui" + FrontendRememberSort = "frontend_remember_sort" Logo = "logo" Favicon = "favicon" @@ -126,19 +127,19 @@ const ( FTPTLSPublicCertPath = "ftp_tls_public_cert_path" // frp - FRPEnabled = "frp_enabled" - FRPServerAddr = "frp_server_addr" - FRPServerPort = "frp_server_port" - FRPAuthToken = "frp_auth_token" - FRPProxyName = "frp_proxy_name" - FRPProxyType = "frp_proxy_type" - FRPCustomDomain = "frp_custom_domain" - FRPSubdomain = "frp_subdomain" - FRPRemotePort = "frp_remote_port" - FRPLocalPort = "frp_local_port" - FRPTLSEnable = "frp_tls_enable" + FRPEnabled = "frp_enabled" + FRPServerAddr = "frp_server_addr" + FRPServerPort = "frp_server_port" + FRPAuthToken = "frp_auth_token" + FRPProxyName = "frp_proxy_name" + FRPProxyType = "frp_proxy_type" + FRPCustomDomain = "frp_custom_domain" + FRPSubdomain = "frp_subdomain" + FRPRemotePort = "frp_remote_port" + FRPLocalPort = "frp_local_port" + FRPTLSEnable = "frp_tls_enable" FRPSTCPSecretKey = "frp_stcp_secret_key" - FRPStatus = "frp_status" + FRPStatus = "frp_status" // traffic TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" From 4a23e6a506b191d702ebb56ac5cdae53e0378a4e Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 7 May 2026 14:45:47 +0800 Subject: [PATCH 640/659] fix(guangyapan): expose sorting options --- drivers/guangyapan/driver.go | 2 +- drivers/guangyapan/meta.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go index ff83c9a88fa..2cbcdecddc6 100644 --- a/drivers/guangyapan/driver.go +++ b/drivers/guangyapan/driver.go @@ -159,7 +159,7 @@ func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArg "sortType": d.SortType, "fileTypes": []int{}, } - if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + if err := d.postAPI(ctx, "/userres/v1/file/get_file_list", body, &resp); err != nil { return nil, err } for _, item := range resp.Data.List { diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go index 606d6aec8ee..506d078b206 100644 --- a/drivers/guangyapan/meta.go +++ b/drivers/guangyapan/meta.go @@ -17,8 +17,8 @@ type Addition struct { ClientID string `json:"client_id" default:"aMe-8VSlkrbQXpUR"` DeviceID string `json:"device_id" help:"Optional custom device id (32 hex chars), auto-generated when empty"` PageSize int `json:"page_size" type:"number" default:"100"` - OrderBy int `json:"order_by" type:"number" default:"3" help:"0:name,1:size,2:create_time,3:update_time"` - SortType int `json:"sort_type" type:"number" default:"1" help:"0:asc,1:desc"` + OrderBy int `json:"order_by" type:"number" options:"0,1,2,3,4" default:"3" help:"Sort field used by the file list"` + SortType int `json:"sort_type" type:"number" options:"0,1" default:"1" help:"Sort direction used by the file list"` } var config = driver.Config{ From 756cc65f41085e9f3bb053f8728b3a968791332a Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 7 May 2026 14:54:52 +0800 Subject: [PATCH 641/659] ci: merge companion frontend pull request --- .github/workflows/merge_frontend_pr.yml | 70 +++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/merge_frontend_pr.yml diff --git a/.github/workflows/merge_frontend_pr.yml b/.github/workflows/merge_frontend_pr.yml new file mode 100644 index 00000000000..f4e3434306b --- /dev/null +++ b/.github/workflows/merge_frontend_pr.yml @@ -0,0 +1,70 @@ +name: merge companion frontend pr + +on: + pull_request: + branches: + - 'main' + types: + - closed + workflow_dispatch: + inputs: + frontend_pr: + description: 'Frontend PR reference, e.g. AlistGo/alist-web#301 or https://github.com/AlistGo/alist-web/pull/301' + required: true + type: string + +permissions: + contents: read + +jobs: + merge_frontend_pr: + name: Merge companion frontend PR + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Find frontend PR + id: frontend + env: + INPUT_FRONTEND_PR: ${{ inputs.frontend_pr }} + PR_BODY: ${{ github.event.pull_request.body }} + run: | + set -euo pipefail + + text="${INPUT_FRONTEND_PR:-${PR_BODY:-}}" + ref="$(printf '%s\n' "$text" | grep -Eo '(https://github\.com/(AlistGo|alist-org)/alist-web/pull/[0-9]+)|((AlistGo|alist-org)/alist-web#[0-9]+)|(alist-web#[0-9]+)' | head -n1 || true)" + + if [[ -z "$ref" ]]; then + echo "found=false" >> "$GITHUB_OUTPUT" + echo "No companion frontend PR referenced." + exit 0 + fi + + if [[ "$ref" == *"/pull/"* ]]; then + number="${ref##*/}" + else + number="${ref##*#}" + fi + + echo "found=true" >> "$GITHUB_OUTPUT" + echo "url=https://github.com/AlistGo/alist-web/pull/$number" >> "$GITHUB_OUTPUT" + echo "Found companion frontend PR #$number." + + - name: Merge frontend PR + if: steps.frontend.outputs.found == 'true' + env: + GH_TOKEN: ${{ secrets.MY_TOKEN }} + FRONTEND_PR: ${{ steps.frontend.outputs.url }} + run: | + set -euo pipefail + + state="$(gh pr view "$FRONTEND_PR" --json state --jq .state)" + if [[ "$state" == "MERGED" ]]; then + echo "$FRONTEND_PR is already merged." + exit 0 + fi + if [[ "$state" != "OPEN" ]]; then + echo "$FRONTEND_PR is $state and cannot be merged." + exit 1 + fi + + gh pr merge "$FRONTEND_PR" --auto --merge || gh pr merge "$FRONTEND_PR" --merge From 1dbd503918da4d30fa795094bd000741e857f24b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 16:44:16 +0800 Subject: [PATCH 642/659] chore(deps): bump github.com/jackc/pgx/v5 from 5.5.5 to 5.9.0 (#9482) Bumps [github.com/jackc/pgx/v5](https://github.com/jackc/pgx) from 5.5.5 to 5.9.0. - [Changelog](https://github.com/jackc/pgx/blob/master/CHANGELOG.md) - [Commits](https://github.com/jackc/pgx/compare/v5.5.5...v5.9.0) --- updated-dependencies: - dependency-name: github.com/jackc/pgx/v5 dependency-version: 5.9.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 19 +++++++++---------- go.sum | 24 ++++++++++++------------ 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index a7accd494f0..bce6deae91e 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.1 github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 @@ -104,6 +104,7 @@ require ( github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect github.com/fatedier/golib v0.5.1 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/klauspost/reedsolomon v1.12.0 // indirect @@ -117,12 +118,14 @@ require ( github.com/relvacode/iso8601 v1.3.0 // indirect github.com/samber/lo v1.47.0 // indirect github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect + github.com/spf13/cast v1.7.1 // indirect github.com/templexxx/cpu v0.1.1 // indirect github.com/templexxx/xorsimd v0.4.3 // indirect github.com/tjfoc/gmsm v1.4.1 // indirect github.com/vishvananda/netlink v1.3.0 // indirect github.com/vishvananda/netns v0.0.4 // indirect github.com/xtaci/kcp-go/v5 v5.6.13 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect golang.org/x/mod v0.27.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect @@ -132,10 +135,6 @@ require ( k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/yaml v1.3.0 // indirect - github.com/google/jsonschema-go v0.4.2 // indirect - github.com/relvacode/iso8601 v1.3.0 // indirect - github.com/spf13/cast v1.7.1 // indirect - github.com/yosida95/uritemplate/v3 v3.0.2 // indirect ) require ( @@ -160,7 +159,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/hekmon/cunits/v2 v2.1.0 // indirect github.com/ipfs/boxo v0.12.0 // indirect - github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/microcosm-cc/bluemonday v1.0.27 @@ -233,8 +232,8 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/ipfs/go-cid v0.4.1 github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -304,10 +303,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.16.0 + golang.org/x/sync v0.17.0 golang.org/x/sys v0.35.0 // indirect golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.28.0 + golang.org/x/text v0.29.0 golang.org/x/tools v0.36.0 // indirect google.golang.org/api v0.169.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect diff --git a/go.sum b/go.sum index 34cc3338327..f8c574fd154 100644 --- a/go.sum +++ b/go.sum @@ -418,12 +418,12 @@ github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2 github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= -github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.0 h1:T/dI+2TvmI2H8s/KH1/lXIbz1CUFk3gn5oTjr0/mBsE= +github.com/jackc/pgx/v5 v5.9.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -682,8 +682,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= @@ -895,8 +895,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -974,8 +974,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= From 6bc8946741acd76a7a4f373cc8d576fe6228d90f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 16:44:31 +0800 Subject: [PATCH 643/659] chore(deps): bump github.com/golang-jwt/jwt/v4 from 4.5.0 to 4.5.2 (#9484) Bumps [github.com/golang-jwt/jwt/v4](https://github.com/golang-jwt/jwt) from 4.5.0 to 4.5.2. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Commits](https://github.com/golang-jwt/jwt/compare/v4.5.0...v4.5.2) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v4 dependency-version: 4.5.2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index bce6deae91e..562296369cf 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/go-resty/resty/v2 v2.14.0 github.com/go-webauthn/webauthn v0.11.1 - github.com/golang-jwt/jwt/v4 v4.5.0 + github.com/golang-jwt/jwt/v4 v4.5.2 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/hekmon/transmissionrpc/v3 v3.0.0 diff --git a/go.sum b/go.sum index f8c574fd154..e3bb088e1ef 100644 --- a/go.sum +++ b/go.sum @@ -307,8 +307,9 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= From fb48ea49fb73eac94b3f1e4555103577e65d1c54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 08:46:35 +0000 Subject: [PATCH 644/659] chore(deps): bump google.golang.org/grpc from 1.66.0 to 1.79.3 Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.66.0 to 1.79.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.66.0...v1.79.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.79.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- go.mod | 26 +++++++++---------- go.sum | 80 ++++++++++++++++++++++++++++++++-------------------------- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index 562296369cf..43098d33c83 100644 --- a/go.mod +++ b/go.mod @@ -72,11 +72,11 @@ require ( github.com/xhofe/wopan-sdk-go v0.1.3 github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 - golang.org/x/crypto v0.41.0 + golang.org/x/crypto v0.46.0 golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e golang.org/x/image v0.19.0 - golang.org/x/net v0.43.0 - golang.org/x/oauth2 v0.30.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 golang.org/x/time v0.8.0 google.golang.org/appengine v1.6.8 gopkg.in/ldap.v3 v3.1.0 @@ -103,7 +103,7 @@ require ( github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect github.com/fatedier/golib v0.5.1 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/google/jsonschema-go v0.4.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect @@ -126,7 +126,7 @@ require ( github.com/vishvananda/netns v0.0.4 // indirect github.com/xtaci/kcp-go/v5 v5.6.13 // indirect github.com/yosida95/uritemplate/v3 v3.0.2 // indirect - golang.org/x/mod v0.27.0 // indirect + golang.org/x/mod v0.30.0 // indirect golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect gopkg.in/ini.v1 v1.67.0 // indirect @@ -303,15 +303,15 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect go.etcd.io/bbolt v1.3.8 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/sync v0.17.0 - golang.org/x/sys v0.35.0 // indirect - golang.org/x/term v0.34.0 // indirect - golang.org/x/text v0.29.0 - golang.org/x/tools v0.36.0 // indirect + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 + golang.org/x/tools v0.39.0 // indirect google.golang.org/api v0.169.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect - google.golang.org/grpc v1.66.0 - google.golang.org/protobuf v1.36.5 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/grpc v1.79.3 + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index e3bb088e1ef..ac30c8cc445 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,8 @@ cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= -cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc= -cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= @@ -273,15 +273,15 @@ github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -764,14 +764,20 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= -go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= -go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= -go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= -go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= -go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= @@ -801,8 +807,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -839,8 +845,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= -golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -875,15 +881,15 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -896,8 +902,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -940,8 +946,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -956,8 +962,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -975,8 +981,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= @@ -1014,8 +1020,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= -golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1024,6 +1030,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1054,8 +1062,8 @@ google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 h1:1GBuWVLM/KMVUv1t1En5Gs+gFZCNd360GGb4sSxtrhU= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1065,8 +1073,8 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= -google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1075,8 +1083,8 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From d509a8787e460d975e728fa2f51a16a0631d386d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 16:47:26 +0800 Subject: [PATCH 645/659] feat(lark): add export tools API (#9511) --- drivers/lark/driver.go | 396 ++++++++++++++++++++++++++++----- drivers/lark/meta.go | 12 +- drivers/lark/other.go | 433 +++++++++++++++++++++++++++++++++++++ drivers/lark/other_test.go | 176 +++++++++++++++ drivers/lark/util.go | 48 ++-- go.mod | 2 +- go.sum | 4 +- server/common/proxy.go | 3 + server/handles/fsread.go | 29 ++- server/handles/lark.go | 84 +++++++ server/router.go | 1 + 11 files changed, 1107 insertions(+), 81 deletions(-) create mode 100644 drivers/lark/other.go create mode 100644 drivers/lark/other_test.go create mode 100644 server/handles/lark.go diff --git a/drivers/lark/driver.go b/drivers/lark/driver.go index fbf7529afe3..ca704a2dbf3 100644 --- a/drivers/lark/driver.go +++ b/drivers/lark/driver.go @@ -8,14 +8,18 @@ import ( "net/http" "strconv" "strings" + "sync" "time" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" lark "github.com/larksuite/oapi-sdk-go/v3" larkcore "github.com/larksuite/oapi-sdk-go/v3/core" larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + larkext "github.com/larksuite/oapi-sdk-go/v3/service/ext" + log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -25,8 +29,12 @@ type Lark struct { client *lark.Client rootFolderToken string + tokenMu sync.Mutex } +const larkListPageSize = 200 +const larkTokenRefreshSkew = 5 * time.Minute + func (c *Lark) Config() driver.Config { return config } @@ -41,34 +49,28 @@ func (c *Lark) Init(ctx context.Context) error { paths := strings.Split(c.RootFolderPath, "/") token := "" - var ok bool - var file *larkdrive.File for _, p := range paths { if p == "" { token = "" continue } - resp, err := c.client.Drive.File.ListByIterator(ctx, larkdrive.NewListFileReqBuilder().FolderToken(token).Build()) + files, err := c.listFiles(ctx, token) if err != nil { return err } - for { - ok, file, err = resp.Next() - if !ok { - return errs.ObjectNotFound - } - - if err != nil { - return err - } - + found := false + for _, file := range files { if *file.Type == "folder" && *file.Name == p { token = *file.Token + found = true break } } + if !found { + return errs.ObjectNotFound + } } c.rootFolderToken = token @@ -90,41 +92,262 @@ func (c *Lark) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] return nil, nil } - resp, err := c.client.Drive.File.ListByIterator(ctx, larkdrive.NewListFileReqBuilder().FolderToken(token).Build()) + files, err := c.listFiles(ctx, token) if err != nil { return nil, err } - ok = false - var file *larkdrive.File var res []model.Obj + for _, file := range files { + res = append(res, larkFileToObj(c.RootFolderPath, dir.GetPath(), file)) + } + + return res, nil +} + +func larkFileToObj(rootFolderPath, dirPath string, file *larkdrive.File) model.Obj { + name := larkString(file.Name) + fileType := larkString(file.Type) + modifiedUnix, _ := strconv.ParseInt(larkString(file.ModifiedTime), 10, 64) + createdUnix, _ := strconv.ParseInt(larkString(file.CreatedTime), 10, 64) + obj := model.Object{ + ID: larkString(file.Token), + Path: strings.Join([]string{rootFolderPath, dirPath, name}, "/"), + Name: larkDisplayName(name, fileType), + Size: 0, + Modified: time.Unix(modifiedUnix, 0), + Ctime: time.Unix(createdUnix, 0), + IsFolder: fileType == "folder", + } + if file.Url == nil || *file.Url == "" || obj.IsFolder || !isLarkNativeDocType(fileType) { + return &obj + } + return &model.ObjectURL{ + Object: obj, + Url: model.Url{Url: *file.Url}, + } +} + +func larkDisplayName(name, fileType string) string { + if isLarkCloudDocName(name) { + return name + } + switch fileType { + case "doc": + return name + ".lark-doc" + case "docx": + return name + ".lark-docx" + case "sheet": + return name + ".lark-sheet" + case "bitable": + return name + ".lark-bitable" + case "mindnote": + return name + ".lark-mindnote" + case "slides": + return name + ".lark-slides" + default: + return name + } +} + +func isLarkNativeDocType(fileType string) bool { + switch fileType { + case "doc", "docx", "sheet", "bitable", "mindnote", "slides": + return true + default: + return false + } +} + +func larkString(s *string) string { + if s == nil { + return "" + } + return *s +} + +func (c *Lark) requestOpts(ctx context.Context) ([]larkcore.RequestOptionFunc, error) { + userAccessToken, err := c.ensureUserAccessToken(ctx, false) + if err != nil { + return nil, err + } + if userAccessToken == "" { + return nil, nil + } + return []larkcore.RequestOptionFunc{larkcore.WithUserAccessToken(userAccessToken)}, nil +} + +func (c *Lark) ensureUserAccessToken(ctx context.Context, forceRefresh bool) (string, error) { + if strings.TrimSpace(c.RefreshToken) == "" { + return strings.TrimSpace(c.UserAccessToken), nil + } + if token := strings.TrimSpace(c.UserAccessToken); !forceRefresh && token != "" && !c.userAccessTokenExpired() { + return token, nil + } + + c.tokenMu.Lock() + defer c.tokenMu.Unlock() + + if token := strings.TrimSpace(c.UserAccessToken); !forceRefresh && token != "" && !c.userAccessTokenExpired() { + return token, nil + } + if c.RefreshTokenExpiresAt > 0 && time.Now().After(time.Unix(c.RefreshTokenExpiresAt, 0)) { + return "", errors.New("lark refresh token expired") + } + + resp, err := c.client.Ext.Authen.RefreshAuthenAccessToken(ctx, + larkext.NewRefreshAuthenAccessTokenReqBuilder(). + Body(larkext.NewRefreshAuthenAccessTokenReqBodyBuilder(). + GrantType(larkext.GrantTypeRefreshCode). + RefreshToken(strings.TrimSpace(c.RefreshToken)). + Build()). + Build()) + if err != nil { + return "", err + } + if !resp.Success() { + return "", errors.New(resp.Error()) + } + if resp.Data == nil || resp.Data.AccessToken == "" { + return "", errors.New("lark refresh token response missing access token") + } + + now := time.Now() + c.UserAccessToken = resp.Data.AccessToken + c.UserAccessTokenExpiresAt = now.Add(time.Duration(resp.Data.ExpiresIn) * time.Second).Unix() + if resp.Data.RefreshToken != "" { + c.RefreshToken = resp.Data.RefreshToken + } + if resp.Data.RefreshExpiresIn > 0 { + c.RefreshTokenExpiresAt = now.Add(time.Duration(resp.Data.RefreshExpiresIn) * time.Second).Unix() + } + op.MustSaveDriverStorage(c) + + return c.UserAccessToken, nil +} + +func (c *Lark) forceRefreshUserAccessToken(ctx context.Context) error { + if strings.TrimSpace(c.RefreshToken) == "" { + return nil + } + _, err := c.ensureUserAccessToken(ctx, true) + return err +} + +func (c *Lark) userAccessTokenExpired() bool { + if c.UserAccessTokenExpiresAt <= 0 { + return true + } + return time.Now().Add(larkTokenRefreshSkew).After(time.Unix(c.UserAccessTokenExpiresAt, 0)) +} + +func (c *Lark) listFiles(ctx context.Context, folderToken string) ([]*larkdrive.File, error) { + var files []*larkdrive.File + pageToken := "" + for { - ok, file, err = resp.Next() - if !ok { - break + builder := larkdrive.NewListFileReqBuilder(). + FolderToken(folderToken). + OrderBy("EditedTime"). + Direction("DESC") + if folderToken != "" { + builder.PageSize(larkListPageSize) + if pageToken != "" { + builder.PageToken(pageToken) + } } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.ListFileResp, error) { + return c.client.Drive.V1.File.List(ctx, builder.Build(), opts...) + }) if err != nil { return nil, err } + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + if resp.Data == nil { + return files, nil + } - modifiedUnix, _ := strconv.ParseInt(*file.ModifiedTime, 10, 64) - createdUnix, _ := strconv.ParseInt(*file.CreatedTime, 10, 64) - - f := model.Object{ - ID: *file.Token, - Path: strings.Join([]string{c.RootFolderPath, dir.GetPath(), *file.Name}, "/"), - Name: *file.Name, - Size: 0, - Modified: time.Unix(modifiedUnix, 0), - Ctime: time.Unix(createdUnix, 0), - IsFolder: *file.Type == "folder", + files = append(files, resp.Data.Files...) + if folderToken == "" || resp.Data.HasMore == nil || !*resp.Data.HasMore || + resp.Data.NextPageToken == nil || *resp.Data.NextPageToken == "" { + break } - res = append(res, &f) + pageToken = *resp.Data.NextPageToken } - return res, nil + return files, nil +} + +type larkResp interface { + Success() bool + Error() string +} + +func doDrive[T larkResp](ctx context.Context, c *Lark, call func(...larkcore.RequestOptionFunc) (T, error)) (T, error) { + opts, err := c.requestOpts(ctx) + if err != nil { + var zero T + return zero, err + } + + resp, err := call(opts...) + if err != nil { + var zero T + return zero, err + } + if !isLarkAuthFailed(resp) || strings.TrimSpace(c.RefreshToken) == "" { + return resp, nil + } + + log.WithField("mount_path", c.MountPath).Warn("lark user access token auth failed, refreshing and retrying once") + if err = c.forceRefreshUserAccessToken(ctx); err != nil { + return resp, nil + } + opts, err = c.requestOpts(ctx) + if err != nil { + var zero T + return zero, err + } + return call(opts...) +} + +func isLarkAuthFailed(resp larkResp) bool { + if resp == nil || resp.Success() { + return false + } + switch v := any(resp).(type) { + case *larkdrive.ListFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.CreateFolderFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.MoveFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.CopyFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.DeleteFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadPrepareFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadPartFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadFinishFileResp: + return isLarkAuthFailedCode(v.Code) + default: + return strings.Contains(resp.Error(), "1061005") || + strings.Contains(strings.ToLower(resp.Error()), "auth") + } +} + +func isLarkAuthFailedCode(code int) bool { + return code == 1061005 || code == 99991663 || code == 99991664 || code == 99991668 +} + +func isHTTPAuthFailed(statusCode int) bool { + return statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden } func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -133,17 +356,23 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* return nil, errs.ObjectNotFound } - resp, err := c.client.GetTenantAccessTokenBySelfBuiltApp(ctx, &larkcore.SelfBuiltTenantAccessTokenReq{ - AppID: c.AppId, - AppSecret: c.AppSecret, - }) + if isLarkCloudDocName(file.GetName()) { + return &model.Link{ + URL: c.filePreviewURL(token), + }, nil + } - if err != nil { - return nil, err + if !c.WebProxy || c.ExternalMode { + return &model.Link{ + URL: c.filePreviewURL(token), + }, nil } - if !c.ExternalMode { - accessToken := resp.TenantAccessToken + if c.WebProxy { + accessToken, err := c.downloadAccessToken(ctx, false) + if err != nil { + return nil, err + } url := fmt.Sprintf("https://open.feishu.cn/open-apis/drive/v1/files/%s/download", token) @@ -159,6 +388,20 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* if err != nil { return nil, err } + _ = ar.Body.Close() + + if isHTTPAuthFailed(ar.StatusCode) && strings.TrimSpace(c.RefreshToken) != "" { + accessToken, err = c.downloadAccessToken(ctx, true) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + ar, err = http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + _ = ar.Body.Close() + } if ar.StatusCode != http.StatusPartialContent { return nil, errors.New("failed to get download link") @@ -170,13 +413,46 @@ func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* "Authorization": []string{fmt.Sprintf("Bearer %s", accessToken)}, }, }, nil - } else { - url := strings.Join([]string{c.TenantUrlPrefix, "file", token}, "/") + } - return &model.Link{ - URL: url, - }, nil + return nil, errors.New("lark download requires web proxy") +} + +func (c *Lark) filePreviewURL(token string) string { + prefix := strings.TrimRight(strings.TrimSpace(c.TenantUrlPrefix), "/") + if prefix == "" { + prefix = "https://www.feishu.cn" + } + return prefix + "/file/" + token +} + +func (c *Lark) downloadAccessToken(ctx context.Context, forceRefresh bool) (string, error) { + var accessToken string + var err error + if strings.TrimSpace(c.RefreshToken) != "" || strings.TrimSpace(c.UserAccessToken) != "" { + accessToken, err = c.ensureUserAccessToken(ctx, forceRefresh) + if err != nil { + return "", err + } } + if accessToken != "" { + return accessToken, nil + } + + resp, err := c.client.GetTenantAccessTokenBySelfBuiltApp(ctx, &larkcore.SelfBuiltTenantAccessTokenReq{ + AppID: c.AppId, + AppSecret: c.AppSecret, + }) + if err != nil { + return "", err + } + if !resp.Success() { + return "", errors.New(resp.Error()) + } + if resp.TenantAccessToken == "" { + return "", errors.New("lark tenant access token is empty") + } + return resp.TenantAccessToken, nil } func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { @@ -190,8 +466,10 @@ func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) return nil, err } - resp, err := c.client.Drive.File.CreateFolder(ctx, - larkdrive.NewCreateFolderFileReqBuilder().Body(body).Build()) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CreateFolderFileResp, error) { + return c.client.Drive.File.CreateFolder(ctx, + larkdrive.NewCreateFolderFileReqBuilder().Body(body).Build(), opts...) + }) if err != nil { return nil, err } @@ -228,7 +506,9 @@ func (c *Lark) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e Build() // 发起请求 - resp, err := c.client.Drive.File.Move(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.MoveFileResp, error) { + return c.client.Drive.File.Move(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -265,7 +545,9 @@ func (c *Lark) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, e Build() // 发起请求 - resp, err := c.client.Drive.File.Copy(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CopyFileResp, error) { + return c.client.Drive.File.Copy(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -289,7 +571,9 @@ func (c *Lark) Remove(ctx context.Context, obj model.Obj) error { Build() // 发起请求 - resp, err := c.client.Drive.File.Delete(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.DeleteFileResp, error) { + return c.client.Drive.File.Delete(ctx, req, opts...) + }) if err != nil { return err } @@ -324,7 +608,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return nil, err } - resp, err := c.client.Drive.File.UploadPrepare(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadPrepareFileResp, error) { + return c.client.Drive.File.UploadPrepare(ctx, req, opts...) + }) if err != nil { return nil, err } @@ -360,7 +646,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return nil, err } - resp, err := c.client.Drive.File.UploadPart(ctx, req) + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadPartFileResp, error) { + return c.client.Drive.File.UploadPart(ctx, req, opts...) + }) if err != nil { return nil, err @@ -382,7 +670,9 @@ func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea Build() // 发起请求 - closeResp, err := c.client.Drive.File.UploadFinish(ctx, closeReq) + closeResp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadFinishFileResp, error) { + return c.client.Drive.File.UploadFinish(ctx, closeReq, opts...) + }) if err != nil { return nil, err } diff --git a/drivers/lark/meta.go b/drivers/lark/meta.go index 221345e222c..fe6149d99a4 100644 --- a/drivers/lark/meta.go +++ b/drivers/lark/meta.go @@ -9,10 +9,14 @@ type Addition struct { // Usually one of two driver.RootPath // define other - AppId string `json:"app_id" type:"text" help:"app id"` - AppSecret string `json:"app_secret" type:"text" help:"app secret"` - ExternalMode bool `json:"external_mode" type:"bool" help:"external mode"` - TenantUrlPrefix string `json:"tenant_url_prefix" type:"text" help:"tenant url prefix"` + AppId string `json:"app_id" type:"text" help:"app id"` + AppSecret string `json:"app_secret" type:"text" help:"app secret"` + UserAccessToken string `json:"user_access_token" type:"text" help:"optional cached user access token for personal drive access"` + RefreshToken string `json:"refresh_token" type:"text" help:"optional refresh token for user access token auto refresh"` + UserAccessTokenExpiresAt int64 `json:"user_access_token_expires_at" type:"number" help:"user access token expires at unix timestamp"` + RefreshTokenExpiresAt int64 `json:"refresh_token_expires_at" type:"number" help:"refresh token expires at unix timestamp"` + ExternalMode bool `json:"external_mode" type:"bool" help:"external mode"` + TenantUrlPrefix string `json:"tenant_url_prefix" type:"text" help:"tenant url prefix"` } var config = driver.Config{ diff --git a/drivers/lark/other.go b/drivers/lark/other.go new file mode 100644 index 00000000000..f0217b72621 --- /dev/null +++ b/drivers/lark/other.go @@ -0,0 +1,433 @@ +package lark + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + larkcore "github.com/larksuite/oapi-sdk-go/v3/core" + larkbitable "github.com/larksuite/oapi-sdk-go/v3/service/bitable/v1" + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + larksheets "github.com/larksuite/oapi-sdk-go/v3/service/sheets/v3" + "github.com/pkg/errors" +) + +const ( + larkExportOptionsMethod = "lark_export_options" + larkExportCreateMethod = "lark_export_create" + larkExportStatusMethod = "lark_export_status" + + larkExportStatusPending = "pending" + larkExportStatusProcessing = "processing" + larkExportStatusSuccess = "success" + larkExportStatusFailed = "failed" +) + +type larkExportCreateReq struct { + Format string `json:"format"` + SubID string `json:"sub_id"` +} + +type larkExportStatusReq struct { + Ticket string `json:"ticket"` +} + +type LarkExportCreateResp struct { + Ticket string `json:"ticket"` + Token string `json:"token"` + Type string `json:"type"` + Format string `json:"format"` + SubID string `json:"sub_id,omitempty"` +} + +type LarkExportOption struct { + Value string `json:"value"` + Label string `json:"label"` + RequiresSubID bool `json:"requires_sub_id"` +} + +type LarkExportSubResource struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` +} + +type LarkExportOptionsResp struct { + Type string `json:"type"` + Formats []LarkExportOption `json:"formats"` + SubResources []LarkExportSubResource `json:"sub_resources,omitempty"` + SubResourceError string `json:"sub_resource_error,omitempty"` +} + +type LarkExportStatusResp struct { + Status string `json:"status"` + FileToken string `json:"file_token,omitempty"` + FileSize int `json:"file_size,omitempty"` + JobStatus int `json:"job_status,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ErrorDetail string `json:"error_detail,omitempty"` +} + +type larkAPIErrorResp interface { + Error() string + ErrorResp() string +} + +func (c *Lark) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch strings.ToLower(strings.TrimSpace(args.Method)) { + case larkExportOptionsMethod: + return c.getExportOptions(ctx, args.Obj) + case larkExportCreateMethod: + var req larkExportCreateReq + if err := decodeOtherData(args.Data, &req); err != nil { + return nil, err + } + return c.createExportTask(ctx, args.Obj, req) + case larkExportStatusMethod: + var req larkExportStatusReq + if err := decodeOtherData(args.Data, &req); err != nil { + return nil, err + } + return c.getExportTask(ctx, args.Obj, req) + default: + return nil, fmt.Errorf("unsupported lark method: %s", args.Method) + } +} + +func decodeOtherData(data interface{}, v interface{}) error { + b, err := json.Marshal(data) + if err != nil { + return errors.WithMessage(err, "failed to encode request data") + } + if err = json.Unmarshal(b, v); err != nil { + return errors.WithMessage(err, "failed to decode request data") + } + return nil +} + +func (c *Lark) createExportTask(ctx context.Context, obj model.Obj, req larkExportCreateReq) (*LarkExportCreateResp, error) { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + docType, err := larkExportType(obj.GetName()) + if err != nil { + return nil, err + } + format := strings.ToLower(strings.TrimSpace(req.Format)) + if !larkExportFormatAllowed(docType, format) { + return nil, fmt.Errorf("unsupported export format %q for lark type %q", format, docType) + } + subID := strings.TrimSpace(req.SubID) + if larkExportFormatRequiresSubID(docType, format) && subID == "" { + return nil, fmt.Errorf("sub_id is required when exporting lark type %q as %q", docType, format) + } + + builder := larkdrive.NewExportTaskBuilder(). + Token(token). + Type(docType). + FileExtension(format). + FileName(larkExportBaseName(obj.GetName())) + if subID != "" { + builder.SubId(subID) + } + exportTask := builder.Build() + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CreateExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Create(ctx, + larkdrive.NewCreateExportTaskReqBuilder().ExportTask(exportTask).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil || resp.Data.Ticket == nil || *resp.Data.Ticket == "" { + return nil, errors.New("lark export task response missing ticket") + } + return &LarkExportCreateResp{ + Ticket: *resp.Data.Ticket, + Token: token, + Type: docType, + Format: format, + SubID: subID, + }, nil +} + +func (c *Lark) getExportOptions(ctx context.Context, obj model.Obj) (*LarkExportOptionsResp, error) { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + docType, err := larkExportType(obj.GetName()) + if err != nil { + return nil, err + } + out := &LarkExportOptionsResp{ + Type: docType, + Formats: larkExportOptions(docType), + } + switch docType { + case larkdrive.TypeSheet: + out.SubResources, err = c.listSheetSubResources(ctx, token) + case larkdrive.TypeBitable: + out.SubResources, err = c.listBitableSubResources(ctx, token) + } + if err != nil { + out.SubResourceError = err.Error() + } + return out, nil +} + +func (c *Lark) getExportTask(ctx context.Context, obj model.Obj, req larkExportStatusReq) (*LarkExportStatusResp, error) { + ticket := strings.TrimSpace(req.Ticket) + if ticket == "" { + return nil, errors.New("ticket is required") + } + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.GetExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Get(ctx, + larkdrive.NewGetExportTaskReqBuilder().Ticket(ticket).Token(token).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil || resp.Data.Result == nil { + return nil, errors.New("lark export task response missing result") + } + result := resp.Data.Result + out := &LarkExportStatusResp{ + Status: larkExportJobStatus(result), + } + if result.FileToken != nil { + out.FileToken = *result.FileToken + } + if result.FileSize != nil { + out.FileSize = *result.FileSize + } + if result.JobStatus != nil { + out.JobStatus = *result.JobStatus + } + if out.Status == larkExportStatusFailed { + out.ErrorMessage = larkExportTaskErrorMessage(result) + out.ErrorDetail = larkExportTaskErrorDetail(result) + } else if result.JobErrorMsg != nil { + out.ErrorMessage = strings.TrimSpace(*result.JobErrorMsg) + } + return out, nil +} + +func (c *Lark) DownloadExportFile(ctx context.Context, fileToken string) (io.Reader, string, error) { + fileToken = strings.TrimSpace(fileToken) + if fileToken == "" { + return nil, "", errors.New("file_token is required") + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.DownloadExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Download(ctx, + larkdrive.NewDownloadExportTaskReqBuilder().FileToken(fileToken).Build(), opts...) + }) + if err != nil { + return nil, "", err + } + if !resp.Success() { + return nil, "", larkAPIError(resp) + } + if resp.File == nil { + return nil, "", errors.New("lark export download response missing file") + } + return resp.File, resp.FileName, nil +} + +func larkExportType(name string) (string, error) { + switch { + case strings.HasSuffix(name, ".lark-doc"): + return larkdrive.TypeDoc, nil + case strings.HasSuffix(name, ".lark-docx"): + return larkdrive.TypeDocx, nil + case strings.HasSuffix(name, ".lark-sheet"): + return larkdrive.TypeSheet, nil + case strings.HasSuffix(name, ".lark-bitable"): + return larkdrive.TypeBitable, nil + default: + return "", fmt.Errorf("unsupported lark export file type: %s", name) + } +} + +func larkExportFormatAllowed(docType, format string) bool { + switch docType { + case larkdrive.TypeDoc, larkdrive.TypeDocx: + return format == larkdrive.FileExtensionPdf || format == larkdrive.FileExtensionDocx + case larkdrive.TypeSheet, larkdrive.TypeBitable: + return format == larkdrive.FileExtensionXlsx || format == larkdrive.FileExtensionCsv + default: + return false + } +} + +func larkExportFormatRequiresSubID(docType, format string) bool { + return (docType == larkdrive.TypeSheet || docType == larkdrive.TypeBitable) && format == larkdrive.FileExtensionCsv +} + +func larkExportOptions(docType string) []LarkExportOption { + switch docType { + case larkdrive.TypeDoc, larkdrive.TypeDocx: + return []LarkExportOption{ + {Value: larkdrive.FileExtensionPdf, Label: "PDF"}, + {Value: larkdrive.FileExtensionDocx, Label: "DOCX"}, + } + case larkdrive.TypeSheet, larkdrive.TypeBitable: + return []LarkExportOption{ + {Value: larkdrive.FileExtensionXlsx, Label: "XLSX"}, + {Value: larkdrive.FileExtensionCsv, Label: "CSV", RequiresSubID: true}, + } + default: + return nil + } +} + +func larkExportBaseName(name string) string { + return trimLarkDisplayExt(name) +} + +func larkExportJobStatus(result *larkdrive.ExportTask) string { + if result == nil { + return larkExportStatusProcessing + } + if result.FileToken != nil && *result.FileToken != "" { + return larkExportStatusSuccess + } + if result.JobStatus != nil && *result.JobStatus == 2 { + return larkExportStatusFailed + } + if result.JobErrorMsg != nil && *result.JobErrorMsg != "" && !strings.EqualFold(*result.JobErrorMsg, "success") { + return larkExportStatusFailed + } + return larkExportStatusProcessing +} + +func larkAPIError(resp larkAPIErrorResp) error { + msg := strings.TrimSpace(resp.ErrorResp()) + if msg == "" || msg == "{}" || msg == "null" { + msg = strings.TrimSpace(resp.Error()) + } + return errors.New(msg) +} + +func larkExportTaskErrorMessage(result *larkdrive.ExportTask) string { + if result == nil { + return "" + } + if result.JobErrorMsg != nil { + msg := strings.TrimSpace(*result.JobErrorMsg) + if msg != "" && !strings.EqualFold(msg, "success") { + return msg + } + } + if result.JobStatus != nil { + return fmt.Sprintf("job_status=%d", *result.JobStatus) + } + return "" +} + +func larkExportTaskErrorDetail(result *larkdrive.ExportTask) string { + if result == nil { + return "" + } + detail := map[string]interface{}{} + if result.JobStatus != nil { + detail["job_status"] = *result.JobStatus + } + if result.JobErrorMsg != nil { + detail["job_error_msg"] = strings.TrimSpace(*result.JobErrorMsg) + } + if len(detail) == 0 { + return "" + } + b, err := json.Marshal(detail) + if err != nil { + return "" + } + return string(b) +} + +func (c *Lark) listSheetSubResources(ctx context.Context, token string) ([]LarkExportSubResource, error) { + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larksheets.QuerySpreadsheetSheetResp, error) { + return c.client.Sheets.SpreadsheetSheet.Query(ctx, + larksheets.NewQuerySpreadsheetSheetReqBuilder().SpreadsheetToken(token).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil { + return nil, nil + } + var res []LarkExportSubResource + for _, sheet := range resp.Data.Sheets { + if sheet == nil || sheet.SheetId == nil || strings.TrimSpace(*sheet.SheetId) == "" { + continue + } + name := strings.TrimSpace(larkString(sheet.Title)) + if name == "" { + name = *sheet.SheetId + } + res = append(res, LarkExportSubResource{ + ID: *sheet.SheetId, + Name: name, + Type: strings.TrimSpace(larkString(sheet.ResourceType)), + }) + } + return res, nil +} + +func (c *Lark) listBitableSubResources(ctx context.Context, token string) ([]LarkExportSubResource, error) { + var res []LarkExportSubResource + pageToken := "" + for { + builder := larkbitable.NewListAppTableReqBuilder().AppToken(token).PageSize(100) + if pageToken != "" { + builder.PageToken(pageToken) + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkbitable.ListAppTableResp, error) { + return c.client.Bitable.AppTable.List(ctx, builder.Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil { + return res, nil + } + for _, table := range resp.Data.Items { + if table == nil || table.TableId == nil || strings.TrimSpace(*table.TableId) == "" { + continue + } + name := strings.TrimSpace(larkString(table.Name)) + if name == "" { + name = *table.TableId + } + res = append(res, LarkExportSubResource{ + ID: *table.TableId, + Name: name, + Type: "table", + }) + } + if resp.Data.HasMore == nil || !*resp.Data.HasMore || resp.Data.PageToken == nil || *resp.Data.PageToken == "" { + return res, nil + } + pageToken = *resp.Data.PageToken + } +} diff --git a/drivers/lark/other_test.go b/drivers/lark/other_test.go new file mode 100644 index 00000000000..6887c6bcd65 --- /dev/null +++ b/drivers/lark/other_test.go @@ -0,0 +1,176 @@ +package lark + +import ( + "testing" + + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" +) + +func TestLarkExportType(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + }{ + {name: "doc.lark-doc", want: larkdrive.TypeDoc}, + {name: "doc.lark-docx", want: larkdrive.TypeDocx}, + {name: "sheet.lark-sheet", want: larkdrive.TypeSheet}, + {name: "base.lark-bitable", want: larkdrive.TypeBitable}, + {name: "file.pdf", wantErr: true}, + } + for _, tt := range tests { + got, err := larkExportType(tt.name) + if tt.wantErr { + if err == nil { + t.Fatalf("larkExportType(%q) expected error", tt.name) + } + continue + } + if err != nil { + t.Fatalf("larkExportType(%q) unexpected error: %v", tt.name, err) + } + if got != tt.want { + t.Fatalf("larkExportType(%q) = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportFormatAllowed(t *testing.T) { + tests := []struct { + docType string + format string + want bool + }{ + {docType: larkdrive.TypeDoc, format: larkdrive.FileExtensionPdf, want: true}, + {docType: larkdrive.TypeDocx, format: larkdrive.FileExtensionDocx, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionXlsx, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionXlsx, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionPdf, want: false}, + } + for _, tt := range tests { + if got := larkExportFormatAllowed(tt.docType, tt.format); got != tt.want { + t.Fatalf("larkExportFormatAllowed(%q, %q) = %v, want %v", tt.docType, tt.format, got, tt.want) + } + } +} + +func TestLarkExportFormatRequiresSubID(t *testing.T) { + tests := []struct { + docType string + format string + want bool + }{ + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionXlsx, want: false}, + {docType: larkdrive.TypeDocx, format: larkdrive.FileExtensionDocx, want: false}, + } + for _, tt := range tests { + if got := larkExportFormatRequiresSubID(tt.docType, tt.format); got != tt.want { + t.Fatalf("larkExportFormatRequiresSubID(%q, %q) = %v, want %v", tt.docType, tt.format, got, tt.want) + } + } +} + +func TestLarkExportOptions(t *testing.T) { + tests := []struct { + docType string + want []LarkExportOption + }{ + {docType: larkdrive.TypeDocx, want: []LarkExportOption{ + {Value: larkdrive.FileExtensionPdf, Label: "PDF"}, + {Value: larkdrive.FileExtensionDocx, Label: "DOCX"}, + }}, + {docType: larkdrive.TypeSheet, want: []LarkExportOption{ + {Value: larkdrive.FileExtensionXlsx, Label: "XLSX"}, + {Value: larkdrive.FileExtensionCsv, Label: "CSV", RequiresSubID: true}, + }}, + } + for _, tt := range tests { + got := larkExportOptions(tt.docType) + if len(got) != len(tt.want) { + t.Fatalf("larkExportOptions(%q) len = %d, want %d", tt.docType, len(got), len(tt.want)) + } + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("larkExportOptions(%q)[%d] = %+v, want %+v", tt.docType, i, got[i], tt.want[i]) + } + } + } +} + +func TestLarkExportBaseName(t *testing.T) { + tests := map[string]string{ + "weekly.lark-docx": "weekly", + "weekly.report.lark-doc": "weekly.report", + "table.lark-sheet": "table", + "plain.pdf": "plain.pdf", + } + for name, want := range tests { + if got := larkExportBaseName(name); got != want { + t.Fatalf("larkExportBaseName(%q) = %q, want %q", name, got, want) + } + } +} + +func TestLarkExportJobStatus(t *testing.T) { + successToken := "box_token" + successCode := 0 + failCode := 2 + failMsg := "failed" + successMsg := "success" + + tests := []struct { + name string + result *larkdrive.ExportTask + want string + }{ + {name: "nil", result: nil, want: larkExportStatusProcessing}, + {name: "file token wins", result: &larkdrive.ExportTask{FileToken: &successToken, JobStatus: &failCode}, want: larkExportStatusSuccess}, + {name: "status failure", result: &larkdrive.ExportTask{JobStatus: &failCode}, want: larkExportStatusFailed}, + {name: "error message failure", result: &larkdrive.ExportTask{JobStatus: &successCode, JobErrorMsg: &failMsg}, want: larkExportStatusFailed}, + {name: "success message without token still processing", result: &larkdrive.ExportTask{JobStatus: &successCode, JobErrorMsg: &successMsg}, want: larkExportStatusProcessing}, + } + for _, tt := range tests { + if got := larkExportJobStatus(tt.result); got != tt.want { + t.Fatalf("%s: larkExportJobStatus() = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportTaskErrorMessage(t *testing.T) { + failCode := 2 + failMsg := "source document cannot be exported" + successMsg := "success" + + tests := []struct { + name string + result *larkdrive.ExportTask + want string + }{ + {name: "nil", result: nil, want: ""}, + {name: "job error message", result: &larkdrive.ExportTask{JobStatus: &failCode, JobErrorMsg: &failMsg}, want: failMsg}, + {name: "status fallback", result: &larkdrive.ExportTask{JobStatus: &failCode, JobErrorMsg: &successMsg}, want: "job_status=2"}, + } + for _, tt := range tests { + if got := larkExportTaskErrorMessage(tt.result); got != tt.want { + t.Fatalf("%s: larkExportTaskErrorMessage() = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportTaskErrorDetail(t *testing.T) { + failCode := 2 + failMsg := "source document cannot be exported" + + got := larkExportTaskErrorDetail(&larkdrive.ExportTask{ + JobStatus: &failCode, + JobErrorMsg: &failMsg, + }) + want := `{"job_error_msg":"source document cannot be exported","job_status":2}` + if got != want { + t.Fatalf("larkExportTaskErrorDetail() = %q, want %q", got, want) + } +} diff --git a/drivers/lark/util.go b/drivers/lark/util.go index 8c6828bd176..ebef4300c49 100644 --- a/drivers/lark/util.go +++ b/drivers/lark/util.go @@ -3,9 +3,9 @@ package lark import ( "context" "github.com/Xhofe/go-cache" - larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" log "github.com/sirupsen/logrus" "path" + "strings" "time" ) @@ -36,31 +36,47 @@ func (c *Lark) getObjToken(ctx context.Context, folderPath string) (string, bool return emptyFolderToken, false } - req := larkdrive.NewListFileReqBuilder().FolderToken(parentToken).Build() - resp, err := c.client.Drive.File.ListByIterator(ctx, req) - + files, err := c.listFiles(ctx, parentToken) if err != nil { log.WithError(err).Error("failed to list files") return emptyFolderToken, false } - var file *larkdrive.File - for { - found, file, err = resp.Next() - if !found { - break + for _, file := range files { + if *file.Name == name || *file.Name == trimLarkDisplayExt(name) { + objTokenCache.Set(folderPath, *file.Token, exOpts) + return *file.Token, true } + } + + return emptyFolderToken, false +} - if err != nil { - log.WithError(err).Error("failed to get next file") - break +func trimLarkDisplayExt(name string) string { + for _, suffix := range larkCloudDocSuffixes() { + if strings.HasSuffix(name, suffix) { + return strings.TrimSuffix(name, suffix) } + } + return name +} - if *file.Name == name { - objTokenCache.Set(folderPath, *file.Token, exOpts) - return *file.Token, true +func isLarkCloudDocName(name string) bool { + for _, suffix := range larkCloudDocSuffixes() { + if strings.HasSuffix(name, suffix) { + return true } } + return false +} - return emptyFolderToken, false +func larkCloudDocSuffixes() []string { + return []string{ + ".lark-doc", + ".lark-docx", + ".lark-sheet", + ".lark-bitable", + ".lark-mindnote", + ".lark-slides", + } } diff --git a/go.mod b/go.mod index 562296369cf..a550404e31d 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 github.com/kdomanski/iso9660 v0.4.0 - github.com/larksuite/oapi-sdk-go/v3 v3.3.1 + github.com/larksuite/oapi-sdk-go/v3 v3.6.1 github.com/mark3labs/mcp-go v0.48.0 github.com/maruel/natural v1.1.1 github.com/meilisearch/meilisearch-go v0.27.2 diff --git a/go.sum b/go.sum index e3bb088e1ef..2dd61e11a2e 100644 --- a/go.sum +++ b/go.sum @@ -474,8 +474,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/larksuite/oapi-sdk-go/v3 v3.3.1 h1:DLQQEgHUAGZB6RVlceB1f6A94O206exxW2RIMH+gMUc= -github.com/larksuite/oapi-sdk-go/v3 v3.3.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1 h1:vAdu+sX9yXNkKnKnYQeIv6yBkjP37Q1JEJHmMa2eCjQ= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= diff --git a/server/common/proxy.go b/server/common/proxy.go index 97bf84efa12..dae97c9bf1c 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -80,6 +80,9 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. defer res.Body.Close() maps.Copy(w.Header(), res.Header) + if r.URL.Query().Get("type") == "preview" { + w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.PathEscape(file.GetName()))) + } w.WriteHeader(res.StatusCode) if r.Method == http.MethodHead { return nil diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 67f84e5d70a..9ca84edda52 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -333,6 +333,7 @@ type FsGetResp struct { Readme string `json:"readme"` Header string `json:"header"` Provider string `json:"provider"` + WebProxy bool `json:"web_proxy"` Related []ObjLabelResp `json:"related"` } @@ -367,14 +368,14 @@ func FsGet(c *gin.Context) { } var rawURL string - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) provider := "unknown" - if err == nil { + if storageErr == nil { provider = storage.Config().Name } if !obj.IsDir() { - if err != nil { - common.ErrorResp(c, err, 500) + if storageErr != nil { + common.ErrorResp(c, storageErr, 500) return } query := "" @@ -382,6 +383,7 @@ func FsGet(c *gin.Context) { query = "?sign=" + sign.Sign(reqPath) } forceRedirectRawURL := storage.GetStorage().Driver == "BaiduYouth" + forcePreviewRawURL := storage.GetStorage().Driver == "Lark" && isLarkCloudDocName(obj.GetName()) forceProxyRawURL := storage.GetStorage().Driver == "Quark" && utils.GetFileType(obj.GetName()) == conf.VIDEO if forceRedirectRawURL { // Baidu Youth direct links are minted per request and are not stable enough @@ -391,7 +393,7 @@ func FsGet(c *gin.Context) { common.GetApiUrl(c.Request), utils.EncodePath(reqPath, true), query) - } else if storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL { + } else if !forcePreviewRawURL && (storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL) { if storage.GetStorage().DownProxyUrl != "" { rawURL = common.BuildDownProxyURL( storage.GetStorage().DownProxyUrl, @@ -453,6 +455,7 @@ func FsGet(c *gin.Context) { Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, + WebProxy: storageErr == nil && storage.GetStorage().WebProxy, Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), }) } @@ -471,6 +474,22 @@ func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { return related } +func isLarkCloudDocName(name string) bool { + for _, suffix := range []string{ + ".lark-doc", + ".lark-docx", + ".lark-sheet", + ".lark-bitable", + ".lark-mindnote", + ".lark-slides", + } { + if strings.HasSuffix(name, suffix) { + return true + } + } + return false +} + type FsOtherReq struct { model.FsOtherArgs Password string `json:"password" form:"password"` diff --git a/server/handles/lark.go b/server/handles/lark.go new file mode 100644 index 00000000000..b8f6f7f9d60 --- /dev/null +++ b/server/handles/lark.go @@ -0,0 +1,84 @@ +package handles + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type larkExportDownloader interface { + DownloadExportFile(ctx context.Context, fileToken string) (io.Reader, string, error) +} + +func LarkExportDownload(c *gin.Context) { + rawPath := c.Query("path") + fileToken := c.Query("file_token") + password := c.Query("password") + filename := strings.TrimSpace(c.Query("filename")) + if rawPath == "" || fileToken == "" { + common.ErrorStrResp(c, "path and file_token are required", 400) + return + } + + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(rawPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500) + return + } + } + if !common.CanAccessWithRoles(user, meta, reqPath, password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + downloader, ok := storage.(larkExportDownloader) + if !ok || storage.GetStorage().Driver != "Lark" { + common.ErrorStrResp(c, "lark export download is not supported for this storage", 400) + return + } + + reader, respFilename, err := downloader.DownloadExportFile(c, fileToken) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if filename == "" { + filename = respFilename + } + if filename == "" { + filename = stdpath.Base(reqPath) + } + + c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, filename, url.PathEscape(filename))) + c.Header("Content-Type", utils.GetMimeType(filename)) + c.Status(http.StatusOK) + if _, err = io.Copy(c.Writer, reader); err != nil { + common.ErrorResp(c, err, 500) + return + } +} diff --git a/server/router.go b/server/router.go index 84c8a831e43..e930177910a 100644 --- a/server/router.go +++ b/server/router.go @@ -222,6 +222,7 @@ func _fs(g *gin.RouterGroup) { g.Any("/search", middlewares.SearchIndex, handles.Search) g.Any("/get", handles.FsGet) g.Any("/other", handles.FsOther) + g.GET("/lark/export/download", handles.LarkExportDownload) g.Any("/dirs", handles.FsDirs) g.POST("/mkdir", handles.FsMkdir) g.POST("/rename", handles.FsRename) From 1c0fc0b213fedff8d0fd5f3d4f8b3c754a65c714 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 May 2026 08:47:35 +0000 Subject: [PATCH 646/659] chore(deps): bump github.com/golang-jwt/jwt/v5 from 5.2.1 to 5.2.2 Bumps [github.com/golang-jwt/jwt/v5](https://github.com/golang-jwt/jwt) from 5.2.1 to 5.2.2. - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Commits](https://github.com/golang-jwt/jwt/compare/v5.2.1...v5.2.2) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v5 dependency-version: 5.2.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 562296369cf..6a3d69d5836 100644 --- a/go.mod +++ b/go.mod @@ -221,7 +221,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/go-webauthn/x v0.1.12 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/go.sum b/go.sum index e3bb088e1ef..a5b6a8f53f3 100644 --- a/go.sum +++ b/go.sum @@ -310,8 +310,8 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= From 0fa863ed27da714832675567d7e6c444dde3fc92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 16:47:56 +0800 Subject: [PATCH 647/659] fix: support all pagination mode (#9512) --- server/handles/fsread.go | 17 +++++++++++++++-- server/handles/share_public.go | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 9ca84edda52..1d85dcae935 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -82,6 +82,7 @@ type ObjLabelResp struct { const ( DefaultPerPage = 200 MaxPerPage = 500 + AllPerPage = -1 ) func FsList(c *gin.Context) { @@ -136,7 +137,7 @@ func FsList(c *gin.Context) { total, pageObjs := pagination(filtered, &req.PageReq) respContent := toObjsResp(pageObjs, reqPath, isEncrypt(meta, reqPath)) pagesTotal := calcPagesTotal(total, req.PerPage) - hasMore := req.Page*req.PerPage < total + hasMore := req.PerPage != AllPerPage && req.Page*req.PerPage < total common.SuccessResp(c, FsListResp{ Content: respContent, @@ -253,7 +254,10 @@ func normalizeListPage(page, perPage int) (int, int) { effPage = 1 } effPerPage := perPage - if effPerPage <= 0 { + if effPerPage < 0 { + return effPage, AllPerPage + } + if effPerPage == 0 { effPerPage = DefaultPerPage } if effPerPage > MaxPerPage { @@ -263,6 +267,12 @@ func normalizeListPage(page, perPage int) (int, int) { } func calcPagesTotal(total, perPage int) int { + if perPage == AllPerPage { + if total > 0 { + return 1 + } + return 0 + } if total <= 0 || perPage <= 0 { return 0 } @@ -272,6 +282,9 @@ func calcPagesTotal(total, perPage int) int { func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { pageIndex, pageSize := req.Page, req.PerPage total := len(objs) + if pageSize == AllPerPage { + return total, objs + } start := (pageIndex - 1) * pageSize if start > total { return total, []model.Obj{} diff --git a/server/handles/share_public.go b/server/handles/share_public.go index 83ca79f3591..f88d5b338c5 100644 --- a/server/handles/share_public.go +++ b/server/handles/share_public.go @@ -147,7 +147,7 @@ func ListPublicShare(c *gin.Context) { Total: int64(total), Page: req.Page, PerPage: req.PerPage, - HasMore: req.Page*req.PerPage < total, + HasMore: req.PerPage != AllPerPage && req.Page*req.PerPage < total, PagesTotal: calcPagesTotal(total, req.PerPage), }) } From fc26c3d8b2825093e3dafd286db1e77d07d04e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 16:53:35 +0800 Subject: [PATCH 648/659] feat: add GuangYaPan offline download (#9505) --- drivers/guangyapan/offline.go | 165 ++++++++++++++++++ drivers/guangyapan/types.go | 73 ++++++++ internal/conf/const.go | 27 +-- internal/offline_download/all.go | 1 + .../offline_download/guangyapan/guangyapan.go | 131 ++++++++++++++ internal/offline_download/guangyapan/util.go | 77 ++++++++ internal/offline_download/tool/add.go | 11 ++ internal/offline_download/tool/download.go | 5 +- server/handles/offline_download.go | 45 +++++ server/router.go | 1 + 10 files changed, 523 insertions(+), 13 deletions(-) create mode 100644 drivers/guangyapan/offline.go create mode 100644 internal/offline_download/guangyapan/guangyapan.go create mode 100644 internal/offline_download/guangyapan/util.go diff --git a/drivers/guangyapan/offline.go b/drivers/guangyapan/offline.go new file mode 100644 index 00000000000..d523aaa677c --- /dev/null +++ b/drivers/guangyapan/offline.go @@ -0,0 +1,165 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" +) + +func (d *GuangYaPan) ResolveOfflineResource(ctx context.Context, fileURL string) (*OfflineResolveData, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + fileURL = strings.TrimSpace(fileURL) + if fileURL == "" { + return nil, errors.New("offline url is empty") + } + + var resp offlineResolveResp + if err := d.postAPI(ctx, "/cloudcollection/v1/resolve_res", map[string]any{ + "url": fileURL, + }, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("resolve offline resource failed: %s", strings.TrimSpace(resp.Msg)) + } + return &resp.Data, nil +} + +func (d *GuangYaPan) OfflineDownload(ctx context.Context, fileURL string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + resolved, err := d.ResolveOfflineResource(ctx, fileURL) + if err != nil { + return nil, err + } + + parentID := parentDir.GetID() + if parentID == d.RootFolderID { + parentID = "" + } + + taskURL := strings.TrimSpace(resolved.URL) + if taskURL == "" { + taskURL = strings.TrimSpace(fileURL) + } + name := strings.TrimSpace(fileName) + if name == "" { + name = resolved.defaultName(taskURL) + } + + body := map[string]any{ + "url": taskURL, + "parentId": parentID, + "newName": name, + } + if indexes := resolved.fileIndexes(); len(indexes) > 0 { + body["fileIndexes"] = indexes + } + + var resp offlineCreateResp + if err := d.postAPI(ctx, "/cloudcollection/v1/create_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("create offline task failed: %s", strings.TrimSpace(resp.Msg)) + } + taskID := strings.TrimSpace(resp.Data.TaskID) + if taskID == "" { + return nil, errors.New("create offline task failed: empty task id") + } + return &OfflineTask{ + TaskID: taskID, + FileName: name, + Res: taskURL, + }, nil +} + +func (d *GuangYaPan) OfflineList(ctx context.Context, taskIDs []string, statuses []int, cursor string, pageSize int) ([]OfflineTask, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + body := map[string]any{} + if len(taskIDs) > 0 { + body["taskIds"] = taskIDs + } + if len(statuses) > 0 { + body["status"] = statuses + } + if cursor = strings.TrimSpace(cursor); cursor != "" { + body["cursor"] = cursor + } + if pageSize > 0 { + body["pageSize"] = pageSize + } + + var resp offlineListResp + if err := d.postAPI(ctx, "/cloudcollection/v1/list_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("list offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return resp.Data.List, nil +} + +func (d *GuangYaPan) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if len(taskIDs) == 0 { + return nil + } + + var resp offlineDeleteResp + if err := d.postAPI(ctx, "/cloudcollection/v2/delete_task", map[string]any{ + "taskIds": taskIDs, + }, &resp); err != nil { + return err + } + if !isSuccessMsg(resp.Msg) { + return fmt.Errorf("delete offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return nil +} + +func (d OfflineResolveData) defaultName(fileURL string) string { + if d.BTResInfo != nil && strings.TrimSpace(d.BTResInfo.FileName) != "" { + return strings.TrimSpace(d.BTResInfo.FileName) + } + u, err := url.Parse(fileURL) + if err == nil { + name := strings.TrimSpace(stdpath.Base(u.Path)) + if name != "" && name != "." && name != "/" { + if decoded, err := url.PathUnescape(name); err == nil { + name = decoded + } + return name + } + } + return "offline_download" +} + +func (d OfflineResolveData) fileIndexes() []int { + if d.BTResInfo == nil || len(d.BTResInfo.Subfiles) == 0 { + return nil + } + indexes := make([]int, 0, len(d.BTResInfo.Subfiles)) + for i, file := range d.BTResInfo.Subfiles { + if file.FileIndex != nil { + indexes = append(indexes, *file.FileIndex) + continue + } + indexes = append(indexes, i) + } + return indexes +} + +func isSuccessMsg(msg string) bool { + msg = strings.TrimSpace(msg) + return msg == "" || strings.EqualFold(msg, "success") +} diff --git a/drivers/guangyapan/types.go b/drivers/guangyapan/types.go index bd0094f3070..1a1c129c06a 100644 --- a/drivers/guangyapan/types.go +++ b/drivers/guangyapan/types.go @@ -133,6 +133,79 @@ type taskInfoResp struct { } `json:"data"` } +type offlineResolveResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data OfflineResolveData `json:"data"` +} + +type OfflineResolveData struct { + ResType int `json:"resType"` + BTResInfo *OfflineBTResInfo `json:"btResInfo"` + URL string `json:"url"` +} + +type OfflineBTResInfo struct { + InfoHash string `json:"infoHash"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + SubfilesNum int `json:"subfilesNum"` + Subfiles []OfflineSubfile `json:"subfiles"` + CreateTime int64 `json:"createTime"` + ExcludeIndices []int `json:"excludeIndices"` +} + +type OfflineSubfile struct { + FileName string `json:"fileName"` + FileIndex *int `json:"fileIndex"` + FileSize int64 `json:"fileSize"` +} + +type offlineCreateResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + URL string `json:"url"` + } `json:"data"` +} + +type offlineDeleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskIDs []string `json:"taskIds"` + } `json:"data"` +} + +type offlineListResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + StatusCounts []struct { + Status int `json:"status"` + Count int `json:"count"` + } `json:"statusCounts"` + Cursor string `json:"cursor"` + List []OfflineTask `json:"list"` + Total int `json:"total"` + } `json:"data"` +} + +type OfflineTask struct { + TaskID string `json:"taskId"` + FileName string `json:"fileName"` + TotalSize int64 `json:"totalSize"` + Status int `json:"status"` + CreateTime int64 `json:"createTime"` + Res string `json:"res"` + ResType int `json:"resType"` + Progress int `json:"progress"` + FileID string `json:"fileId"` + IsDir bool `json:"isDir"` + Exist bool `json:"exist"` +} + func unixOrZero(v int64) time.Time { if v <= 0 { return time.Time{} diff --git a/internal/conf/const.go b/internal/conf/const.go index 5bcb6eb5f91..4a3bbfa0b8f 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -76,6 +76,9 @@ const ( // thunder ThunderTempDir = "thunder_temp_dir" + // guangyapan + GuangYaPanTempDir = "guangyapan_temp_dir" + // single Token = "token" IndexProgress = "index_progress" @@ -126,19 +129,19 @@ const ( FTPTLSPublicCertPath = "ftp_tls_public_cert_path" // frp - FRPEnabled = "frp_enabled" - FRPServerAddr = "frp_server_addr" - FRPServerPort = "frp_server_port" - FRPAuthToken = "frp_auth_token" - FRPProxyName = "frp_proxy_name" - FRPProxyType = "frp_proxy_type" - FRPCustomDomain = "frp_custom_domain" - FRPSubdomain = "frp_subdomain" - FRPRemotePort = "frp_remote_port" - FRPLocalPort = "frp_local_port" - FRPTLSEnable = "frp_tls_enable" + FRPEnabled = "frp_enabled" + FRPServerAddr = "frp_server_addr" + FRPServerPort = "frp_server_port" + FRPAuthToken = "frp_auth_token" + FRPProxyName = "frp_proxy_name" + FRPProxyType = "frp_proxy_type" + FRPCustomDomain = "frp_custom_domain" + FRPSubdomain = "frp_subdomain" + FRPRemotePort = "frp_remote_port" + FRPLocalPort = "frp_local_port" + FRPTLSEnable = "frp_tls_enable" FRPSTCPSecretKey = "frp_stcp_secret_key" - FRPStatus = "frp_status" + FRPStatus = "frp_status" // traffic TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 3d0c7c73a0b..1ba191e47e8 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -3,6 +3,7 @@ package offline_download import ( _ "github.com/alist-org/alist/v3/internal/offline_download/115" _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" + _ "github.com/alist-org/alist/v3/internal/offline_download/guangyapan" _ "github.com/alist-org/alist/v3/internal/offline_download/http" _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" diff --git a/internal/offline_download/guangyapan/guangyapan.go b/internal/offline_download/guangyapan/guangyapan.go new file mode 100644 index 00000000000..eb58af0bd4a --- /dev/null +++ b/internal/offline_download/guangyapan/guangyapan.go @@ -0,0 +1,131 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" +) + +type GuangYaPan struct { + refreshTaskCache bool +} + +func (g *GuangYaPan) Name() string { + return "GuangYaPan" +} + +func (g *GuangYaPan) Items() []model.SettingItem { + return nil +} + +func (g *GuangYaPan) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (g *GuangYaPan) Init() (string, error) { + g.refreshTaskCache = false + return "ok", nil +} + +func (g *GuangYaPan) IsReady() bool { + tempDir := setting.GetStr(conf.GuangYaPanTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + return false + } + return true +} + +func (g *GuangYaPan) AddURL(args *tool.AddUrlArgs) (string, error) { + g.refreshTaskCache = true + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return "", errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + task, err := driver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return task.TaskID, nil +} + +func (g *GuangYaPan) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + ctx := context.Background() + if err := driver.DeleteOfflineTasks(ctx, []string{task.GID}, false); err != nil { + return err + } + g.DelTaskCache(driver, task.GID) + return nil +} + +func (g *GuangYaPan) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return nil, errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + tasks, err := g.GetTasks(driver, task.GID) + if err != nil { + return nil, err + } + status := &tool.Status{ + Status: "the task has been deleted", + } + for _, t := range tasks { + if t.TaskID != task.GID { + continue + } + status.Progress = float64(t.Progress) + status.TotalBytes = t.TotalSize + status.Completed = t.Status == offlineStatusCompleted || t.Status == offlineStatusPartiallyCompleted + status.Status = taskStatusText(t) + if t.Status == offlineStatusFailed || t.Status == offlineStatusCanceled { + status.Err = errors.New(status.Status) + } + return status, nil + } + status.Err = errors.New("the task has been deleted") + return status, nil +} + +func init() { + tool.Tools.Add(&GuangYaPan{}) +} diff --git a/internal/offline_download/guangyapan/util.go b/internal/offline_download/guangyapan/util.go new file mode 100644 index 00000000000..cb7bb926ead --- /dev/null +++ b/internal/offline_download/guangyapan/util.go @@ -0,0 +1,77 @@ +package guangyapan + +import ( + "context" + "fmt" + "time" + + "github.com/Xhofe/go-cache" + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +const ( + offlineStatusQueued = 0 + offlineStatusRunning = 1 + offlineStatusCompleted = 2 + offlineStatusFailed = 3 + offlineStatusCanceled = 4 + offlineStatusPartiallyCompleted = 5 +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]guangyapandriver.OfflineTask](16)) +var taskG singleflight.Group[[]guangyapandriver.OfflineTask] + +func (g *GuangYaPan) GetTasks(driver *guangyapandriver.GuangYaPan, taskID string) ([]guangyapandriver.OfflineTask, error) { + key := op.Key(driver, "/cloudcollection/v1/list_task/"+taskID) + if !g.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + g.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]guangyapandriver.OfflineTask, error) { + ctx := context.Background() + tasks, err := driver.OfflineList(ctx, []string{taskID}, nil, "", 0) + if err != nil { + return nil, err + } + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]guangyapandriver.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} + +func (g *GuangYaPan) DelTaskCache(driver *guangyapandriver.GuangYaPan, taskID string) { + taskCache.Del(op.Key(driver, "/cloudcollection/v1/list_task/"+taskID)) +} + +func taskStatusText(task guangyapandriver.OfflineTask) string { + switch task.Status { + case offlineStatusQueued: + return "queued" + case offlineStatusRunning: + if task.Progress > 0 { + return fmt.Sprintf("running (%d%%)", task.Progress) + } + return "running" + case offlineStatusCompleted: + return "completed" + case offlineStatusFailed: + return "failed" + case offlineStatusCanceled: + return "canceled" + case offlineStatusPartiallyCompleted: + return "partially completed" + default: + return fmt.Sprintf("unknown status %d", task.Status) + } +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index d64e43e8615..92e7a3e1dfb 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -7,6 +7,7 @@ import ( "path/filepath" _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/guangyapan" "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" @@ -103,6 +104,16 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro } else { tempDir = filepath.Join(setting.GetStr(conf.ThunderTempDir), uid) } + case "GuangYaPan": + if _, ok := storage.(*guangyapan.GuangYaPan); ok { + tempDir = args.DstDirPath + } else { + tempBase := setting.GetStr(conf.GuangYaPanTempDir) + if tempBase == "" { + return nil, errors.New("GuangYaPan temp dir is not set") + } + tempDir = filepath.Join(tempBase, uid) + } } taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index 42b2dbfb2cb..c6ad09947e1 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -87,6 +87,9 @@ outer: if t.tool.Name() == "Thunder" { return nil } + if t.tool.Name() == "GuangYaPan" { + return nil + } if t.tool.Name() == "115 Cloud" { // hack for 115 <-time.After(time.Second * 1) @@ -159,7 +162,7 @@ func (t *DownloadTask) Update() (bool, error) { func (t *DownloadTask) Transfer() error { toolName := t.tool.Name() - if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" { + if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" || toolName == "GuangYaPan" { // 如果不是直接下载到目标路径，则进行转存 if t.TempDir != t.DstDirPath { return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go index 8aade9eae57..68a922efda7 100644 --- a/server/handles/offline_download.go +++ b/server/handles/offline_download.go @@ -2,6 +2,7 @@ package handles import ( _115 "github.com/alist-org/alist/v3/drivers/115" + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" "github.com/alist-org/alist/v3/drivers/pikpak" "github.com/alist-org/alist/v3/drivers/thunder" "github.com/alist-org/alist/v3/internal/conf" @@ -240,6 +241,50 @@ func SetThunder(c *gin.Context) { common.SuccessResp(c, "ok") } +type SetGuangYaPanReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetGuangYaPan(c *gin.Context) { + var req SetGuangYaPanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only GuangYaPan is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.GuangYaPanTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("GuangYaPan") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + func OfflineDownloadTools(c *gin.Context) { tools := tool.Tools.Names() common.SuccessResp(c, tools) diff --git a/server/router.go b/server/router.go index e930177910a..e42d09472d1 100644 --- a/server/router.go +++ b/server/router.go @@ -181,6 +181,7 @@ func admin(g *gin.RouterGroup) { setting.POST("/set_115", handles.Set115) setting.POST("/set_pikpak", handles.SetPikPak) setting.POST("/set_thunder", handles.SetThunder) + setting.POST("/set_guangyapan", handles.SetGuangYaPan) setting.POST("/set_frp", handles.SetFRP) setting.POST("/stop_frp", handles.StopFRP) setting.GET("/frp_runtime", handles.GetFRPRuntime) From f4459fcbadbb6d1c7006ff8f1c65646859220e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 16:54:35 +0800 Subject: [PATCH 649/659] fix(meta): expire missing meta cache (#9504) --- internal/bootstrap/data/setting.go | 1 + internal/conf/const.go | 1 + internal/op/meta.go | 35 ++++++++++++++++++++++++++++-- 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index 265a6502b60..2faba91ea31 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -169,6 +169,7 @@ func InitialSettings() []model.SettingItem { {Key: conf.MaxDevices, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL}, {Key: conf.DeviceEvictPolicy, Value: "deny", Type: conf.TypeSelect, Options: "deny,evict_oldest", Group: model.GLOBAL}, {Key: conf.DeviceSessionTTL, Value: "86400", Type: conf.TypeNumber, Group: model.GLOBAL}, + {Key: conf.MetaNotFoundCacheExpire, Value: "60", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE, Help: "Negative cache expiration for missing meta records, in seconds. Set 0 to disable."}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, diff --git a/internal/conf/const.go b/internal/conf/const.go index 4a3bbfa0b8f..9218e28f74e 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -52,6 +52,7 @@ const ( MaxDevices = "max_devices" DeviceEvictPolicy = "device_evict_policy" DeviceSessionTTL = "device_session_ttl" + MetaNotFoundCacheExpire = "meta_not_found_cache_expire" // index SearchIndex = "search_index" diff --git a/internal/op/meta.go b/internal/op/meta.go index 930f49634c3..29146fcc72c 100644 --- a/internal/op/meta.go +++ b/internal/op/meta.go @@ -2,9 +2,11 @@ package op import ( stdpath "path" + "strconv" "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -19,6 +21,17 @@ var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2)) // metaG maybe not needed var metaG singleflight.Group[*model.Meta] +const ( + metaCacheExpiration = time.Hour + defaultMetaNotFoundCacheSec = 60 +) + +func init() { + RegisterSettingChangingCallback(func() { + metaCache.Clear() + }) +} + func GetNearestMeta(path string) (*model.Meta, error) { return getNearestMeta(utils.FixAndCleanPath(path)) } @@ -51,17 +64,34 @@ func getMetaByPath(path string) (*model.Meta, error) { _meta, err := db.GetMetaByPath(path) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { - metaCache.Set(path, nil) + if ex := metaNotFoundCacheExpiration(); ex > 0 { + metaCache.Set(path, nil, cache.WithEx[*model.Meta](ex)) + } return nil, errs.MetaNotFound } return nil, err } - metaCache.Set(path, _meta, cache.WithEx[*model.Meta](time.Hour)) + metaCache.Set(path, _meta, cache.WithEx[*model.Meta](metaCacheExpiration)) return _meta, nil }) return meta, err } +func metaNotFoundCacheExpiration() time.Duration { + item, err := GetSettingItemByKey(conf.MetaNotFoundCacheExpire) + if err != nil || item == nil { + return time.Second * defaultMetaNotFoundCacheSec + } + seconds, err := strconv.Atoi(item.Value) + if err != nil { + return time.Second * defaultMetaNotFoundCacheSec + } + if seconds <= 0 { + return 0 + } + return time.Second * time.Duration(seconds) +} + func DeleteMetaById(id uint) error { old, err := db.GetMetaById(id) if err != nil { @@ -78,6 +108,7 @@ func UpdateMeta(u *model.Meta) error { return err } metaCache.Del(old.Path) + metaCache.Del(u.Path) return db.UpdateMeta(u) } From 7d7c921179d0d092c22d761bf700e13404c70e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 18:50:59 +0800 Subject: [PATCH 650/659] chore(auto_lang): update Go version in auto_lang workflow --- .github/workflows/auto_lang.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/auto_lang.yml b/.github/workflows/auto_lang.yml index 9108dd1bf2b..9cfe57fffdc 100644 --- a/.github/workflows/auto_lang.yml +++ b/.github/workflows/auto_lang.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.21' ] + go-version: [ '1.25' ] name: auto generate lang.json runs-on: ${{ matrix.platform }} steps: From b45f57d72ea2957d5f6a2a925852a8568f158faf Mon Sep 17 00:00:00 2001 From: OrbisAI Security Date: Fri, 15 May 2026 18:06:00 +0530 Subject: [PATCH 651/659] fixing the build by replacing := with = --- drivers/local/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/local/util.go b/drivers/local/util.go index bbd652bd7e5..8940d44e254 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -93,7 +93,7 @@ func resizeImageToBufferWithFFmpegGo(inputFile string, width int, outputFormat s outputArgs["q:v"] = "3" } - err := ffmpeg.Input(inputFile). + err = ffmpeg.Input(inputFile). Output("pipe:", outputArgs). // Output to pipe (stdout) GlobalArgs("-loglevel", "error"). Silent(true). // Suppress ffmpeg's own console output From de56968a2d59d1cdd67cdafd68cfa1ee7024d6ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Fri, 15 May 2026 21:08:50 +0800 Subject: [PATCH 652/659] fix(guangyapan): resolve offline root folder lookup (#9516) --- drivers/guangyapan/offline.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/guangyapan/offline.go b/drivers/guangyapan/offline.go index d523aaa677c..a53b8adc33f 100644 --- a/drivers/guangyapan/offline.go +++ b/drivers/guangyapan/offline.go @@ -39,7 +39,11 @@ func (d *GuangYaPan) OfflineDownload(ctx context.Context, fileURL string, parent } parentID := parentDir.GetID() - if parentID == d.RootFolderID { + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + if parentID == rootID { parentID = "" } From 9c8c16945cfcc68581069c575afb53be950fec6b Mon Sep 17 00:00:00 2001 From: OrbisAI Security Date: Sat, 16 May 2026 07:15:46 +0530 Subject: [PATCH 653/659] ci: re-trigger build after upstream guangyapan fix The previous CI failure was caused by a temporary broken state in upstream AlistGo/alist main between commits dd19d0db (removed RootFolderID field from driver.go) and de56968a (fixed offline.go to use getRootFolderID method). Our PR's CI ran in that 32-minute window. The upstream fix is now merged; this empty commit forces a fresh merge commit against the repaired upstream. Co-Authored-By: Claude Sonnet 4.6 From 02c09a776504fd3e6f017fb5150f008bee9fbf76 Mon Sep 17 00:00:00 2001 From: orbisai0security Date: Fri, 22 May 2026 09:29:48 +0000 Subject: [PATCH 654/659] fix: CVE-2026-34986 security vulnerability Automated dependency upgrade by OrbisAI Security --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 46d43d9d116..a306287ce69 100644 --- a/go.mod +++ b/go.mod @@ -103,7 +103,7 @@ require ( github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect github.com/fatedier/golib v0.5.1 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/google/jsonschema-go v0.4.2 // indirect github.com/gorilla/mux v1.8.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect diff --git a/go.sum b/go.sum index 0d42e1fe1f2..5360822db69 100644 --- a/go.sum +++ b/go.sum @@ -275,6 +275,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= From 1db7a942bf118f0800580efd4efab07a247235b8 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Thu, 28 May 2026 20:16:55 +0800 Subject: [PATCH 655/659] feat(settings): add preview_settings for per-extension preview management MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a new admin setting `preview_settings` (text/JSON, default `{}`) in the PREVIEW group. It is the per-extension override layer (order + disabled IDs) consumed by the new admin UI in alist-web. The setting is orthogonal to iframe_previews / external_previews — no migration code, no shadow writes. Backup/restore works across version boundaries because old clients simply ignore the unknown key. Companion frontend PR: AlistGo/alist-web#306 Closes #9542 --- internal/bootstrap/data/setting.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index ad3d181405e..71486335240 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -137,6 +137,7 @@ func InitialSettings() []model.SettingItem { "EPUB.js":"https://alist-org.github.io/static/epub.js/viewer.html?url=$e_url" } }`, Type: conf.TypeText, Group: model.PREVIEW}, + {Key: "preview_settings", Value: `{}`, Type: conf.TypeText, Group: model.PREVIEW}, // {Key: conf.OfficeViewers, Value: `{ // "Microsoft":"https://view.officeapps.live.com/op/view.aspx?src=$url", // "Google":"https://docs.google.com/gview?url=$url&embedded=true", From 0e9f874637f419381d8315dfb629136bf68f3dc8 Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 29 May 2026 09:37:13 +0800 Subject: [PATCH 656/659] fix(net): synchronize Buf.Close with Write/Read to prevent nil-pointer panic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit internal/net.Buf had Close() nilling the underlying bytes.Buffer without taking the rw mutex, racing against concurrent Buf.Write() / Buf.Read() calls. When the consumer of MultiReadCloser closes mid-download (via downloader.interrupt()), in-flight chunk-download goroutines could dereference a nil bytes.Buffer and panic the entire process — matching the SIGSEGV stack reported in #9190 and one crash mode from #9537. Make Close acquire the lock and have Read/Write return io.ErrClosedPipe when the buffer has been nilled, instead of panicking. Add a race-detector regression test (TestBufCloseWriteRace) that panics many times without this fix and passes cleanly with it. Co-Authored-By: Claude Opus 4.7 --- internal/net/buf_race_test.go | 44 +++++++++++++++++++++++++++++++++++ internal/net/request.go | 9 +++++++ 2 files changed, 53 insertions(+) create mode 100644 internal/net/buf_race_test.go diff --git a/internal/net/buf_race_test.go b/internal/net/buf_race_test.go new file mode 100644 index 00000000000..f8b614424c5 --- /dev/null +++ b/internal/net/buf_race_test.go @@ -0,0 +1,44 @@ +package net + +import ( + "context" + "sync" + "testing" +) + +// TestBufCloseWriteRace exercises the race fixed for issue #9537/#9190: +// Buf.Close() must synchronize with concurrent Buf.Write() calls so that +// nilling the underlying bytes.Buffer cannot panic an in-flight writer. +// Run with `go test -race ./internal/net/...` to be meaningful. +func TestBufCloseWriteRace(t *testing.T) { + const iters = 200 + const writers = 8 + + for i := 0; i < iters; i++ { + buf := NewBuf(context.Background(), 1024) + var wg sync.WaitGroup + for w := 0; w < writers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + t.Errorf("panic in Write: %v", r) + } + }() + for j := 0; j < 50; j++ { + _, _ = buf.Write([]byte("x")) + } + }() + } + go func() { + defer func() { + if r := recover(); r != nil { + t.Errorf("panic in Close: %v", r) + } + }() + buf.Close() + }() + wg.Wait() + } +} diff --git a/internal/net/request.go b/internal/net/request.go index a1ff6d20cf9..f04747435db 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -643,6 +643,10 @@ func (br *Buf) Read(p []byte) (n int, err error) { return 0, io.EOF } br.rw.Lock() + if br.buffer == nil { + br.rw.Unlock() + return 0, io.ErrClosedPipe + } n, err = br.buffer.Read(p) br.rw.Unlock() if err == nil { @@ -672,10 +676,15 @@ func (br *Buf) Write(p []byte) (n int, err error) { } br.rw.Lock() defer br.rw.Unlock() + if br.buffer == nil { + return 0, io.ErrClosedPipe + } n, err = br.buffer.Write(p) return } func (br *Buf) Close() { + br.rw.Lock() + defer br.rw.Unlock() br.buffer = nil } From 69464a2825d08672f15d6570c64eb7470c03fe8a Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 29 May 2026 09:37:25 +0800 Subject: [PATCH 657/659] fix(mcp): initialize task manager so async fs operations don't panic cmd/mcp.go (added in v3.60.0) calls Init() and LoadStorages() but skips bootstrap.InitTaskManager(). As a result fs.CopyTaskManager, fs.UploadTaskManager, fs.MoveTaskManager and friends are nil, and any MCP fs_copy / fs_move on a cross-storage target panics at internal/fs/copy.go (CopyTaskManager.Add) with a nil-pointer dereference that the MCP handler surfaces as: panic recovered in fs_copy tool handler: runtime error: invalid memory address or nil pointer dereference Mirror the cmd/server.go bootstrap order so the MCP command initializes the task manager too. Co-Authored-By: Claude Opus 4.7 --- cmd/mcp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/mcp.go b/cmd/mcp.go index 356b8bfc1b8..9ddd9977e24 100644 --- a/cmd/mcp.go +++ b/cmd/mcp.go @@ -14,6 +14,7 @@ var MCPCmd = &cobra.Command{ Run: func(cmd *cobra.Command, args []string) { Init() bootstrap.LoadStorages() + bootstrap.InitTaskManager() username, _ := cmd.Flags().GetString("user") if err := mcpserver.ServeStdio(username); err != nil { utils.Log.Fatalf("MCP STDIO server error: %v", err) From e36c68e4db3c9db9b2ea3c0bb2f36b057f2c779d Mon Sep 17 00:00:00 2001 From: okatu-loli Date: Fri, 29 May 2026 10:33:43 +0800 Subject: [PATCH 658/659] feat(api): add virtual_path field on fs/list and fs/get responses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The existing `path` field on ObjResp/ObjLabelResp returns whatever the storage driver writes into model.Object.Path, and that contract has quietly diverged across drivers: - Local sets it to the physical disk path (e.g. `/data/data/com.termux/files/home/storage/download/foo.tar.gz`) - Cloud drivers (Quark / Baidu / 115 / …) leave it empty - Some other drivers fill it with their own internal id-like path Clients that need the canonical alist virtual path (e.g. for `share`, `fs/copy`, navigation) cannot rely on `path` alone. The frontend has been working around this with branchy code such as `pathJoin(getCurrentPath(), name)` in some places and `obj.path` in others; one such mismatch caused a share regression on Local mounts whose root_folder_path differs from the mount path (visible as `failed get storage: storage not found; rawPath: ...`). Add a `virtual_path` field that is always the alist virtual path for the object: - `toObjsResp` -> `FixAndCleanPath(stdpath.Join(parent, obj.Name))` - `FsGet` -> `FixAndCleanPath(reqPath)` `path` is left untouched for backwards compatibility. Clients should prefer `virtual_path` for anything that talks to other alist APIs. --- server/handles/fsread.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 1d85dcae935..085694e3e32 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -35,6 +35,7 @@ type DirReq struct { type ObjResp struct { Id string `json:"id"` Path string `json:"path"` + VirtualPath string `json:"virtual_path"` Name string `json:"name"` Size int64 `json:"size"` IsDir bool `json:"is_dir"` @@ -65,6 +66,7 @@ type FsListResp struct { type ObjLabelResp struct { Id string `json:"id"` Path string `json:"path"` + VirtualPath string `json:"virtual_path"` Name string `json:"name"` Size int64 `json:"size"` IsDir bool `json:"is_dir"` @@ -318,6 +320,7 @@ func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjLabelResp { resp = append(resp, ObjLabelResp{ Id: obj.GetID(), Path: obj.GetPath(), + VirtualPath: utils.FixAndCleanPath(stdpath.Join(parent, obj.GetName())), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), @@ -452,6 +455,7 @@ func FsGet(c *gin.Context) { ObjResp: ObjResp{ Id: obj.GetID(), Path: obj.GetPath(), + VirtualPath: utils.FixAndCleanPath(reqPath), Name: obj.GetName(), Size: obj.GetSize(), IsDir: obj.IsDir(), From d0cec67718d9b0f3750715fe850f6c0ba9e9e87f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=83=E7=9F=B3?= Date: Mon, 1 Jun 2026 20:39:37 +0800 Subject: [PATCH 659/659] fix(lanzou): handle acw_sc__v2 anti-crawler challenge on all requests (#9548) The acw_sc__v2 challenge can be served on any pan.lanzoui.com request (share page, iframe page, ajaxm.php), but it was only handled for the first share page. This caused intermittent failures: - "uid variable not find" when mydisk.php returned the challenge on init - "not find data" when the iframe download page returned the challenge - empty "failed get link" when ajaxm.php POST returned the challenge (the challenge HTML was parsed as JSON, zt=0 -> empty info) Also fix HexXor, which was broken since the driver was introduced and made every challenge unsolvable: - bytes.NewBuffer(make([]byte, len(hex1))) prefixed the result with len(hex1) null bytes - missing zero-padding produced a single hex char when xor result < 0x10, misaligning the whole string Move the challenge solve/retry loop down into request() so every GET/POST transparently handles it, and simplify getHtml accordingly. --- drivers/lanzou/help.go | 9 ++-- drivers/lanzou/util.go | 120 +++++++++++++++++++++-------------------- 2 files changed, 67 insertions(+), 62 deletions(-) diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index b3d69006791..b9e7e5cdbd4 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -1,7 +1,6 @@ package lanzou import ( - "bytes" "fmt" "net/http" "regexp" @@ -144,11 +143,13 @@ func Unbox(hex string) string { } func HexXor(hex1, hex2 string) string { - out := bytes.NewBuffer(make([]byte, len(hex1))) - for i := 0; i < len(hex1) && i < len(hex2); i += 2 { + var out strings.Builder + out.Grow(len(hex1)) + for i := 0; i+2 <= len(hex1) && i+2 <= len(hex2); i += 2 { v1, _ := strconv.ParseInt(hex1[i:i+2], 16, 64) v2, _ := strconv.ParseInt(hex2[i:i+2], 16, 64) - out.WriteString(strconv.FormatInt(v1^v2, 16)) + // 必须补足前导 0,否则异或结果 < 0x10 时只输出一个字符,导致整个 hex 串错位 + fmt.Fprintf(&out, "%02x", v1^v2) } return out.String() } diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index be53963c157..2fe584864e1 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -95,35 +95,55 @@ func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{}, } func (d *LanZou) request(url string, method string, callback base.ReqCallback, up bool) ([]byte, error) { - var req *resty.Request + var client *resty.Client if up { once.Do(func() { upClient = base.NewRestyClient().SetTimeout(120 * time.Second) }) - req = upClient.R() + client = upClient } else { - req = base.RestyClient.R() + client = base.RestyClient } - req.SetHeaders(map[string]string{ - "Referer": "https://pc.woozooo.com", - "User-Agent": d.UserAgent, - }) - - if d.Cookie != "" { - req.SetHeader("cookie", d.Cookie) - } + // acw_sc__v2 反爬挑战可能出现在任意页面/接口(分享页、iframe 页、ajaxm.php 等)。 + // 挑战页内嵌 arg1,需据此算出 cookie 后用同一请求重试,故在最底层统一处理。 + var acwScV2 string + var body []byte + for i := 0; i < 3; i++ { + req := client.R() + req.SetHeaders(map[string]string{ + "Referer": "https://pc.woozooo.com", + "User-Agent": d.UserAgent, + }) + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + } + if acwScV2 != "" { + req.SetCookie(&http.Cookie{Name: "acw_sc__v2", Value: acwScV2}) + } + if callback != nil { + callback(req) + } - if callback != nil { - callback(req) - } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + body = res.Body() + log.Debugf("lanzou request: url=>%s ,stats=>%d ,body => %s\n", res.Request.URL, res.StatusCode(), res.String()) - res, err := req.Execute(method, url) - if err != nil { - return nil, err + if findAcwScV2Reg.Match(body) { + vs, e := CalcAcwScV2(string(body)) + if e != nil { + log.Errorf("lanzou: err => acw_sc__v2 validation error ,data => %s\n", body) + return body, e + } + acwScV2 = vs + continue + } + return body, nil } - log.Debugf("lanzou request: url=>%s ,stats=>%d ,body => %s\n", res.Request.URL, res.StatusCode(), res.String()) - return res.Body(), err + return body, errors.New("acw_sc__v2 validation error") } func (d *LanZou) Login() ([]*http.Cookie, error) { @@ -267,42 +287,28 @@ var findDownPageParamReg = regexp.MustCompile(` acw_sc__v2 validation error ,data => %s\n", firstPageDataStr) - return "", err - } - continue - } - return firstPageDataStr, nil + htmlStr, err := d.getHtml(fmt.Sprint(d.ShareUrl, "/", shareID), nil) + if err != nil { + return "", err + } + if strings.Contains(htmlStr, "取消分享") { + return "", ErrFileShareCancel + } + if strings.Contains(htmlStr, "文件不存在") { + return "", ErrFileNotExist } - return "", errors.New("acw_sc__v2 validation error") + return htmlStr, nil } // 通过分享链接获取文件或文件夹 @@ -386,11 +392,10 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( log.Errorf("lanzou: err => not find file page param ,data => %s\n", sharePageData) return nil, fmt.Errorf("not find file page param") } - data, err := d.get(fmt.Sprint(d.ShareUrl, urlpaths[1]), nil) + nextPageData, err := d.getHtml(fmt.Sprint(d.ShareUrl, urlpaths[1]), nil) if err != nil { return nil, err } - nextPageData := RemoveNotes(string(data)) param, err = htmlJsonToMap(nextPageData) if err != nil { return nil, err @@ -542,8 +547,8 @@ func (d *LanZou) getFileRealInfo(downURL string) (*int64, *time.Time) { } func (d *LanZou) getVeiAndUid() (vei string, uid string, err error) { - var resp []byte - resp, err = d.get("https://pc.woozooo.com/mydisk.php", func(req *resty.Request) { + // mydisk.php 同样可能返回 acw_sc__v2 反爬挑战页,需经 getHtml 处理后再解析 + html, err := d.getHtml("https://pc.woozooo.com/mydisk.php", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "item": "files", "action": "index", @@ -553,7 +558,7 @@ func (d *LanZou) getVeiAndUid() (vei string, uid string, err error) { return } // uid - uids := regexp.MustCompile(`uid=([^'"&;]+)`).FindStringSubmatch(string(resp)) + uids := regexp.MustCompile(`uid=([^'"&;]+)`).FindStringSubmatch(html) if len(uids) < 2 { err = fmt.Errorf("uid variable not find") return @@ -561,7 +566,6 @@ func (d *LanZou) getVeiAndUid() (vei string, uid string, err error) { uid = uids[1] // vei - html := RemoveNotes(string(resp)) data, err := htmlJsonToMap(html) if err != nil { return