diff --git a/.air.toml b/.air.toml new file mode 100644 index 00000000000..52d17fb43da --- /dev/null +++ b/.air.toml @@ -0,0 +1,44 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "tmp" + +[build] +args_bin = ["server"] +bin = "./tmp/main" +cmd = "go build -o ./tmp/main ." +delay = 0 +exclude_dir = ["assets", "tmp", "vendor", "testdata"] +exclude_file = [] +exclude_regex = ["_test.go"] +exclude_unchanged = false +follow_symlink = false +full_bin = "" +include_dir = [] +include_ext = ["go", "tpl", "tmpl", "html"] +include_file = [] +kill_delay = "0s" +log = "build-errors.log" +poll = false +poll_interval = 0 +rerun = false +rerun_delay = 500 +send_interrupt = false +stop_on_error = false + +[color] +app = "" +build = "yellow" +main = "magenta" +runner = "green" +watcher = "cyan" + +[log] +main_only = false +time = false + +[misc] +clean_on_exit = false + +[screen] +clear_on_rebuild = false +keep_scroll = true diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index a336406b355..f9e80a5aada 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -10,4 +10,4 @@ liberapay: # Replace with a single Liberapay username issuehunt: # Replace with a single IssueHunt username otechie: # Replace with a single Otechie username lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry -custom: ['https://alist.nn.ci/guide/sponsor.html'] +custom: ['https://alistgo.com/guide/sponsor.html'] diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 07a8338e5ef..f5cfaedacf8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -16,14 +16,14 @@ body: 您必须勾选以下所有内容,否则您的issue可能会被直接关闭。或者您可以去[讨论区](https://github.com/alist-org/alist/discussions) options: - label: | - I have read the [documentation](https://alist.nn.ci). - 我已经阅读了[文档](https://alist.nn.ci)。 + I have read the [documentation](https://alistgo.com). + 我已经阅读了[文档](https://alistgo.com)。 - label: | I'm sure there are no duplicate issues or discussions. 我确定没有重复的issue或讨论。 - label: | - I'm sure it's due to `AList` and not something else(such as [Network](https://alist.nn.ci/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host) ,`Dependencies` or `Operational`). - 我确定是`AList`的问题,而不是其他原因(例如[网络](https://alist.nn.ci/zh/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host),`依赖`或`操作`)。 + I'm sure it's due to `AList` and not something else(such as [Network](https://alistgo.com/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host) ,`Dependencies` or `Operational`). + 我确定是`AList`的问题,而不是其他原因(例如[网络](https://alistgo.com/zh/faq/howto.html#tls-handshake-timeout-read-connection-reset-by-peer-dns-lookup-failed-connect-connection-refused-client-timeout-exceeded-while-awaiting-headers-no-such-host),`依赖`或`操作`)。 - label: | I'm sure this issue is not fixed in the latest version. 我确定这个问题在最新版本中没有被修复。 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index be284ab6178..9012760c8f3 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: Questions & Discussions - url: https://github.com/Xhofe/alist/discussions + url: https://github.com/alist-org/alist/discussions about: Use GitHub discussions for message-board style questions and discussions. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index a16c8f98d24..a118992ce0b 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -7,7 +7,7 @@ body: label: Please make sure of the following things description: You may select more than one, even select all. options: - - label: I have read the [documentation](https://alist.nn.ci). + - label: I have read the [documentation](https://alistgo.com). - label: I'm sure there are no duplicate issues or discussions. - label: I'm sure this feature is not implemented. - label: I'm sure it's a reasonable and popular requirement. diff --git a/.github/workflows/auto_lang.yml b/.github/workflows/auto_lang.yml index 4c20efcbf4e..9cfe57fffdc 100644 --- a/.github/workflows/auto_lang.yml +++ b/.github/workflows/auto_lang.yml @@ -20,22 +20,22 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.20' ] + go-version: [ '1.25' ] name: auto generate lang.json runs-on: ${{ matrix.platform }} steps: - name: Setup go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - name: Checkout alist - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: alist - name: Checkout alist-web - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: 'alist-org/alist-web' ref: main diff --git a/.github/workflows/beta_release.yml b/.github/workflows/beta_release.yml new file mode 100644 index 00000000000..27e0142ea3f --- /dev/null +++ b/.github/workflows/beta_release.yml @@ -0,0 +1,138 @@ +name: beta release + +on: + push: + branches: [ 'main' ] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + changelog: + strategy: + matrix: + platform: [ ubuntu-latest ] + go-version: [ '1.21' ] + name: Beta Release Changelog + runs-on: ${{ matrix.platform }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Create or update ref + id: create-or-update-ref + uses: ovsds/create-or-update-ref-action@v1 + with: + ref: tags/beta + sha: ${{ github.sha }} + + - name: Delete beta tag + run: git tag -d beta + continue-on-error: true + + - name: changelog # or changelogithub@0.12 if ensure the stable result + id: changelog + run: | + git tag -l + npx changelogithub --output CHANGELOG.md +# npx changelogen@latest --output CHANGELOG.md + + - name: Upload assets + uses: softprops/action-gh-release@v2 + with: + body_path: CHANGELOG.md + files: CHANGELOG.md + prerelease: true + tag_name: beta + + release: + needs: + - changelog + strategy: + matrix: + include: + - target: '!(*musl*|*windows-arm64*|*android*|*freebsd*)' # xgo + hash: "md5" + - target: 'linux-!(arm*)-musl*' #musl-not-arm + hash: "md5-linux-musl" + - target: 'linux-arm*-musl*' #musl-arm + hash: "md5-linux-musl-arm" + - target: 'windows-arm64' #win-arm64 + hash: "md5-windows-arm64" + - target: 'android-*' #android + hash: "md5-android" + - target: 'freebsd-*' #freebsd + hash: "md5-freebsd" + + name: Beta Release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Setup web + run: bash build.sh dev web + + - name: Build + uses: go-cross/cgo-actions@v1 + with: + targets: ${{ matrix.target }} + musl-target-format: $os-$musl-$arch + out-dir: build + x-flags: | + github.com/alist-org/alist/v3/internal/conf.BuiltAt=$built_at + github.com/alist-org/alist/v3/internal/conf.GitAuthor=Xhofe + github.com/alist-org/alist/v3/internal/conf.GitCommit=$git_commit + github.com/alist-org/alist/v3/internal/conf.Version=$tag + github.com/alist-org/alist/v3/internal/conf.WebVersion=dev + + - name: Compress + run: | + bash build.sh zip ${{ matrix.hash }} + + - name: Upload assets + uses: softprops/action-gh-release@v2 + with: + files: build/compress/* + prerelease: true + tag_name: beta + + desktop: + needs: + - release + name: Beta Release Desktop + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + repository: AlistGo/desktop-release + ref: main + persist-credentials: false + fetch-depth: 0 + + - name: Commit + run: | + git config --local user.email "bot@nn.ci" + git config --local user.name "IlaBot" + git commit --allow-empty -m "Trigger build for ${{ github.sha }}" + + - name: Push commit + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.MY_TOKEN }} + branch: main + repository: AlistGo/desktop-release \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8a8f41be434..cf6eff39e48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,31 +15,49 @@ jobs: strategy: matrix: platform: [ubuntu-latest] - go-version: [ '1.20' ] + target: + - darwin-amd64 + - darwin-arm64 + - windows-amd64 + - linux-arm64-musl + - linux-amd64-musl + - windows-arm64 + - android-arm64 name: Build runs-on: ${{ matrix.platform }} + env: + GOPROXY: https://proxy.golang.org,direct steps: - - name: Setup Go - uses: actions/setup-go@v4 - with: - go-version: ${{ matrix.go-version }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Install dependencies - run: | - sudo snap install zig --classic --beta - docker pull crazymax/xgo:latest - go install github.com/crazy-max/xgo@latest - sudo apt install upx + - uses: benjlevesque/short-sha@v3.0 + id: short-sha + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.22' + + - name: Setup web + run: bash build.sh dev web - name: Build - run: | - bash build.sh dev + uses: go-cross/cgo-actions@v1 + with: + targets: ${{ matrix.target }} + musl-target-format: $os-$musl-$arch + out-dir: build + x-flags: | + github.com/alist-org/alist/v3/internal/conf.BuiltAt=$built_at + github.com/alist-org/alist/v3/internal/conf.GitAuthor=Xhofe + github.com/alist-org/alist/v3/internal/conf.GitCommit=$git_commit + github.com/alist-org/alist/v3/internal/conf.Version=$tag + github.com/alist-org/alist/v3/internal/conf.WebVersion=dev - name: Upload artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: alist - path: dist \ No newline at end of file + name: alist_${{ env.SHA }}_${{ matrix.target }} + path: build/* \ No newline at end of file diff --git a/.github/workflows/build_docker.yml b/.github/workflows/build_docker.yml deleted file mode 100644 index 48311e8f72c..00000000000 --- a/.github/workflows/build_docker.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: build_docker - -on: - push: - branches: [ main ] - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - build_docker: - name: Build docker - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 - with: - images: xhofe/alist - - name: Replace release with dev - run: | - sed -i 's/release/dev/g' Dockerfile - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - name: Login to DockerHub - uses: docker/login-action@v2 - with: - username: xhofe - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push - id: docker_build - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - - build_docker_with_aria2: - needs: build_docker - name: Build docker with aria2 - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v3 - with: - repository: alist-org/with_aria2 - ref: main - persist-credentials: false - fetch-depth: 0 - - - name: Commit - run: | - git config --local user.email "bot@nn.ci" - git config --local user.name "IlaBot" - git commit --allow-empty -m "Trigger build for ${{ github.sha }}" - - - name: Push commit - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.MY_TOKEN }} - branch: main - repository: alist-org/with_aria2 \ No newline at end of file diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index b0cfeaa8a63..056883d6d96 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -3,7 +3,7 @@ name: auto changelog on: push: tags: - - '*' + - 'v*' jobs: changelog: @@ -11,9 +11,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 + + - name: Delete beta tag + run: git tag -d beta + continue-on-error: true + - run: npx changelogithub # or changelogithub@0.12 if ensure the stable result env: GITHUB_TOKEN: ${{secrets.MY_TOKEN}} diff --git a/.github/workflows/issue_question.yml b/.github/workflows/issue_question.yml index 1cc9839a68b..3b285ac04ac 100644 --- a/.github/workflows/issue_question.yml +++ b/.github/workflows/issue_question.yml @@ -10,7 +10,7 @@ jobs: if: github.event.label.name == 'question' steps: - name: Create comment - uses: actions-cool/issues-helper@v3.5.2 + uses: actions-cool/issues-helper@v3.6.0 with: actions: 'create-comment' token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/merge_frontend_pr.yml b/.github/workflows/merge_frontend_pr.yml new file mode 100644 index 00000000000..f4e3434306b --- /dev/null +++ b/.github/workflows/merge_frontend_pr.yml @@ -0,0 +1,70 @@ +name: merge companion frontend pr + +on: + pull_request: + branches: + - 'main' + types: + - closed + workflow_dispatch: + inputs: + frontend_pr: + description: 'Frontend PR reference, e.g. AlistGo/alist-web#301 or https://github.com/AlistGo/alist-web/pull/301' + required: true + type: string + +permissions: + contents: read + +jobs: + merge_frontend_pr: + name: Merge companion frontend PR + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Find frontend PR + id: frontend + env: + INPUT_FRONTEND_PR: ${{ inputs.frontend_pr }} + PR_BODY: ${{ github.event.pull_request.body }} + run: | + set -euo pipefail + + text="${INPUT_FRONTEND_PR:-${PR_BODY:-}}" + ref="$(printf '%s\n' "$text" | grep -Eo '(https://github\.com/(AlistGo|alist-org)/alist-web/pull/[0-9]+)|((AlistGo|alist-org)/alist-web#[0-9]+)|(alist-web#[0-9]+)' | head -n1 || true)" + + if [[ -z "$ref" ]]; then + echo "found=false" >> "$GITHUB_OUTPUT" + echo "No companion frontend PR referenced." + exit 0 + fi + + if [[ "$ref" == *"/pull/"* ]]; then + number="${ref##*/}" + else + number="${ref##*#}" + fi + + echo "found=true" >> "$GITHUB_OUTPUT" + echo "url=https://github.com/AlistGo/alist-web/pull/$number" >> "$GITHUB_OUTPUT" + echo "Found companion frontend PR #$number." + + - name: Merge frontend PR + if: steps.frontend.outputs.found == 'true' + env: + GH_TOKEN: ${{ secrets.MY_TOKEN }} + FRONTEND_PR: ${{ steps.frontend.outputs.url }} + run: | + set -euo pipefail + + state="$(gh pr view "$FRONTEND_PR" --json state --jq .state)" + if [[ "$state" == "MERGED" ]]; then + echo "$FRONTEND_PR is already merged." + exit 0 + fi + if [[ "$state" != "OPEN" ]]; then + echo "$FRONTEND_PR is $state and cannot be merged." + exit 1 + fi + + gh pr merge "$FRONTEND_PR" --auto --merge || gh pr merge "$FRONTEND_PR" --merge diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6697363d2f0..2257826b064 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,10 +9,27 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.20' ] + go-version: [ '1.21' ] name: Release runs-on: ${{ matrix.platform }} steps: + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: true + swap-storage: true + - name: Prerelease uses: irongut/EditRelease@v1.2.0 with: @@ -21,12 +38,12 @@ jobs: prerelease: true - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -42,7 +59,7 @@ jobs: bash build.sh release - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: build/compress/* prerelease: false @@ -53,9 +70,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: - repository: alist-org/desktop-release + repository: AlistGo/desktop-release ref: main persist-credentials: false fetch-depth: 0 @@ -72,4 +89,4 @@ jobs: with: github_token: ${{ secrets.MY_TOKEN }} branch: main - repository: alist-org/desktop-release \ No newline at end of file + repository: AlistGo/desktop-release \ No newline at end of file diff --git a/.github/workflows/release_android.yml b/.github/workflows/release_android.yml new file mode 100644 index 00000000000..7e071cbe2ff --- /dev/null +++ b/.github/workflows/release_android.yml @@ -0,0 +1,34 @@ +name: release_android + +on: + release: + types: [ published ] + +jobs: + release_android: + strategy: + matrix: + platform: [ ubuntu-latest ] + go-version: [ '1.21' ] + name: Release + runs-on: ${{ matrix.platform }} + steps: + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build + run: | + bash build.sh release android + + - name: Upload assets + uses: softprops/action-gh-release@v2 + with: + files: build/compress/* diff --git a/.github/workflows/release_docker.yml b/.github/workflows/release_docker.yml index a16cf49d2d5..1c31b2fd20f 100644 --- a/.github/workflows/release_docker.yml +++ b/.github/workflows/release_docker.yml @@ -3,66 +3,146 @@ name: release_docker on: push: tags: - - '*' + - 'v*' + branches: + - main + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + REGISTRY: 'xhofe/alist' + REGISTRY_USERNAME: 'xhofe' + REGISTRY_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + GITHUB_CR_REPO: ghcr.io/${{ github.repository }} + ARTIFACT_NAME: 'binaries_docker_release' + RELEASE_PLATFORMS: 'linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x,linux/ppc64le,linux/riscv64' + IMAGE_PUSH: ${{ github.event_name == 'push' }} + IMAGE_IS_PROD: ${{ github.ref_type == 'tag' }} + IMAGE_TAGS_BETA: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=raw,value=beta,enable={{is_default_branch}} jobs: - release_docker: - name: Release Docker + build_binary: + name: Build Binaries for Docker Release runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - - name: Docker meta - id: meta - uses: docker/metadata-action@v4 + - uses: actions/setup-go@v5 + with: + go-version: 'stable' + + - name: Cache Musl + id: cache-musl + uses: actions/cache@v4 with: - images: xhofe/alist + path: build/musl-libs + key: docker-musl-libs-v2 + + - name: Download Musl Library + if: steps.cache-musl.outputs.cache-hit != 'true' + run: bash build.sh prepare docker-multiplatform + + - name: Build go binary (beta) + if: env.IMAGE_IS_PROD != 'true' + run: bash build.sh beta docker-multiplatform + + - name: Build go binary (release) + if: env.IMAGE_IS_PROD == 'true' + run: bash build.sh release docker-multiplatform + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + overwrite: true + path: | + build/ + !build/*.tgz + !build/musl-libs/** + + release_docker: + needs: build_binary + name: Release Docker image + runs-on: ubuntu-latest + strategy: + matrix: + image: ["latest", "ffmpeg", "aria2", "aio"] + include: + - image: "latest" + build_arg: "" + tag_favor: "" + - image: "ffmpeg" + build_arg: INSTALL_FFMPEG=true + tag_favor: "suffix=-ffmpeg,onlatest=true" + - image: "aria2" + build_arg: INSTALL_ARIA2=true + tag_favor: "suffix=-aria2,onlatest=true" + - image: "aio" + build_arg: | + INSTALL_FFMPEG=true + INSTALL_ARIA2=true + tag_favor: "suffix=-aio,onlatest=true" + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: ${{ env.ARTIFACT_NAME }} + path: 'build/' - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v2 + if: env.IMAGE_PUSH == 'true' + uses: docker/login-action@v3 + with: + logout: true + username: ${{ env.REGISTRY_USERNAME }} + password: ${{ env.REGISTRY_PASSWORD }} + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + logout: true + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 with: - username: xhofe - password: ${{ secrets.DOCKERHUB_TOKEN }} + images: | + ${{ env.REGISTRY }} + ${{ env.GITHUB_CR_REPO }} + tags: ${{ env.IMAGE_IS_PROD == 'true' && '' || env.IMAGE_TAGS_BETA }} + flavor: | + ${{ env.IMAGE_IS_PROD == 'true' && 'latest=true' || '' }} + ${{ matrix.tag_favor }} - name: Build and push id: docker_build - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v6 with: context: . - push: true + file: Dockerfile.ci + push: ${{ env.IMAGE_PUSH == 'true' }} + build-args: ${{ matrix.build_arg }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/386,linux/arm/v6,linux/s390x - - release_docker_with_aria2: - needs: release_docker - name: Release docker with aria2 - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v3 - with: - repository: alist-org/with_aria2 - ref: main - persist-credentials: false - fetch-depth: 0 - - - name: Add tag - run: | - git config --local user.email "bot@nn.ci" - git config --local user.name "IlaBot" - git tag -a ${{ github.ref_name }} -m "release ${{ github.ref_name }}" - - - name: Push tags - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.MY_TOKEN }} - branch: main - repository: alist-org/with_aria2 + platforms: ${{ env.RELEASE_PLATFORMS }} \ No newline at end of file diff --git a/.github/workflows/release_freebsd.yml b/.github/workflows/release_freebsd.yml new file mode 100644 index 00000000000..70dcecb10f9 --- /dev/null +++ b/.github/workflows/release_freebsd.yml @@ -0,0 +1,34 @@ +name: release_freebsd + +on: + release: + types: [ published ] + +jobs: + release_freebsd: + strategy: + matrix: + platform: [ ubuntu-latest ] + go-version: [ '1.21' ] + name: Release + runs-on: ${{ matrix.platform }} + steps: + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Build + run: | + bash build.sh release freebsd + + - name: Upload assets + uses: softprops/action-gh-release@v2 + with: + files: build/compress/* diff --git a/.github/workflows/release_linux_musl.yml b/.github/workflows/release_linux_musl.yml index dd298c49b43..bb5291a9ac2 100644 --- a/.github/workflows/release_linux_musl.yml +++ b/.github/workflows/release_linux_musl.yml @@ -9,18 +9,18 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.20' ] + go-version: [ '1.21' ] name: Release runs-on: ${{ matrix.platform }} steps: - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -29,6 +29,6 @@ jobs: bash build.sh release linux_musl - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: build/compress/* diff --git a/.github/workflows/release_linux_musl_arm.yml b/.github/workflows/release_linux_musl_arm.yml index f5e69c1c81b..0e8a9618a76 100644 --- a/.github/workflows/release_linux_musl_arm.yml +++ b/.github/workflows/release_linux_musl_arm.yml @@ -9,18 +9,18 @@ jobs: strategy: matrix: platform: [ ubuntu-latest ] - go-version: [ '1.20' ] + go-version: [ '1.21' ] name: Release runs-on: ${{ matrix.platform }} steps: - name: Setup Go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -29,6 +29,6 @@ jobs: bash build.sh release linux_musl_arm - name: Upload assets - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: files: build/compress/* diff --git a/.gitignore b/.gitignore index 9f5ade897ca..1d71f0d608c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ output/ *.json /build /data/ +/tmp/ /log/ /lang/ /daemon/ diff --git a/Dockerfile b/Dockerfile index 97d1b9e8811..f5e91bee230 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,42 @@ -FROM alpine:3.18 as builder +FROM alpine:edge as builder LABEL stage=go-builder WORKDIR /app/ +RUN apk add --no-cache bash curl gcc git go musl-dev +COPY go.mod go.sum ./ +RUN go mod download COPY ./ ./ -RUN apk add --no-cache bash curl gcc git go musl-dev; \ - bash build.sh release docker +RUN bash build.sh release docker -FROM alpine:3.18 +FROM alpine:edge + +ARG INSTALL_FFMPEG=false +ARG INSTALL_ARIA2=false LABEL MAINTAINER="i@nn.ci" -VOLUME /opt/alist/data/ + WORKDIR /opt/alist/ -COPY --from=builder /app/bin/alist ./ -COPY entrypoint.sh /entrypoint.sh -RUN apk add --no-cache bash ca-certificates su-exec tzdata; \ - chmod +x /entrypoint.sh -ENV PUID=0 PGID=0 UMASK=022 + +RUN apk update && \ + apk upgrade --no-cache && \ + apk add --no-cache bash ca-certificates su-exec tzdata; \ + [ "$INSTALL_FFMPEG" = "true" ] && apk add --no-cache ffmpeg; \ + [ "$INSTALL_ARIA2" = "true" ] && apk add --no-cache curl aria2 && \ + mkdir -p /opt/aria2/.aria2 && \ + wget https://github.com/P3TERX/aria2.conf/archive/refs/heads/master.tar.gz -O /tmp/aria-conf.tar.gz && \ + tar -zxvf /tmp/aria-conf.tar.gz -C /opt/aria2/.aria2 --strip-components=1 && rm -f /tmp/aria-conf.tar.gz && \ + sed -i 's|rpc-secret|#rpc-secret|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/script.conf && \ + sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/script.conf && \ + touch /opt/aria2/.aria2/aria2.session && \ + /opt/aria2/.aria2/tracker.sh ; \ + rm -rf /var/cache/apk/* + +COPY --chmod=755 --from=builder /app/bin/alist ./ +COPY --chmod=755 entrypoint.sh /entrypoint.sh +RUN /entrypoint.sh version + +ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} +VOLUME /opt/alist/data/ EXPOSE 5244 5245 -CMD [ "/entrypoint.sh" ] +CMD [ "/entrypoint.sh" ] \ No newline at end of file diff --git a/Dockerfile.ci b/Dockerfile.ci new file mode 100644 index 00000000000..6075acc639a --- /dev/null +++ b/Dockerfile.ci @@ -0,0 +1,34 @@ +FROM alpine:3.20.7 + +ARG TARGETPLATFORM +ARG INSTALL_FFMPEG=false +ARG INSTALL_ARIA2=false +LABEL MAINTAINER="i@nn.ci" + +WORKDIR /opt/alist/ + +RUN apk update && \ + apk upgrade --no-cache && \ + apk add --no-cache bash ca-certificates su-exec tzdata; \ + [ "$INSTALL_FFMPEG" = "true" ] && apk add --no-cache ffmpeg; \ + [ "$INSTALL_ARIA2" = "true" ] && apk add --no-cache curl aria2 && \ + mkdir -p /opt/aria2/.aria2 && \ + wget https://github.com/P3TERX/aria2.conf/archive/refs/heads/master.tar.gz -O /tmp/aria-conf.tar.gz && \ + tar -zxvf /tmp/aria-conf.tar.gz -C /opt/aria2/.aria2 --strip-components=1 && rm -f /tmp/aria-conf.tar.gz && \ + sed -i 's|rpc-secret|#rpc-secret|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root/.aria2|/opt/aria2/.aria2|g' /opt/aria2/.aria2/script.conf && \ + sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/aria2.conf && \ + sed -i 's|/root|/opt/aria2|g' /opt/aria2/.aria2/script.conf && \ + touch /opt/aria2/.aria2/aria2.session && \ + /opt/aria2/.aria2/tracker.sh ; \ + rm -rf /var/cache/apk/* + +COPY --chmod=755 /build/${TARGETPLATFORM}/alist ./ +COPY --chmod=755 entrypoint.sh /entrypoint.sh +RUN /entrypoint.sh version + +ENV PUID=0 PGID=0 UMASK=022 RUN_ARIA2=${INSTALL_ARIA2} +VOLUME /opt/alist/data/ +EXPOSE 5244 5245 +CMD [ "/entrypoint.sh" ] diff --git a/README.md b/README.md index 3f8fc4eebbd..032c2d17362 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,17 @@
- logo + logo

🗂️A file list program that supports multiple storages, powered by Gin and Solidjs.

latest version - + License - + Build status - + latest version @@ -19,19 +19,19 @@
- + discussions discussions - + Downloads Downloads - + sponsor
@@ -39,13 +39,13 @@ --- -English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing](./CONTRIBUTING.md) | [CODE_OF_CONDUCT](./CODE_OF_CONDUCT.md) +English | [中文](./README_cn.md) | [日本語](./README_ja.md) | [Contributing](./CONTRIBUTING.md) | [CODE_OF_CONDUCT](./CODE_OF_CONDUCT.md) ## Features - [x] Multiple storages - [x] Local storage - - [x] [Aliyundrive](https://www.aliyundrive.com/) + - [x] [Aliyundrive](https://www.alipan.com/) - [x] OneDrive / Sharepoint ([global](https://www.office.com/), [cn](https://portal.partner.microsoftonline.cn),de,us) - [x] [189cloud](https://cloud.189.cn) (Personal, Family) - [x] [GoogleDrive](https://drive.google.com/) @@ -57,8 +57,10 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) - - [x] [139yun](https://yun.139.com/) (Personal, Family) + - [x] [ProtonDrive](https://proton.me/drive) + - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) - [x] [Terabox](https://www.terabox.com/main) @@ -66,7 +68,8 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com/) - - [x] [Aliyundrive share](https://www.aliyundrive.com/) + - [x] [ILanzou](https://www.ilanzou.com/) + - [x] [Aliyundrive share](https://www.alipan.com/) - [x] [Google photo](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) - [x] [Baidu photo](https://photo.baidu.com/) @@ -74,6 +77,9 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] [115](https://115.com/) - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) + - [x] [FeijiPan](https://www.feijipan.com/) + - [x] [dogecloud](https://www.dogecloud.com/product/oss) + - [x] [Azure Blob Storage](https://azure.microsoft.com/products/storage/blobs) - [x] Easy to deploy and out-of-the-box - [x] File preview (PDF, markdown, code, plain text, ...) - [x] Image preview in gallery mode @@ -84,7 +90,7 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] - [x] Dark mode - [x] I18n - [x] Protected routes (password protection and authentication) -- [x] WebDav (see https://alist.nn.ci/guide/webdav.html for details) +- [x] WebDav (see https://alistgo.com/guide/webdav.html for details) - [x] [Docker Deploy](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare Workers proxy - [x] File/Folder package download @@ -95,7 +101,11 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Document - + + +## API Documentation (via Apifox): + + ## Demo @@ -103,18 +113,16 @@ English | [中文](./README_cn.md)| [日本語](./README_ja.md) | [Contributing] ## Discussion -Please go to our [discussion forum](https://github.com/Xhofe/alist/discussions) for general questions, **issues are for bug reports and feature requests only.** +Please go to our [discussion forum](https://github.com/alist-org/alist/discussions) for general questions, **issues are for bug reports and feature requests only.** ## Sponsor AList is an open-source software, if you happen to like this project and want me to keep going, please consider sponsoring me or providing a single donation! Thanks for all the love and support: -https://alist.nn.ci/guide/sponsor.html +https://alistgo.com/guide/sponsor.html ### Special sponsors -- [亚洲云 - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商](https://www.asiayun.com/aff/QQCOOQKZ) (sponsored Chinese API server) -- [找资源 - 阿里云盘资源搜索引擎](https://zhaoziyuan.pw/) -- [JetBrains: Essential tools for software developers and teams](https://www.jetbrains.com/) +- [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. ## Contributors @@ -135,4 +143,4 @@ The `AList` is open-source software licensed under the AGPL-3.0 license. --- -> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) +> [@GitHub](https://github.com/alist-org) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) diff --git a/README_cn.md b/README_cn.md index 6c7100d0eca..cf1b1e1c29a 100644 --- a/README_cn.md +++ b/README_cn.md @@ -1,17 +1,17 @@
- logo + logo

🗂一个支持多存储的文件列表程序,使用 Gin 和 Solidjs。

@@ -45,7 +45,7 @@ - [x] 多种存储 - [x] 本地存储 - - [x] [阿里云盘](https://www.aliyundrive.com/) + - [x] [阿里云盘](https://www.alipan.com/) - [x] OneDrive / Sharepoint([国际版](https://www.office.com/), [世纪互联](https://portal.partner.microsoftonline.cn),de,us) - [x] [天翼云盘](https://cloud.189.cn) (个人云, 家庭云) - [x] [GoogleDrive](https://drive.google.com/) @@ -57,15 +57,18 @@ - [x] [又拍云对象存储](https://www.upyun.com/products/file-storage) - [x] WebDav(支持无API的OneDrive/SharePoint) - [x] Teambition([中国](https://www.teambition.com/ ),[国际](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [分秒帧](https://www.mediatrack.cn/) - - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云) + - [x] [ProtonDrive](https://proton.me/drive) + - [x] [和彩云](https://yun.139.com/) (个人云, 家庭云,共享群组) - [x] [Yandex.Disk](https://disk.yandex.com/) - [x] [百度网盘](http://pan.baidu.com/) - [x] [UC网盘](https://drive.uc.cn) - [x] [夸克网盘](https://pan.quark.cn) - [x] [迅雷网盘](https://pan.xunlei.com) - [x] [蓝奏云](https://www.lanzou.com/) - - [x] [阿里云盘分享](https://www.aliyundrive.com/) + - [x] [蓝奏云优享版](https://www.ilanzou.com/) + - [x] [阿里云盘分享](https://www.alipan.com/) - [x] [谷歌相册](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) - [x] [一刻相册](https://photo.baidu.com/) @@ -73,6 +76,8 @@ - [x] [115](https://115.com/) - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) + - [x] [飞机盘](https://www.feijipan.com/) + - [x] [多吉云](https://www.dogecloud.com/product/oss) - [x] 部署方便,开箱即用 - [x] 文件预览(PDF、markdown、代码、纯文本……) - [x] 画廊模式下的图像预览 @@ -83,7 +88,7 @@ - [x] 黑暗模式 - [x] 国际化 - [x] 受保护的路由(密码保护和身份验证) -- [x] WebDav (具体见 https://alist.nn.ci/zh/guide/webdav.html) +- [x] WebDav (具体见 https://alistgo.com/zh/guide/webdav.html) - [x] [Docker 部署](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare workers 中转 - [x] 文件/文件夹打包下载 @@ -94,7 +99,11 @@ ## 文档 - + + +## API 文档(通过 Apifox 提供) + + ## Demo @@ -102,17 +111,15 @@ ## 讨论 -一般问题请到[讨论论坛](https://github.com/Xhofe/alist/discussions) ,**issue仅针对错误报告和功能请求。** +一般问题请到[讨论论坛](https://github.com/alist-org/alist/discussions) ,**issue仅针对错误报告和功能请求。** ## 赞助 -AList 是一个开源软件,如果你碰巧喜欢这个项目,并希望我继续下去,请考虑赞助我或提供一个单一的捐款!感谢所有的爱和支持:https://alist.nn.ci/zh/guide/sponsor.html +AList 是一个开源软件,如果你碰巧喜欢这个项目,并希望我继续下去,请考虑赞助我或提供一个单一的捐款!感谢所有的爱和支持:https://alistgo.com/zh/guide/sponsor.html ### 特别赞助 -- [亚洲云 - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商](https://www.asiayun.com/aff/QQCOOQKZ) (国内API服务器赞助) -- [找资源 - 阿里云盘资源搜索引擎](https://zhaoziyuan.pw/) -- [JetBrains: Essential tools for software developers and teams](https://www.jetbrains.com/) +- [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - 苹果生态下优雅的网盘视频播放器,iPhone,iPad,Mac,Apple TV全平台支持。 ## 贡献者 @@ -133,4 +140,4 @@ Thanks goes to these wonderful people: --- -> [@博客](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@Telegram群](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) +> [@博客](https://nn.ci/) · [@GitHub](https://github.com/alist-org) · [@Telegram群](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) diff --git a/README_ja.md b/README_ja.md index 0efcf4e33b7..a1a21253068 100644 --- a/README_ja.md +++ b/README_ja.md @@ -1,17 +1,17 @@
- logo + logo

🗂️Gin と Solidjs による、複数のストレージをサポートするファイルリストプログラム。

@@ -45,7 +45,7 @@ - [x] マルチストレージ - [x] ローカルストレージ - - [x] [Aliyundrive](https://www.aliyundrive.com/) + - [x] [Aliyundrive](https://www.alipan.com/) - [x] OneDrive / Sharepoint ([グローバル](https://www.office.com/), [cn](https://portal.partner.microsoftonline.cn),de,us) - [x] [189cloud](https://cloud.189.cn) (Personal, Family) - [x] [GoogleDrive](https://drive.google.com/) @@ -57,8 +57,10 @@ - [x] [UPYUN Storage Service](https://www.upyun.com/products/file-storage) - [x] WebDav(Support OneDrive/SharePoint without API) - [x] Teambition([China](https://www.teambition.com/ ),[International](https://us.teambition.com/ )) + - [x] [MediaFire](https://www.mediafire.com) - [x] [Mediatrack](https://www.mediatrack.cn/) - - [x] [139yun](https://yun.139.com/) (Personal, Family) + - [x] [ProtonDrive](https://proton.me/drive) + - [x] [139yun](https://yun.139.com/) (Personal, Family, Group) - [x] [YandexDisk](https://disk.yandex.com/) - [x] [BaiduNetdisk](http://pan.baidu.com/) - [x] [Terabox](https://www.terabox.com/main) @@ -66,7 +68,8 @@ - [x] [Quark](https://pan.quark.cn) - [x] [Thunder](https://pan.xunlei.com) - [x] [Lanzou](https://www.lanzou.com/) - - [x] [Aliyundrive share](https://www.aliyundrive.com/) + - [x] [ILanzou](https://www.ilanzou.com/) + - [x] [Aliyundrive share](https://www.alipan.com/) - [x] [Google photo](https://photos.google.com/) - [x] [Mega.nz](https://mega.nz) - [x] [Baidu photo](https://photo.baidu.com/) @@ -74,6 +77,8 @@ - [x] [115](https://115.com/) - [X] Cloudreve - [x] [Dropbox](https://www.dropbox.com/) + - [x] [FeijiPan](https://www.feijipan.com/) + - [x] [dogecloud](https://www.dogecloud.com/product/oss) - [x] デプロイが簡単で、すぐに使える - [x] ファイルプレビュー (PDF, マークダウン, コード, プレーンテキスト, ...) - [x] ギャラリーモードでの画像プレビュー @@ -84,7 +89,7 @@ - [x] ダークモード - [x] 国際化 - [x] 保護されたルート (パスワード保護と認証) -- [x] WebDav (詳細は https://alist.nn.ci/guide/webdav.html を参照) +- [x] WebDav (詳細は https://alistgo.com/guide/webdav.html を参照) - [x] [Docker デプロイ](https://hub.docker.com/r/xhofe/alist) - [x] Cloudflare ワーカープロキシ - [x] ファイル/フォルダパッケージのダウンロード @@ -95,7 +100,11 @@ ## ドキュメント - + + +## APIドキュメント(Apifox 提供) + + ## デモ @@ -103,18 +112,16 @@ ## ディスカッション -一般的なご質問は[ディスカッションフォーラム](https://github.com/Xhofe/alist/discussions)をご利用ください。**問題はバグレポートと機能リクエストのみです。** +一般的なご質問は[ディスカッションフォーラム](https://github.com/alist-org/alist/discussions)をご利用ください。**問題はバグレポートと機能リクエストのみです。** ## スポンサー AList はオープンソースのソフトウェアです。もしあなたがこのプロジェクトを気に入ってくださり、続けて欲しいと思ってくださるなら、ぜひスポンサーになってくださるか、1口でも寄付をしてくださるようご検討ください!すべての愛とサポートに感謝します: -https://alist.nn.ci/guide/sponsor.html +https://alistgo.com/guide/sponsor.html ### スペシャルスポンサー -- [亚洲云 - 高防服务器|服务器租用|福州高防|广东电信|香港服务器|美国服务器|海外服务器 - 国内靠谱的企业级云计算服务提供商](https://www.asiayun.com/aff/QQCOOQKZ) (sponsored Chinese API server) -- [找资源 - 阿里云盘资源搜索引擎](https://zhaoziyuan.pw/) -- [JetBrains: Essential tools for software developers and teams](https://www.jetbrains.com/) +- [VidHub](https://apps.apple.com/app/apple-store/id1659622164?pt=118612019&ct=alist&mt=8) - An elegant cloud video player within the Apple ecosystem. Support for iPhone, iPad, Mac, and Apple TV. ## コントリビューター @@ -135,4 +142,4 @@ https://alist.nn.ci/guide/sponsor.html --- -> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/Xhofe) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) +> [@Blog](https://nn.ci/) · [@GitHub](https://github.com/alist-org) · [@TelegramGroup](https://t.me/alist_chat) · [@Discord](https://discord.gg/F4ymsH4xv2) diff --git a/build.sh b/build.sh index 7ca010a7a14..4045820adfd 100644 --- a/build.sh +++ b/build.sh @@ -1,13 +1,16 @@ appName="alist" builtAt="$(date +'%F %T %z')" -goVersion=$(go version | sed 's/go version //') gitAuthor="Xhofe " gitCommit=$(git log --pretty=format:"%h" -1) if [ "$1" = "dev" ]; then version="dev" webVersion="dev" +elif [ "$1" = "beta" ]; then + version="beta" + webVersion="dev" else + git tag -d beta version=$(git describe --abbrev=0 --tags) webVersion=$(wget -qO- -t1 -T2 "https://api.github.com/repos/alist-org/alist-web/releases/latest" | grep "tag_name" | head -n 1 | awk -F ":" '{print $2}' | sed 's/\"//g;s/,//g;s/ //g') fi @@ -18,7 +21,6 @@ echo "frontend version: $webVersion" ldflags="\ -w -s \ -X 'github.com/alist-org/alist/v3/internal/conf.BuiltAt=$builtAt' \ --X 'github.com/alist-org/alist/v3/internal/conf.GoVersion=$goVersion' \ -X 'github.com/alist-org/alist/v3/internal/conf.GitAuthor=$gitAuthor' \ -X 'github.com/alist-org/alist/v3/internal/conf.GitCommit=$gitCommit' \ -X 'github.com/alist-org/alist/v3/internal/conf.Version=$version' \ @@ -49,6 +51,7 @@ BuildWinArm64() { export GOARCH=arm64 export CC=$(pwd)/wrapper/zcc-arm64 export CXX=$(pwd)/wrapper/zcxx-arm64 + export CGO_ENABLED=1 go build -o "$1" -ldflags="$ldflags" -tags=jsoniter . } @@ -75,7 +78,7 @@ BuildDev() { export CGO_ENABLED=1 go build -o ./dist/$appName-$os_arch -ldflags="$muslflags" -tags=jsoniter . done - xgo -targets=windows/amd64,darwin/amd64 -out "$appName" -ldflags="$ldflags" -tags=jsoniter . + xgo -targets=windows/amd64,darwin/amd64,darwin/arm64 -out "$appName" -ldflags="$ldflags" -tags=jsoniter . mv alist-* dist cd dist cp ./alist-windows-amd64.exe ./alist-windows-amd64-upx.exe @@ -88,6 +91,57 @@ BuildDocker() { go build -o ./bin/alist -ldflags="$ldflags" -tags=jsoniter . } +PrepareBuildDockerMusl() { + mkdir -p build/musl-libs + BASE="https://github.com/go-cross/musl-toolchain-archive/releases/latest/download/" + FILES=(x86_64-linux-musl-cross aarch64-linux-musl-cross i486-linux-musl-cross s390x-linux-musl-cross armv6-linux-musleabihf-cross armv7l-linux-musleabihf-cross riscv64-linux-musl-cross powerpc64le-linux-musl-cross) + for i in "${FILES[@]}"; do + url="${BASE}${i}.tgz" + lib_tgz="build/${i}.tgz" + curl -L -o "${lib_tgz}" "${url}" + tar xf "${lib_tgz}" --strip-components 1 -C build/musl-libs + rm -f "${lib_tgz}" + done +} + +BuildDockerMultiplatform() { + go mod download + + # run PrepareBuildDockerMusl before build + export PATH=$PATH:$PWD/build/musl-libs/bin + + docker_lflags="--extldflags '-static -fpic' $ldflags" + export CGO_ENABLED=1 + + OS_ARCHES=(linux-amd64 linux-arm64 linux-386 linux-s390x linux-riscv64 linux-ppc64le) + CGO_ARGS=(x86_64-linux-musl-gcc aarch64-linux-musl-gcc i486-linux-musl-gcc s390x-linux-musl-gcc riscv64-linux-musl-gcc powerpc64le-linux-musl-gcc) + for i in "${!OS_ARCHES[@]}"; do + os_arch=${OS_ARCHES[$i]} + cgo_cc=${CGO_ARGS[$i]} + os=${os_arch%%-*} + arch=${os_arch##*-} + export GOOS=$os + export GOARCH=$arch + export CC=${cgo_cc} + echo "building for $os_arch" + go build -o build/$os/$arch/alist -ldflags="$docker_lflags" -tags=jsoniter . + done + + DOCKER_ARM_ARCHES=(linux-arm/v6 linux-arm/v7) + CGO_ARGS=(armv6-linux-musleabihf-gcc armv7l-linux-musleabihf-gcc) + GO_ARM=(6 7) + export GOOS=linux + export GOARCH=arm + for i in "${!DOCKER_ARM_ARCHES[@]}"; do + docker_arch=${DOCKER_ARM_ARCHES[$i]} + cgo_cc=${CGO_ARGS[$i]} + export GOARM=${GO_ARM[$i]} + export CC=${cgo_cc} + echo "building for $docker_arch" + go build -o build/${docker_arch%%-*}/${docker_arch##*-}/alist -ldflags="$docker_lflags" -tags=jsoniter . + done +} + BuildRelease() { rm -rf .git/ mkdir -p "build" @@ -159,6 +213,50 @@ BuildReleaseLinuxMuslArm() { done } +BuildReleaseAndroid() { + rm -rf .git/ + mkdir -p "build" + wget https://dl.google.com/android/repository/android-ndk-r26b-linux.zip + unzip android-ndk-r26b-linux.zip + rm android-ndk-r26b-linux.zip + OS_ARCHES=(amd64 arm64 386 arm) + CGO_ARGS=(x86_64-linux-android24-clang aarch64-linux-android24-clang i686-linux-android24-clang armv7a-linux-androideabi24-clang) + for i in "${!OS_ARCHES[@]}"; do + os_arch=${OS_ARCHES[$i]} + cgo_cc=$(realpath android-ndk-r26b/toolchains/llvm/prebuilt/linux-x86_64/bin/${CGO_ARGS[$i]}) + echo building for android-${os_arch} + export GOOS=android + export GOARCH=${os_arch##*-} + export CC=${cgo_cc} + export CGO_ENABLED=1 + go build -o ./build/$appName-android-$os_arch -ldflags="$ldflags" -tags=jsoniter . + android-ndk-r26b/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip ./build/$appName-android-$os_arch + done +} + +BuildReleaseFreeBSD() { + rm -rf .git/ + mkdir -p "build/freebsd" + OS_ARCHES=(amd64 arm64 i386) + GO_ARCHES=(amd64 arm64 386) + CGO_ARGS=(x86_64-unknown-freebsd14.1 aarch64-unknown-freebsd14.1 i386-unknown-freebsd14.1) + for i in "${!OS_ARCHES[@]}"; do + os_arch=${OS_ARCHES[$i]} + cgo_cc="clang --target=${CGO_ARGS[$i]} --sysroot=/opt/freebsd/${os_arch}" + echo building for freebsd-${os_arch} + sudo mkdir -p "/opt/freebsd/${os_arch}" + wget -q https://download.freebsd.org/releases/${os_arch}/14.3-RELEASE/base.txz + sudo tar -xf ./base.txz -C /opt/freebsd/${os_arch} + rm base.txz + export GOOS=freebsd + export GOARCH=${GO_ARCHES[$i]} + export CC=${cgo_cc} + export CGO_ENABLED=1 + export CGO_LDFLAGS="-fuse-ld=lld" + go build -o ./build/$appName-freebsd-$os_arch -ldflags="$ldflags" -tags=jsoniter . + done +} + MakeRelease() { cd build mkdir compress @@ -166,12 +264,22 @@ MakeRelease() { cp "$i" alist tar -czvf compress/"$i".tar.gz alist rm -f alist + done + for i in $(find . -type f -name "$appName-android-*"); do + cp "$i" alist + tar -czvf compress/"$i".tar.gz alist + rm -f alist done for i in $(find . -type f -name "$appName-darwin-*"); do cp "$i" alist tar -czvf compress/"$i".tar.gz alist rm -f alist done + for i in $(find . -type f -name "$appName-freebsd-*"); do + cp "$i" alist + tar -czvf compress/"$i".tar.gz alist + rm -f alist + done for i in $(find . -type f -name "$appName-windows-*"); do cp "$i" alist.exe zip compress/$(echo $i | sed 's/\.[^.]*$//').zip alist.exe @@ -187,23 +295,47 @@ if [ "$1" = "dev" ]; then FetchWebDev if [ "$2" = "docker" ]; then BuildDocker + elif [ "$2" = "docker-multiplatform" ]; then + BuildDockerMultiplatform + elif [ "$2" = "web" ]; then + echo "web only" else BuildDev fi -elif [ "$1" = "release" ]; then - FetchWebRelease +elif [ "$1" = "release" -o "$1" = "beta" ]; then + if [ "$1" = "beta" ]; then + FetchWebDev + else + FetchWebRelease + fi if [ "$2" = "docker" ]; then BuildDocker + elif [ "$2" = "docker-multiplatform" ]; then + BuildDockerMultiplatform elif [ "$2" = "linux_musl_arm" ]; then BuildReleaseLinuxMuslArm MakeRelease "md5-linux-musl-arm.txt" elif [ "$2" = "linux_musl" ]; then BuildReleaseLinuxMusl MakeRelease "md5-linux-musl.txt" + elif [ "$2" = "android" ]; then + BuildReleaseAndroid + MakeRelease "md5-android.txt" + elif [ "$2" = "freebsd" ]; then + BuildReleaseFreeBSD + MakeRelease "md5-freebsd.txt" + elif [ "$2" = "web" ]; then + echo "web only" else BuildRelease MakeRelease "md5.txt" fi +elif [ "$1" = "prepare" ]; then + if [ "$2" = "docker-multiplatform" ]; then + PrepareBuildDockerMusl + fi +elif [ "$1" = "zip" ]; then + MakeRelease "$2".txt else echo -e "Parameter error" fi diff --git a/cmd/common.go b/cmd/common.go index b4a7081c33f..d88a86eb09d 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -1,6 +1,7 @@ package cmd import ( + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_46_0" "os" "path/filepath" "strconv" @@ -16,8 +17,16 @@ func Init() { bootstrap.InitConfig() bootstrap.Log() bootstrap.InitDB() + + if v3_46_0.IsLegacyRoleDetected() { + utils.Log.Warnf("Detected legacy role format, executing ConvertLegacyRoles patch early...") + v3_46_0.ConvertLegacyRoles() + } + data.InitData() + bootstrap.InitStreamLimit() bootstrap.InitIndex() + bootstrap.InitUpgradePatch() } func Release() { diff --git a/cmd/kill.go b/cmd/kill.go new file mode 100644 index 00000000000..3378fd70988 --- /dev/null +++ b/cmd/kill.go @@ -0,0 +1,54 @@ +package cmd + +import ( + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "os" +) + +// KillCmd represents the kill command +var KillCmd = &cobra.Command{ + Use: "kill", + Short: "Force kill alist server process by daemon/pid file", + Run: func(cmd *cobra.Command, args []string) { + kill() + }, +} + +func kill() { + initDaemon() + if pid == -1 { + log.Info("Seems not have been started. Try use `alist start` to start server.") + return + } + process, err := os.FindProcess(pid) + if err != nil { + log.Errorf("failed to find process by pid: %d, reason: %v", pid, process) + return + } + err = process.Kill() + if err != nil { + log.Errorf("failed to kill process %d: %v", pid, err) + } else { + log.Info("killed process: ", pid) + } + err = os.Remove(pidFile) + if err != nil { + log.Errorf("failed to remove pid file") + } + pid = -1 +} + +func init() { + RootCmd.AddCommand(KillCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // stopCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // stopCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/lang.go b/cmd/lang.go index 8d816ca2b30..5d8ce837878 100644 --- a/cmd/lang.go +++ b/cmd/lang.go @@ -12,6 +12,7 @@ import ( "strings" _ "github.com/alist-org/alist/v3/drivers" + "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/bootstrap/data" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/op" @@ -137,9 +138,10 @@ var LangCmd = &cobra.Command{ Use: "lang", Short: "Generate language json file", Run: func(cmd *cobra.Command, args []string) { + bootstrap.InitConfig() err := os.MkdirAll("lang", 0777) if err != nil { - utils.Log.Fatal("failed create folder: %s", err.Error()) + utils.Log.Fatalf("failed create folder: %s", err.Error()) } generateDriversJson() generateSettingsJson() diff --git a/cmd/mcp.go b/cmd/mcp.go new file mode 100644 index 00000000000..9ddd9977e24 --- /dev/null +++ b/cmd/mcp.go @@ -0,0 +1,28 @@ +package cmd + +import ( + "github.com/alist-org/alist/v3/internal/bootstrap" + mcpserver "github.com/alist-org/alist/v3/server/mcp" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/spf13/cobra" +) + +var MCPCmd = &cobra.Command{ + Use: "mcp", + Short: "Start MCP server in STDIO mode", + Long: `Start an MCP (Model Context Protocol) server that communicates via STDIO, suitable for integration with AI assistants like Claude Desktop.`, + Run: func(cmd *cobra.Command, args []string) { + Init() + bootstrap.LoadStorages() + bootstrap.InitTaskManager() + username, _ := cmd.Flags().GetString("user") + if err := mcpserver.ServeStdio(username); err != nil { + utils.Log.Fatalf("MCP STDIO server error: %v", err) + } + }, +} + +func init() { + MCPCmd.Flags().String("user", "admin", "Username for MCP operations") + RootCmd.AddCommand(MCPCmd) +} diff --git a/cmd/root.go b/cmd/root.go index 297eb7f8940..cd50529728b 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -5,6 +5,9 @@ import ( "os" "github.com/alist-org/alist/v3/cmd/flags" + _ "github.com/alist-org/alist/v3/drivers" + _ "github.com/alist-org/alist/v3/internal/archive" + _ "github.com/alist-org/alist/v3/internal/offline_download" "github.com/spf13/cobra" ) @@ -13,7 +16,7 @@ var RootCmd = &cobra.Command{ Short: "A file list program that supports multiple storage.", Long: `A file list program that supports multiple storage, built with love by Xhofe and friends in Go/Solid.js. -Complete documentation is available at https://alist.nn.ci/`, +Complete documentation is available at https://alistgo.com/`, } func Execute() { diff --git a/cmd/server.go b/cmd/server.go index 94a60c7208f..9e76333a627 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "errors" "fmt" "net" "net/http" @@ -12,15 +13,21 @@ import ( "syscall" "time" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/KirCute/sftpd-alist" "github.com/alist-org/alist/v3/cmd/flags" - _ "github.com/alist-org/alist/v3/drivers" "github.com/alist-org/alist/v3/internal/bootstrap" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" + "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server" + mcpserver "github.com/alist-org/alist/v3/server/mcp" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" ) // ServerCmd represents the server command @@ -35,23 +42,28 @@ the address is defined in config file`, utils.Log.Infof("delayed start for %d seconds", conf.Conf.DelayedStart) time.Sleep(time.Duration(conf.Conf.DelayedStart) * time.Second) } - bootstrap.InitAria2() - bootstrap.InitQbittorrent() + bootstrap.InitOfflineDownloadTools() bootstrap.LoadStorages() + bootstrap.InitTaskManager() + bootstrap.InitFRP() if !flags.Debug && !flags.Dev { gin.SetMode(gin.ReleaseMode) } r := gin.New() r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) server.Init(r) + var httpHandler http.Handler = r + if conf.Conf.Scheme.EnableH2c { + httpHandler = h2c.NewHandler(r, &http2.Server{}) + } var httpSrv, httpsSrv, unixSrv *http.Server if conf.Conf.Scheme.HttpPort != -1 { httpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.Scheme.HttpPort) utils.Log.Infof("start HTTP server @ %s", httpBase) - httpSrv = &http.Server{Addr: httpBase, Handler: r} + httpSrv = &http.Server{Addr: httpBase, Handler: httpHandler} go func() { err := httpSrv.ListenAndServe() - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { utils.Log.Fatalf("failed to start http: %s", err.Error()) } }() @@ -62,14 +74,14 @@ the address is defined in config file`, httpsSrv = &http.Server{Addr: httpsBase, Handler: r} go func() { err := httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { utils.Log.Fatalf("failed to start https: %s", err.Error()) } }() } if conf.Conf.Scheme.UnixFile != "" { utils.Log.Infof("start unix server @ %s", conf.Conf.Scheme.UnixFile) - unixSrv = &http.Server{Handler: r} + unixSrv = &http.Server{Handler: httpHandler} go func() { listener, err := net.Listen("unix", conf.Conf.Scheme.UnixFile) if err != nil { @@ -86,11 +98,81 @@ the address is defined in config file`, } } err = unixSrv.Serve(listener) - if err != nil && err != http.ErrServerClosed { + if err != nil && !errors.Is(err, http.ErrServerClosed) { utils.Log.Fatalf("failed to start unix: %s", err.Error()) } }() } + if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable { + s3r := gin.New() + s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out)) + server.InitS3(s3r) + s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port) + utils.Log.Infof("start S3 server @ %s", s3Base) + go func() { + var err error + if conf.Conf.S3.SSL { + httpsSrv = &http.Server{Addr: s3Base, Handler: s3r} + err = httpsSrv.ListenAndServeTLS(conf.Conf.Scheme.CertFile, conf.Conf.Scheme.KeyFile) + } + if !conf.Conf.S3.SSL { + httpSrv = &http.Server{Addr: s3Base, Handler: s3r} + err = httpSrv.ListenAndServe() + } + if err != nil && !errors.Is(err, http.ErrServerClosed) { + utils.Log.Fatalf("failed to start s3 server: %s", err.Error()) + } + }() + } + var ftpDriver *server.FtpMainDriver + var ftpServer *ftpserver.FtpServer + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable { + var err error + ftpDriver, err = server.NewMainDriver() + if err != nil { + utils.Log.Fatalf("failed to start ftp driver: %s", err.Error()) + } else { + utils.Log.Infof("start ftp server on %s", conf.Conf.FTP.Listen) + go func() { + ftpServer = ftpserver.NewFtpServer(ftpDriver) + err = ftpServer.ListenAndServe() + if err != nil { + utils.Log.Fatalf("problem ftp server listening: %s", err.Error()) + } + }() + } + } + var sftpDriver *server.SftpDriver + var sftpServer *sftpd.SftpServer + if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable { + var err error + sftpDriver, err = server.NewSftpDriver() + if err != nil { + utils.Log.Fatalf("failed to start sftp driver: %s", err.Error()) + } else { + utils.Log.Infof("start sftp server on %s", conf.Conf.SFTP.Listen) + go func() { + sftpServer = sftpd.NewSftpServer(sftpDriver) + err = sftpServer.RunServer() + if err != nil { + utils.Log.Fatalf("problem sftp server listening: %s", err.Error()) + } + }() + } + } + var mcpHttpSrv *http.Server + if conf.Conf.MCP.Port != -1 && conf.Conf.MCP.Enable { + mcpHandler := mcpserver.NewHTTPHandler() + mcpBase := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.MCP.Port) + utils.Log.Infof("start MCP server @ %s", mcpBase) + mcpHttpSrv = &http.Server{Addr: mcpBase, Handler: mcpHandler} + go func() { + err := mcpHttpSrv.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + utils.Log.Fatalf("failed to start MCP server: %s", err.Error()) + } + }() + } // Wait for interrupt signal to gracefully shutdown the server with // a timeout of 1 second. quit := make(chan os.Signal, 1) @@ -100,6 +182,8 @@ the address is defined in config file`, signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) <-quit utils.Log.Println("Shutdown server...") + fs.ArchiveContentUploadTaskManager.RemoveAll() + frp.Instance.Stop() Release() ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() @@ -131,6 +215,34 @@ the address is defined in config file`, } }() } + if conf.Conf.FTP.Listen != "" && conf.Conf.FTP.Enable && ftpServer != nil && ftpDriver != nil { + wg.Add(1) + go func() { + defer wg.Done() + ftpDriver.Stop() + if err := ftpServer.Stop(); err != nil { + utils.Log.Fatal("FTP server shutdown err: ", err) + } + }() + } + if conf.Conf.SFTP.Listen != "" && conf.Conf.SFTP.Enable && sftpServer != nil && sftpDriver != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := sftpServer.Close(); err != nil { + utils.Log.Fatal("SFTP server shutdown err: ", err) + } + }() + } + if conf.Conf.MCP.Port != -1 && conf.Conf.MCP.Enable && mcpHttpSrv != nil { + wg.Add(1) + go func() { + defer wg.Done() + if err := mcpHttpSrv.Shutdown(ctx); err != nil { + utils.Log.Fatal("MCP server shutdown err: ", err) + } + }() + } wg.Wait() utils.Log.Println("Server exit") }, diff --git a/cmd/stop.go b/cmd/stop_default.go similarity index 87% rename from cmd/stop.go rename to cmd/stop_default.go index 09fba7b759d..8f133940a29 100644 --- a/cmd/stop.go +++ b/cmd/stop_default.go @@ -1,10 +1,10 @@ -/* -Copyright © 2022 NAME HERE -*/ +//go:build !windows + package cmd import ( "os" + "syscall" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -30,11 +30,11 @@ func stop() { log.Errorf("failed to find process by pid: %d, reason: %v", pid, process) return } - err = process.Kill() + err = process.Signal(syscall.SIGTERM) if err != nil { - log.Errorf("failed to kill process %d: %v", pid, err) + log.Errorf("failed to terminate process %d: %v", pid, err) } else { - log.Info("killed process: ", pid) + log.Info("terminated process: ", pid) } err = os.Remove(pidFile) if err != nil { diff --git a/cmd/stop_windows.go b/cmd/stop_windows.go new file mode 100644 index 00000000000..e086eab1ea8 --- /dev/null +++ b/cmd/stop_windows.go @@ -0,0 +1,34 @@ +//go:build windows + +package cmd + +import ( + "github.com/spf13/cobra" +) + +// StopCmd represents the stop command +var StopCmd = &cobra.Command{ + Use: "stop", + Short: "Same as the kill command", + Run: func(cmd *cobra.Command, args []string) { + stop() + }, +} + +func stop() { + kill() +} + +func init() { + RootCmd.AddCommand(StopCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // stopCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // stopCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/version.go b/cmd/version.go index cdf4d71fcee..a758816ed96 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -6,6 +6,7 @@ package cmd import ( "fmt" "os" + "runtime" "github.com/alist-org/alist/v3/internal/conf" "github.com/spf13/cobra" @@ -16,14 +17,15 @@ var VersionCmd = &cobra.Command{ Use: "version", Short: "Show current version of AList", Run: func(cmd *cobra.Command, args []string) { + goVersion := fmt.Sprintf("%s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH) + fmt.Printf(`Built At: %s Go Version: %s Author: %s Commit ID: %s Version: %s WebVersion: %s -`, - conf.BuiltAt, conf.GoVersion, conf.GitAuthor, conf.GitCommit, conf.Version, conf.WebVersion) +`, conf.BuiltAt, goVersion, conf.GitAuthor, conf.GitCommit, conf.Version, conf.WebVersion) os.Exit(0) }, } diff --git a/drivers/115/appver.go b/drivers/115/appver.go new file mode 100644 index 00000000000..78e11a5443f --- /dev/null +++ b/drivers/115/appver.go @@ -0,0 +1,43 @@ +package _115 + +import ( + driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" + log "github.com/sirupsen/logrus" +) + +var ( + md5Salt = "Qclm8MGWUv59TnrR0XPg" + appVer = "27.0.5.7" +) + +func (d *Pan115) getAppVersion() ([]driver115.AppVersion, error) { + result := driver115.VersionResp{} + resp, err := base.RestyClient.R().Get(driver115.ApiGetVersion) + + err = driver115.CheckErr(err, &result, resp) + if err != nil { + return nil, err + } + + return result.Data.GetAppVersions(), nil +} + +func (d *Pan115) getAppVer() string { + // todo add some cache? + vers, err := d.getAppVersion() + if err != nil { + log.Warnf("[115] get app version failed: %v", err) + return appVer + } + for _, ver := range vers { + if ver.AppName == "win" { + return ver.Version + } + } + return appVer +} + +func (d *Pan115) initAppVer() { + appVer = d.getAppVer() +} diff --git a/drivers/115/driver.go b/drivers/115/driver.go index 15f6b4087b4..60fe60e68e7 100644 --- a/drivers/115/driver.go +++ b/drivers/115/driver.go @@ -3,6 +3,7 @@ package _115 import ( "context" "strings" + "sync" driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/driver" @@ -16,8 +17,9 @@ import ( type Pan115 struct { model.Storage Addition - client *driver115.Pan115Client - limiter *rate.Limiter + client *driver115.Pan115Client + limiter *rate.Limiter + appVerOnce sync.Once } func (d *Pan115) Config() driver.Config { @@ -29,6 +31,7 @@ func (d *Pan115) GetAddition() driver.Additional { } func (d *Pan115) Init(ctx context.Context) error { + d.appVerOnce.Do(d.initAppVer) if d.LimitRate > 0 { d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) } @@ -63,8 +66,11 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err := d.WaitLimit(ctx); err != nil { return nil, err } - downloadInfo, err := d.client. - DownloadWithUA(file.(*FileObj).PickCode, driver115.UA115Browser) + userAgent := "" + if args.Header != nil { + userAgent = args.Header.Get("User-Agent") + } + downloadInfo, err := d.client.DownloadWithUA(file.(*FileObj).PickCode, userAgent) if err != nil { return nil, err } @@ -75,28 +81,60 @@ func (d *Pan115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) return link, nil } -func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { +func (d *Pan115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err } - if _, err := d.client.Mkdir(parentDir.GetID(), dirName); err != nil { - return err + + result := driver115.MkdirResp{} + form := map[string]string{ + "pid": parentDir.GetID(), + "cname": dirName, } - return nil + req := d.client.NewRequest(). + SetFormData(form). + SetResult(&result). + ForceContentType("application/json;charset=UTF-8") + + resp, err := req.Post(driver115.ApiDirAdd) + + err = driver115.CheckErr(err, &result, resp) + if err != nil { + return nil, err + } + f, err := d.getNewFile(result.FileID) + if err != nil { + return nil, nil + } + return f, nil } -func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *Pan115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err + } + if err := d.client.Move(dstDir.GetID(), srcObj.GetID()); err != nil { + return nil, err } - return d.client.Move(dstDir.GetID(), srcObj.GetID()) + f, err := d.getNewFile(srcObj.GetID()) + if err != nil { + return nil, nil + } + return f, nil } -func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) error { +func (d *Pan115) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err + } + if err := d.client.Rename(srcObj.GetID(), newName); err != nil { + return nil, err } - return d.client.Rename(srcObj.GetID(), newName) + f, err := d.getNewFile((srcObj.GetID())) + if err != nil { + return nil, nil + } + return f, nil } func (d *Pan115) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -113,9 +151,9 @@ func (d *Pan115) Remove(ctx context.Context, obj model.Obj) error { return d.client.Delete(obj.GetID()) } -func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if err := d.WaitLimit(ctx); err != nil { - return err + return nil, err } var ( @@ -124,10 +162,10 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr ) if ok, err := d.client.UploadAvailable(); err != nil || !ok { - return err + return nil, err } if stream.GetSize() > d.client.UploadMetaInfo.SizeLimit { - return driver115.ErrUploadTooLarge + return nil, driver115.ErrUploadTooLarge } //if digest, err = d.client.GetDigestResult(stream); err != nil { // return err @@ -140,22 +178,22 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: hashSize}) if err != nil { - return err + return nil, err } preHash, err := utils.HashReader(utils.SHA1, reader) if err != nil { - return err + return nil, err } preHash = strings.ToUpper(preHash) fullHash := stream.GetHash().GetHash(utils.SHA1) if len(fullHash) <= 0 { tmpF, err := stream.CacheFullInTempFile() if err != nil { - return err + return nil, err } fullHash, err = utils.HashFile(utils.SHA1, tmpF) if err != nil { - return err + return nil, err } } fullHash = strings.ToUpper(fullHash) @@ -164,21 +202,52 @@ func (d *Pan115) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr // note that 115 add timeout for rapid-upload, // and "sig invalid" err is thrown even when the hash is correct after timeout. if fastInfo, err = d.rapidUpload(stream.GetSize(), stream.GetName(), dirID, preHash, fullHash, stream); err != nil { - return err + return nil, err } if matched, err := fastInfo.Ok(); err != nil { - return err + return nil, err } else if matched { - return nil + f, err := d.getNewFileByPickCode(fastInfo.PickCode) + if err != nil { + return nil, nil + } + return f, nil } + var uploadResult *UploadResult // 闪传失败,上传 - if stream.GetSize() <= utils.KB { // 文件大小小于1KB,改用普通模式上传 - return d.client.UploadByOSS(&fastInfo.UploadOSSParams, stream, dirID) + if stream.GetSize() <= 10*utils.MB { // 文件大小小于10MB,改用普通模式上传 + if uploadResult, err = d.UploadByOSS(ctx, &fastInfo.UploadOSSParams, stream, dirID, up); err != nil { + return nil, err + } + } else { + // 分片上传 + if uploadResult, err = d.UploadByMultipart(ctx, &fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID, up); err != nil { + return nil, err + } } - // 分片上传 - return d.UploadByMultipart(&fastInfo.UploadOSSParams, stream.GetSize(), stream, dirID) + file, err := d.getNewFile(uploadResult.Data.FileID) + if err != nil { + return nil, nil + } + return file, nil +} + +func (d *Pan115) OfflineList(ctx context.Context) ([]*driver115.OfflineTask, error) { + resp, err := d.client.ListOfflineTask(0) + if err != nil { + return nil, err + } + return resp.Tasks, nil +} + +func (d *Pan115) OfflineDownload(ctx context.Context, uris []string, dstDir model.Obj) ([]string, error) { + return d.client.AddOfflineTaskURIs(uris, dstDir.GetID(), driver115.WithAppVer(appVer)) +} + +func (d *Pan115) DeleteOfflineTasks(ctx context.Context, hashes []string, deleteFiles bool) error { + return d.client.DeleteOfflineTasks(hashes, deleteFiles) } var _ driver.Driver = (*Pan115)(nil) diff --git a/drivers/115/meta.go b/drivers/115/meta.go index 16ec22cdab8..bcea174922c 100644 --- a/drivers/115/meta.go +++ b/drivers/115/meta.go @@ -6,19 +6,20 @@ import ( ) type Addition struct { - Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` - QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` - PageSize int64 `json:"page_size" type:"number" default:"56" help:"list api per page size of 115 driver"` - LimitRate float64 `json:"limit_rate" type:"number" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` + Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` + QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` + QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` + PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` + LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate ([limit]r/1s)"` driver.RootID } var config = driver.Config{ Name: "115 Cloud", DefaultRoot: "0", - OnlyProxy: true, - //OnlyLocal: true, - NoOverwriteUpload: true, + // OnlyProxy: true, + // OnlyLocal: true, + // NoOverwriteUpload: true, } func init() { diff --git a/drivers/115/types.go b/drivers/115/types.go index 830e347b44e..7a80e3ef047 100644 --- a/drivers/115/types.go +++ b/drivers/115/types.go @@ -1,16 +1,18 @@ package _115 import ( + "time" + "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" - "time" ) var _ model.Obj = (*FileObj)(nil) type FileObj struct { driver.File + ThumbURL string } func (f *FileObj) CreateTime() time.Time { @@ -20,3 +22,22 @@ func (f *FileObj) CreateTime() time.Time { func (f *FileObj) GetHash() utils.HashInfo { return utils.NewHashInfo(utils.SHA1, f.Sha1) } + +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + +type UploadResult struct { + driver.BasicResp + Data struct { + PickCode string `json:"pick_code"` + FileSize int `json:"file_size"` + FileID string `json:"file_id"` + ThumbURL string `json:"thumb_url"` + Sha1 string `json:"sha1"` + Aid int `json:"aid"` + FileName string `json:"file_name"` + Cid string `json:"cid"` + IsVideo int `json:"is_video"` + } `json:"data"` +} diff --git a/drivers/115/util.go b/drivers/115/util.go index 35d1fbda759..79d869178cb 100644 --- a/drivers/115/util.go +++ b/drivers/115/util.go @@ -2,51 +2,72 @@ package _115 import ( "bytes" + "context" + "crypto/md5" "crypto/tls" + "encoding/hex" "encoding/json" "fmt" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/aliyun/aliyun-oss-go-sdk/oss" - "github.com/orzogc/fake115uploader/cipher" "io" "net/url" - "path/filepath" "strconv" "strings" "sync" + "sync/atomic" "time" - "github.com/SheltonZhu/115driver/pkg/driver" - driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + + cipher "github.com/SheltonZhu/115driver/pkg/crypto/ec115" + driver115 "github.com/SheltonZhu/115driver/pkg/driver" "github.com/pkg/errors" ) -var UserAgent = driver.UA115Desktop +type fileInfoWithThumb struct { + driver115.FileInfo + ThumbURL string `json:"u"` +} + +type fileListRespWithThumb struct { + driver115.BasicResp + CategoryID driver115.IntString `json:"cid"` + Count int `json:"count"` + Offset int `json:"offset"` + Files []fileInfoWithThumb `json:"data"` +} + +type getFileInfoResponseWithThumb struct { + driver115.BasicResp + Files []*fileInfoWithThumb `json:"data"` +} +// var UserAgent = driver115.UA115Browser func (d *Pan115) login() error { var err error - opts := []driver.Option{ - driver.UA(UserAgent), - func(c *driver.Pan115Client) { + opts := []driver115.Option{ + driver115.UA(d.getUA()), + func(c *driver115.Pan115Client) { c.Client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) }, } - d.client = driver.New(opts...) - cr := &driver.Credential{} - if d.Addition.QRCodeToken != "" { - s := &driver.QRCodeSession{ - UID: d.Addition.QRCodeToken, + d.client = driver115.New(opts...) + cr := &driver115.Credential{} + if d.QRCodeToken != "" { + s := &driver115.QRCodeSession{ + UID: d.QRCodeToken, } - if cr, err = d.client.QRCodeLogin(s); err != nil { + if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { return errors.Wrap(err, "failed to login by qrcode") } - d.Addition.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s", cr.UID, cr.CID, cr.SEID) - d.Addition.QRCodeToken = "" - } else if d.Addition.Cookie != "" { - if err = cr.FromCookie(d.Addition.Cookie); err != nil { + d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s;KID=%s", cr.UID, cr.CID, cr.SEID, cr.KID) + d.QRCodeToken = "" + } else if d.Cookie != "" { + if err = cr.FromCookie(d.Cookie); err != nil { return errors.Wrap(err, "failed to login by cookies") } d.client.ImportCredential(cr) @@ -59,21 +80,123 @@ func (d *Pan115) login() error { func (d *Pan115) getFiles(fileId string) ([]FileObj, error) { res := make([]FileObj, 0) if d.PageSize <= 0 { - d.PageSize = driver.FileListLimit + d.PageSize = driver115.FileListLimit + } + limit := d.PageSize + if limit > driver115.MaxDirPageLimit { + limit = driver115.MaxDirPageLimit + } + + opts := driver115.DefaultListOptions() + driver115.WithMultiUrls()(opts) + if len(opts.ApiURLs) == 0 { + opts.ApiURLs = []string{driver115.ApiFileList} + } + + offset := int64(0) + for i := 0; ; i++ { + result, err := d.getFilesPageWithThumb(fileId, opts.ApiURLs[i%len(opts.ApiURLs)], limit, offset) + if err != nil { + return nil, err + } + for _, fileInfo := range result.Files { + res = append(res, fileObjFromInfo(&fileInfo)) + } + offset = int64(result.Offset) + limit + if offset >= int64(result.Count) { + break + } } - files, err := d.client.ListWithLimit(fileId, d.PageSize) + + return res, nil +} + +func (d *Pan115) getNewFile(fileId string) (*FileObj, error) { + fileInfo, err := d.getFileInfoWithThumb("file_id", fileId) if err != nil { return nil, err } - for _, file := range *files { - res = append(res, FileObj{file}) + file := fileObjFromInfo(fileInfo) + return &file, nil +} + +func (d *Pan115) getNewFileByPickCode(pickCode string) (*FileObj, error) { + fileInfo, err := d.getFileInfoWithThumb("pick_code", pickCode) + if err != nil { + return nil, err } - return res, nil + file := fileObjFromInfo(fileInfo) + return &file, nil } -const ( - appVer = "2.0.3.6" -) +func (d *Pan115) getUA() string { + return fmt.Sprintf("Mozilla/5.0 115Browser/%s", appVer) +} + +func fileObjFromInfo(fileInfo *fileInfoWithThumb) FileObj { + file := &driver115.File{} + file.From(&fileInfo.FileInfo) + return FileObj{ + File: *file, + ThumbURL: fileInfo.ThumbURL, + } +} + +func (d *Pan115) getFileInfoWithThumb(queryKey, queryVal string) (*fileInfoWithThumb, error) { + result := getFileInfoResponseWithThumb{} + req := d.client.NewRequest(). + SetQueryParam(queryKey, queryVal). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + resp, err := req.Get(driver115.ApiFileInfo) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + if len(result.Files) == 0 { + return nil, errors.New("not get file info") + } + return result.Files[0], nil +} + +func (d *Pan115) getFilesPageWithThumb(dirID, apiURL string, limit, offset int64) (*fileListRespWithThumb, error) { + if dirID == "" { + dirID = "0" + } + result := fileListRespWithThumb{} + params := map[string]string{ + "aid": "1", + "cid": dirID, + "o": driver115.FileOrderByTime, + "asc": "1", + "offset": strconv.FormatInt(offset, 10), + "show_dir": "1", + "limit": strconv.FormatInt(limit, 10), + "snap": "0", + "natsort": "0", + "record_open_time": "1", + "format": "json", + "fc_mix": "0", + } + req := d.client.NewRequest(). + ForceContentType("application/json;charset=UTF-8"). + SetQueryParams(params). + SetResult(&result) + resp, err := req.Get(apiURL) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + if dirID != string(result.CategoryID) { + return nil, driver115.ErrUnexpected + } + return &result, nil +} + +func (c *Pan115) GenerateToken(fileID, preID, timeStamp, fileSize, signKey, signVal string) string { + userID := strconv.FormatInt(c.client.UserID, 10) + userIDMd5 := md5.Sum([]byte(userID)) + tokenMd5 := md5.Sum([]byte(md5Salt + fileID + fileSize + signKey + signVal + userID + timeStamp + hex.EncodeToString(userIDMd5[:]) + appVer)) + return hex.EncodeToString(tokenMd5[:]) +} func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID string, stream model.FileStreamer) (*driver115.UploadInitResp, error) { var ( @@ -104,7 +227,7 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri signKey, signVal := "", "" for retry := true; retry; { - t := driver115.Now() + t := driver115.NowMilli() if encodedToken, err = ecdhCipher.EncodeToken(t.ToInt64()); err != nil { return nil, err @@ -115,7 +238,7 @@ func (d *Pan115) rapidUpload(fileSize int64, fileName, dirID, preID, fileID stri } form.Set("t", t.String()) - form.Set("token", d.client.GenerateToken(fileID, preID, t.String(), fileSizeStr, signKey, signVal)) + form.Set("token", d.GenerateToken(fileID, preID, t.String(), fileSizeStr, signKey, signVal)) if signKey != "" && signVal != "" { form.Set("sign_key", signKey) form.Set("sign_val", signVal) @@ -168,6 +291,9 @@ func UploadDigestRange(stream model.FileStreamer, rangeSpec string) (result stri length := end - start + 1 reader, err := stream.RangeRead(http_range.Range{Start: start, Length: length}) + if err != nil { + return "", err + } hashStr, err := utils.HashReader(utils.SHA1, reader) if err != nil { return "", err @@ -176,8 +302,43 @@ func UploadDigestRange(stream model.FileStreamer, rangeSpec string) (result stri return } +// UploadByOSS use aliyun sdk to upload +func (c *Pan115) UploadByOSS(ctx context.Context, params *driver115.UploadOSSParams, s model.FileStreamer, dirID string, up driver.UpdateProgress) (*UploadResult, error) { + ossToken, err := c.client.GetOSSToken() + if err != nil { + return nil, err + } + ossClient, err := oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret) + if err != nil { + return nil, err + } + bucket, err := ossClient.Bucket(params.Bucket) + if err != nil { + return nil, err + } + + var bodyBytes []byte + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + if err = bucket.PutObject(params.Object, r, append( + driver115.OssOption(params, ossToken), + oss.CallbackResult(&bodyBytes), + )...); err != nil { + return nil, err + } + + var uploadResult UploadResult + if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { + return nil, err + } + return &uploadResult, uploadResult.Err(string(bodyBytes)) +} + // UploadByMultipart upload by mutipart blocks -func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize int64, stream model.FileStreamer, dirID string, opts ...driver115.UploadMultipartOption) error { +func (d *Pan115) UploadByMultipart(ctx context.Context, params *driver115.UploadOSSParams, fileSize int64, s model.FileStreamer, + dirID string, up driver.UpdateProgress, opts ...driver115.UploadMultipartOption) (*UploadResult, error) { var ( chunks []oss.FileChunk parts []oss.UploadPart @@ -185,12 +346,13 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i ossClient *oss.Client bucket *oss.Bucket ossToken *driver115.UploadOSSTokenResp + bodyBytes []byte err error ) - tmpF, err := stream.CacheFullInTempFile() + tmpF, err := s.CacheFullInTempFile() if err != nil { - return err + return nil, err } options := driver115.DefalutUploadMultipartOptions() @@ -199,17 +361,19 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i f(options) } } + // oss 启用Sequential必须按顺序上传 + options.ThreadsNum = 1 if ossToken, err = d.client.GetOSSToken(); err != nil { - return err + return nil, err } - if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret); err != nil { - return err + if ossClient, err = oss.New(driver115.OSSEndpoint, ossToken.AccessKeyID, ossToken.AccessKeySecret, oss.EnableMD5(true), oss.EnableCRC(true)); err != nil { + return nil, err } if bucket, err = ossClient.Bucket(params.Bucket); err != nil { - return err + return nil, err } // ossToken一小时后就会失效,所以每50分钟重新获取一次 @@ -219,14 +383,15 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i timeout := time.NewTimer(options.Timeout) if chunks, err = SplitFile(fileSize); err != nil { - return err + return nil, err } if imur, err = bucket.InitiateMultipartUpload(params.Object, oss.SetHeader(driver115.OssSecurityTokenHeaderName, ossToken.SecurityToken), oss.UserAgentHeader(driver115.OSSUserAgent), + oss.EnableSha1(), oss.Sequential(), ); err != nil { - return err + return nil, err } wg := sync.WaitGroup{} @@ -244,37 +409,41 @@ func (d *Pan115) UploadByMultipart(params *driver115.UploadOSSParams, fileSize i quit <- struct{}{} }() + completedNum := atomic.Int32{} // consumers for i := 0; i < options.ThreadsNum; i++ { go func(threadId int) { defer func() { if r := recover(); r != nil { - errCh <- fmt.Errorf("Recovered in %v", r) + errCh <- fmt.Errorf("recovered in %v", r) } }() for chunk := range chunksCh { var part oss.UploadPart // 出现错误就继续尝试,共尝试3次 for retry := 0; retry < 3; retry++ { select { + case <-ctx.Done(): + break case <-ticker.C: if ossToken, err = d.client.GetOSSToken(); err != nil { // 到时重新获取ossToken errCh <- errors.Wrap(err, "刷新token时出现错误") } default: } - buf := make([]byte, chunk.Size) if _, err = tmpF.ReadAt(buf, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { continue } - - b := bytes.NewBuffer(buf) - if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { + if part, err = bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, bytes.NewReader(buf)), + chunk.Size, chunk.Number, driver115.OssOption(params, ossToken)...); err == nil { break } } if err != nil { - errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误:%v", stream.GetName(), chunk.Number, err)) + errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误:%v", s.GetName(), chunk.Number, err)) + } else { + num := completedNum.Add(1) + up(float64(num) * 100.0 / float64(len(chunks))) } UploadedPartsCh <- part } @@ -293,51 +462,38 @@ LOOP: case <-ticker.C: // 到时重新获取ossToken if ossToken, err = d.client.GetOSSToken(); err != nil { - return err + return nil, err } case <-quit: break LOOP case <-errCh: - return err + return nil, err case <-timeout.C: - return fmt.Errorf("time out") + return nil, fmt.Errorf("time out") } } - // EOF错误是xml的Unmarshal导致的,响应其实是json格式,所以实际上上传是成功的 - if _, err = bucket.CompleteMultipartUpload(imur, parts, driver115.OssOption(params, ossToken)...); err != nil && !errors.Is(err, io.EOF) { - // 当文件名含有 &< 这两个字符之一时响应的xml解析会出现错误,实际上上传是成功的 - if filename := filepath.Base(stream.GetName()); !strings.ContainsAny(filename, "&<") { - return err - } + // 不知道啥原因,oss那边分片上传不计算sha1,导致115服务器校验错误 + // params.Callback.Callback = strings.ReplaceAll(params.Callback.Callback, "${sha1}", params.SHA1) + if _, err := bucket.CompleteMultipartUpload(imur, parts, append( + driver115.OssOption(params, ossToken), + oss.CallbackResult(&bodyBytes), + )...); err != nil { + return nil, err + } + + var uploadResult UploadResult + if err = json.Unmarshal(bodyBytes, &uploadResult); err != nil { + return nil, err } - return d.checkUploadStatus(dirID, params.SHA1) + return &uploadResult, uploadResult.Err(string(bodyBytes)) } + func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { for _, chunk := range chunks { ch <- chunk } } -func (d *Pan115) checkUploadStatus(dirID, sha1 string) error { - // 验证上传是否成功 - req := d.client.NewRequest().ForceContentType("application/json;charset=UTF-8") - opts := []driver115.GetFileOptions{ - driver115.WithOrder(driver115.FileOrderByTime), - driver115.WithShowDirEnable(false), - driver115.WithAsc(false), - driver115.WithLimit(500), - } - fResp, err := driver115.GetFiles(req, dirID, opts...) - if err != nil { - return err - } - for _, fileInfo := range fResp.Files { - if fileInfo.Sha1 == sha1 { - return nil - } - } - return driver115.ErrUploadFailed -} func SplitFile(fileSize int64) (chunks []oss.FileChunk, err error) { for i := int64(1); i < 10; i++ { @@ -374,8 +530,8 @@ func SplitFileByPartNum(fileSize int64, chunkNum int) ([]oss.FileChunk, error) { } var chunks []oss.FileChunk - var chunk = oss.FileChunk{} - var chunkN = (int64)(chunkNum) + chunk := oss.FileChunk{} + chunkN := (int64)(chunkNum) for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * (fileSize / chunkN) @@ -397,13 +553,13 @@ func SplitFileByPartSize(fileSize int64, chunkSize int64) ([]oss.FileChunk, erro return nil, errors.New("chunkSize invalid") } - var chunkN = fileSize / chunkSize + chunkN := fileSize / chunkSize if chunkN >= 10000 { return nil, errors.New("Too many parts, please increase part size") } var chunks []oss.FileChunk - var chunk = oss.FileChunk{} + chunk := oss.FileChunk{} for i := int64(0); i < chunkN; i++ { chunk.Number = int(i + 1) chunk.Offset = i * chunkSize diff --git a/drivers/115_open/driver.go b/drivers/115_open/driver.go new file mode 100644 index 00000000000..6121d3b2ed8 --- /dev/null +++ b/drivers/115_open/driver.go @@ -0,0 +1,335 @@ +package _115_open + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/cmd/flags" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + sdk "github.com/xhofe/115-sdk-go" + "golang.org/x/time/rate" +) + +type Open115 struct { + model.Storage + Addition + client *sdk.Client + limiter *rate.Limiter +} + +func (d *Open115) Config() driver.Config { + return config +} + +func (d *Open115) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Open115) Init(ctx context.Context) error { + d.client = sdk.New(sdk.WithRefreshToken(d.Addition.RefreshToken), + sdk.WithAccessToken(d.Addition.AccessToken), + sdk.WithOnRefreshToken(func(s1, s2 string) { + d.Addition.AccessToken = s1 + d.Addition.RefreshToken = s2 + op.MustSaveDriverStorage(d) + })) + if flags.Debug || flags.Dev { + d.client.SetDebug(true) + } + _, err := d.client.UserInfo(ctx) + if err != nil { + return err + } + if d.Addition.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.Addition.LimitRate), 1) + } + return nil +} + +func (d *Open115) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + +func (d *Open115) Drop(ctx context.Context) error { + return nil +} + +func (d *Open115) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var res []model.Obj + pageSize := int64(200) + offset := int64(0) + for { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + resp, err := d.client.GetFiles(ctx, &sdk.GetFilesReq{ + CID: dir.GetID(), + Limit: pageSize, + Offset: offset, + ASC: d.Addition.OrderDirection == "asc", + O: d.Addition.OrderBy, + // Cur: 1, + ShowDir: true, + }) + if err != nil { + return nil, err + } + res = append(res, utils.MustSliceConvert(resp.Data, func(src sdk.GetFilesResp_File) model.Obj { + obj := Obj(src) + return &obj + })...) + if len(res) >= int(resp.Count) { + break + } + offset += pageSize + } + return res, nil +} + +func (d *Open115) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + var ua string + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + obj, ok := file.(*Obj) + if !ok { + return nil, fmt.Errorf("can't convert obj") + } + pc := obj.Pc + resp, err := d.client.DownURL(ctx, pc, ua) + if err != nil { + return nil, err + } + u, ok := resp[obj.GetID()] + if !ok { + return nil, fmt.Errorf("can't get link") + } + return &model.Link{ + URL: u.URL.URL, + Header: http.Header{ + "User-Agent": []string{ua}, + }, + }, nil +} + +func (d *Open115) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + resp, err := d.client.Mkdir(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + return &Obj{ + Fid: resp.FileID, + Pid: parentDir.GetID(), + Fn: dirName, + Fc: "0", + Upt: time.Now().Unix(), + Uet: time.Now().Unix(), + UpPt: time.Now().Unix(), + }, nil +} + +func (d *Open115) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + _, err := d.client.Move(ctx, &sdk.MoveReq{ + FileIDs: srcObj.GetID(), + ToCid: dstDir.GetID(), + }) + if err != nil { + return nil, err + } + return srcObj, nil +} + +func (d *Open115) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + _, err := d.client.UpdateFile(ctx, &sdk.UpdateFileReq{ + FileID: srcObj.GetID(), + FileNma: newName, + }) + if err != nil { + return nil, err + } + obj, ok := srcObj.(*Obj) + if ok { + obj.Fn = newName + } + return srcObj, nil +} + +func (d *Open115) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + _, err := d.client.Copy(ctx, &sdk.CopyReq{ + PID: dstDir.GetID(), + FileID: srcObj.GetID(), + NoDupli: "1", + }) + if err != nil { + return nil, err + } + return srcObj, nil +} + +func (d *Open115) Remove(ctx context.Context, obj model.Obj) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + _obj, ok := obj.(*Obj) + if !ok { + return fmt.Errorf("can't convert obj") + } + _, err := d.client.DelFile(ctx, &sdk.DelFileReq{ + FileIDs: _obj.GetID(), + ParentID: _obj.Pid, + }) + if err != nil { + return err + } + return nil +} + +func (d *Open115) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if err := d.WaitLimit(ctx); err != nil { + return err + } + tempF, err := file.CacheFullInTempFile() + if err != nil { + return err + } + // cal full sha1 + sha1, err := utils.HashReader(utils.SHA1, tempF) + if err != nil { + return err + } + _, err = tempF.Seek(0, io.SeekStart) + if err != nil { + return err + } + // pre 128k sha1 + sha1128k, err := utils.HashReader(utils.SHA1, io.LimitReader(tempF, 128*1024)) + if err != nil { + return err + } + _, err = tempF.Seek(0, io.SeekStart) + if err != nil { + return err + } + // 1. Init + resp, err := d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + }) + if err != nil { + return err + } + if resp.Status == 2 { + return nil + } + // 2. two way verify + if utils.SliceContains([]int{6, 7, 8}, resp.Status) { + signCheck := strings.Split(resp.SignCheck, "-") //"sign_check": "2392148-2392298" 取2392148-2392298之间的内容(包含2392148、2392298)的sha1 + start, err := strconv.ParseInt(signCheck[0], 10, 64) + if err != nil { + return err + } + end, err := strconv.ParseInt(signCheck[1], 10, 64) + if err != nil { + return err + } + _, err = tempF.Seek(start, io.SeekStart) + if err != nil { + return err + } + signVal, err := utils.HashReader(utils.SHA1, io.LimitReader(tempF, end-start+1)) + if err != nil { + return err + } + _, err = tempF.Seek(0, io.SeekStart) + if err != nil { + return err + } + resp, err = d.client.UploadInit(ctx, &sdk.UploadInitReq{ + FileName: file.GetName(), + FileSize: file.GetSize(), + Target: dstDir.GetID(), + FileID: strings.ToUpper(sha1), + PreID: strings.ToUpper(sha1128k), + SignKey: resp.SignKey, + SignVal: strings.ToUpper(signVal), + }) + if err != nil { + return err + } + if resp.Status == 2 { + return nil + } + } + // 3. get upload token + tokenResp, err := d.client.UploadGetToken(ctx) + if err != nil { + return err + } + // 4. upload + err = d.multpartUpload(ctx, tempF, file, up, tokenResp, resp) + if err != nil { + return err + } + return nil +} + +// func (d *Open115) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { +// // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional +// return nil, errs.NotImplement +// } + +// func (d *Open115) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { +// // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional +// return nil, errs.NotImplement +// } + +// func (d *Open115) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { +// // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional +// return nil, errs.NotImplement +// } + +// func (d *Open115) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { +// // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional +// // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir +// // return errs.NotImplement to use an internal archive tool +// return nil, errs.NotImplement +// } + +//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Open115)(nil) diff --git a/drivers/115_open/meta.go b/drivers/115_open/meta.go new file mode 100644 index 00000000000..66b956c0a9d --- /dev/null +++ b/drivers/115_open/meta.go @@ -0,0 +1,37 @@ +package _115_open + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + RefreshToken string `json:"refresh_token" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"file_name,file_size,user_utime,file_type"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc"` + LimitRate float64 `json:"limit_rate" type:"float" default:"1" help:"limit all api request rate ([limit]r/1s)"` + AccessToken string +} + +var config = driver.Config{ + Name: "115 Open", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Open115{} + }) +} diff --git a/drivers/115_open/types.go b/drivers/115_open/types.go new file mode 100644 index 00000000000..491a368edf2 --- /dev/null +++ b/drivers/115_open/types.go @@ -0,0 +1,59 @@ +package _115_open + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + sdk "github.com/xhofe/115-sdk-go" +) + +type Obj sdk.GetFilesResp_File + +// Thumb implements model.Thumb. +func (o *Obj) Thumb() string { + return o.Thumbnail +} + +// CreateTime implements model.Obj. +func (o *Obj) CreateTime() time.Time { + return time.Unix(o.UpPt, 0) +} + +// GetHash implements model.Obj. +func (o *Obj) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.SHA1, o.Sha1) +} + +// GetID implements model.Obj. +func (o *Obj) GetID() string { + return o.Fid +} + +// GetName implements model.Obj. +func (o *Obj) GetName() string { + return o.Fn +} + +// GetPath implements model.Obj. +func (o *Obj) GetPath() string { + return "" +} + +// GetSize implements model.Obj. +func (o *Obj) GetSize() int64 { + return o.FS +} + +// IsDir implements model.Obj. +func (o *Obj) IsDir() bool { + return o.Fc == "0" +} + +// ModTime implements model.Obj. +func (o *Obj) ModTime() time.Time { + return time.Unix(o.Upt, 0) +} + +var _ model.Obj = (*Obj)(nil) +var _ model.Thumb = (*Obj)(nil) diff --git a/drivers/115_open/upload.go b/drivers/115_open/upload.go new file mode 100644 index 00000000000..282582eff12 --- /dev/null +++ b/drivers/115_open/upload.go @@ -0,0 +1,140 @@ +package _115_open + +import ( + "context" + "encoding/base64" + "io" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/avast/retry-go" + sdk "github.com/xhofe/115-sdk-go" +) + +func calPartSize(fileSize int64) int64 { + var partSize int64 = 20 * utils.MB + if fileSize > partSize { + if fileSize > 1*utils.TB { // file Size over 1TB + partSize = 5 * utils.GB // file part size 5GB + } else if fileSize > 768*utils.GB { // over 768GB + partSize = 109951163 // ≈ 104.8576MB, split 1TB into 10,000 part + } else if fileSize > 512*utils.GB { // over 512GB + partSize = 82463373 // ≈ 78.6432MB + } else if fileSize > 384*utils.GB { // over 384GB + partSize = 54975582 // ≈ 52.4288MB + } else if fileSize > 256*utils.GB { // over 256GB + partSize = 41231687 // ≈ 39.3216MB + } else if fileSize > 128*utils.GB { // over 128GB + partSize = 27487791 // ≈ 26.2144MB + } + } + return partSize +} + +func (d *Open115) singleUpload(ctx context.Context, tempF model.File, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { + ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) + if err != nil { + return err + } + bucket, err := ossClient.Bucket(initResp.Bucket) + if err != nil { + return err + } + + err = bucket.PutObject(initResp.Object, tempF, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), + ) + + return err +} + +// type CallbackResult struct { +// State bool `json:"state"` +// Code int `json:"code"` +// Message string `json:"message"` +// Data struct { +// PickCode string `json:"pick_code"` +// FileName string `json:"file_name"` +// FileSize int64 `json:"file_size"` +// FileID string `json:"file_id"` +// ThumbURL string `json:"thumb_url"` +// Sha1 string `json:"sha1"` +// Aid int `json:"aid"` +// Cid string `json:"cid"` +// } `json:"data"` +// } + +func (d *Open115) multpartUpload(ctx context.Context, tempF model.File, stream model.FileStreamer, up driver.UpdateProgress, tokenResp *sdk.UploadGetTokenResp, initResp *sdk.UploadInitResp) error { + fileSize := stream.GetSize() + chunkSize := calPartSize(fileSize) + + ossClient, err := oss.New(tokenResp.Endpoint, tokenResp.AccessKeyId, tokenResp.AccessKeySecret, oss.SecurityToken(tokenResp.SecurityToken)) + if err != nil { + return err + } + bucket, err := ossClient.Bucket(initResp.Bucket) + if err != nil { + return err + } + + imur, err := bucket.InitiateMultipartUpload(initResp.Object, oss.Sequential()) + if err != nil { + return err + } + + partNum := (stream.GetSize() + chunkSize - 1) / chunkSize + parts := make([]oss.UploadPart, partNum) + offset := int64(0) + for i := int64(1); i <= partNum; i++ { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + + partSize := chunkSize + if i == partNum { + partSize = fileSize - (i-1)*chunkSize + } + rd := utils.NewMultiReadable(io.LimitReader(stream, partSize)) + err = retry.Do(func() error { + _ = rd.Reset() + rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd) + part, err := bucket.UploadPart(imur, rateLimitedRd, partSize, int(i)) + if err != nil { + return err + } + parts[i-1] = part + return nil + }, + retry.Attempts(3), + retry.DelayType(retry.BackOffDelay), + retry.Delay(time.Second)) + if err != nil { + return err + } + + if i == partNum { + offset = fileSize + } else { + offset += partSize + } + up(float64(offset) / float64(fileSize)) + } + + // callbackRespBytes := make([]byte, 1024) + _, err = bucket.CompleteMultipartUpload( + imur, + parts, + oss.Callback(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.Callback))), + oss.CallbackVar(base64.StdEncoding.EncodeToString([]byte(initResp.Callback.Value.CallbackVar))), + // oss.CallbackResult(&callbackRespBytes), + ) + if err != nil { + return err + } + + return nil +} diff --git a/drivers/115_open/util.go b/drivers/115_open/util.go new file mode 100644 index 00000000000..ee0216597e4 --- /dev/null +++ b/drivers/115_open/util.go @@ -0,0 +1,3 @@ +package _115_open + +// do others that not defined in Driver interface diff --git a/drivers/115_share/driver.go b/drivers/115_share/driver.go new file mode 100644 index 00000000000..322b64afd45 --- /dev/null +++ b/drivers/115_share/driver.go @@ -0,0 +1,118 @@ +package _115_share + +import ( + "context" + + driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "golang.org/x/time/rate" +) + +type Pan115Share struct { + model.Storage + Addition + client *driver115.Pan115Client + limiter *rate.Limiter +} + +func (d *Pan115Share) Config() driver.Config { + return config +} + +func (d *Pan115Share) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Pan115Share) Init(ctx context.Context) error { + if d.LimitRate > 0 { + d.limiter = rate.NewLimiter(rate.Limit(d.LimitRate), 1) + } + + return d.login() +} + +func (d *Pan115Share) WaitLimit(ctx context.Context) error { + if d.limiter != nil { + return d.limiter.Wait(ctx) + } + return nil +} + +func (d *Pan115Share) Drop(ctx context.Context) error { + return nil +} + +func (d *Pan115Share) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + + ua := base.UserAgent + files := make([]shareFile, 0) + fileResp, err := d.getShareSnapWithUA(ua, dir.GetID(), driver115.QueryLimit(int(d.PageSize))) + if err != nil { + return nil, err + } + files = append(files, fileResp.Data.List...) + total := fileResp.Data.Count + count := len(fileResp.Data.List) + for total > count { + fileResp, err := d.getShareSnapWithUA(ua, dir.GetID(), driver115.QueryLimit(int(d.PageSize)), driver115.QueryOffset(count)) + if err != nil { + return nil, err + } + files = append(files, fileResp.Data.List...) + count += len(fileResp.Data.List) + } + + return utils.SliceConvert(files, transFunc) +} + +func (d *Pan115Share) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.WaitLimit(ctx); err != nil { + return nil, err + } + ua := "" + if args.Header != nil { + ua = args.Header.Get("User-Agent") + } + if ua == "" { + ua = base.UserAgent + } + downloadInfo, err := d.downloadByShareCodeWithUA(ua, file.GetID()) + if err != nil { + return nil, err + } + + return &model.Link{URL: downloadInfo.URL.URL}, nil +} + +func (d *Pan115Share) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return errs.NotSupport +} + +func (d *Pan115Share) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *Pan115Share) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return errs.NotSupport +} + +func (d *Pan115Share) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *Pan115Share) Remove(ctx context.Context, obj model.Obj) error { + return errs.NotSupport +} + +func (d *Pan115Share) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + return errs.NotSupport +} + +var _ driver.Driver = (*Pan115Share)(nil) diff --git a/drivers/115_share/meta.go b/drivers/115_share/meta.go new file mode 100644 index 00000000000..92f8bf0ff08 --- /dev/null +++ b/drivers/115_share/meta.go @@ -0,0 +1,34 @@ +package _115_share + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + Cookie string `json:"cookie" type:"text" help:"one of QR code token and cookie required"` + QRCodeToken string `json:"qrcode_token" type:"text" help:"one of QR code token and cookie required"` + QRCodeSource string `json:"qrcode_source" type:"select" options:"web,android,ios,tv,alipaymini,wechatmini,qandroid" default:"linux" help:"select the QR code device, default linux"` + PageSize int64 `json:"page_size" type:"number" default:"1000" help:"list api per page size of 115 driver"` + LimitRate float64 `json:"limit_rate" type:"float" default:"2" help:"limit all api request rate (1r/[limit_rate]s)"` + ShareCode string `json:"share_code" type:"text" required:"true" help:"share code of 115 share link"` + ReceiveCode string `json:"receive_code" type:"text" required:"true" help:"receive code of 115 share link"` + driver.RootID +} + +var config = driver.Config{ + Name: "115 Share", + DefaultRoot: "0", + // OnlyProxy: true, + // OnlyLocal: true, + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, + NoUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Pan115Share{} + }) +} diff --git a/drivers/115_share/utils.go b/drivers/115_share/utils.go new file mode 100644 index 00000000000..e36a5ef8ccb --- /dev/null +++ b/drivers/115_share/utils.go @@ -0,0 +1,204 @@ +package _115_share + +import ( + "fmt" + "strconv" + "time" + + driver115 "github.com/SheltonZhu/115driver/pkg/driver" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" +) + +var _ model.Obj = (*FileObj)(nil) + +type FileObj struct { + Size int64 + Sha1 string + Utm time.Time + FileName string + isDir bool + FileID string + ThumbURL string +} + +func (f *FileObj) CreateTime() time.Time { + return f.Utm +} + +func (f *FileObj) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.SHA1, f.Sha1) +} + +func (f *FileObj) GetSize() int64 { + return f.Size +} + +func (f *FileObj) GetName() string { + return f.FileName +} + +func (f *FileObj) ModTime() time.Time { + return f.Utm +} + +func (f *FileObj) IsDir() bool { + return f.isDir +} + +func (f *FileObj) GetID() string { + return f.FileID +} + +func (f *FileObj) GetPath() string { + return "" +} + +func (f *FileObj) Thumb() string { + return f.ThumbURL +} + +type shareFile struct { + FileID string `json:"fid"` + UID int `json:"uid"` + CategoryID driver115.IntString `json:"cid"` + FileName string `json:"n"` + Type string `json:"ico"` + Sha1 string `json:"sha"` + Size driver115.StringInt64 `json:"s"` + Labels []*driver115.LabelInfo `json:"fl"` + UpdateTime string `json:"t"` + IsFile int `json:"fc"` + ParentID string `json:"pid"` + ThumbURL string `json:"u"` +} + +type shareSnapResp struct { + driver115.BasicResp + Data struct { + Count int `json:"count"` + List []shareFile `json:"list"` + } `json:"data"` +} + +type downloadShareResp struct { + driver115.BasicResp + Data driver115.SharedDownloadInfo `json:"data"` +} + +func transFunc(sf shareFile) (model.Obj, error) { + timeInt, err := strconv.ParseInt(sf.UpdateTime, 10, 64) + if err != nil { + return nil, err + } + var ( + utm = time.Unix(timeInt, 0) + isDir = (sf.IsFile == 0) + fileID = string(sf.FileID) + ) + if isDir { + fileID = string(sf.CategoryID) + } + return &FileObj{ + Size: int64(sf.Size), + Sha1: sf.Sha1, + Utm: utm, + FileName: string(sf.FileName), + isDir: isDir, + FileID: fileID, + ThumbURL: sf.ThumbURL, + }, nil +} + +func buildShareReferer(shareCode, receiveCode string) string { + return fmt.Sprintf("https://115cdn.com/s/%s?password=%s&", shareCode, receiveCode) +} + +func (d *Pan115Share) getShareSnapWithUA(ua, dirID string, queries ...driver115.Query) (*shareSnapResp, error) { + result := shareSnapResp{} + query := map[string]string{ + "share_code": d.ShareCode, + "receive_code": d.ReceiveCode, + "cid": dirID, + "limit": "20", + "asc": "0", + "offset": "0", + "format": "json", + } + for _, q := range queries { + q(&query) + } + + req := d.client.NewRequest(). + SetQueryParams(query). + SetHeader("referer", buildShareReferer(d.ShareCode, d.ReceiveCode)). + ForceContentType("application/json;charset=UTF-8"). + SetResult(&result) + if ua != "" { + req = req.SetHeader("User-Agent", ua) + } + + resp, err := req.Get(driver115.ApiShareSnap) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + return &result, nil +} + +func (d *Pan115Share) downloadByShareCodeWithUA(ua, fileID string) (*driver115.SharedDownloadInfo, error) { + result := downloadShareResp{} + params := map[string]string{ + "share_code": d.ShareCode, + "receive_code": d.ReceiveCode, + "file_id": fileID, + "dl": "1", + } + + req := d.client.NewRequest(). + SetQueryParams(params). + ForceContentType("application/json"). + SetHeader("referer", buildShareReferer(d.ShareCode, d.ReceiveCode)). + SetResult(&result) + if ua != "" { + req = req.SetHeader("User-Agent", ua) + } + + resp, err := req.Get(driver115.ApiDownloadGetShareUrl) + if err := driver115.CheckErr(err, &result, resp); err != nil { + return nil, err + } + return &result.Data, nil +} + +func (d *Pan115Share) login() error { + var err error + opts := []driver115.Option{ + driver115.UA(base.UserAgent), + } + d.client = driver115.New(opts...) + if _, err = d.getShareSnapWithUA(base.UserAgent, ""); err != nil { + return errors.Wrap(err, "failed to get share snap") + } + cr := &driver115.Credential{} + if d.QRCodeToken != "" { + s := &driver115.QRCodeSession{ + UID: d.QRCodeToken, + } + if cr, err = d.client.QRCodeLoginWithApp(s, driver115.LoginApp(d.QRCodeSource)); err != nil { + return errors.Wrap(err, "failed to login by qrcode") + } + d.Cookie = fmt.Sprintf("UID=%s;CID=%s;SEID=%s;KID=%s", cr.UID, cr.CID, cr.SEID, cr.KID) + d.QRCodeToken = "" + } else if d.Cookie != "" { + if err = cr.FromCookie(d.Cookie); err != nil { + return errors.Wrap(err, "failed to login by cookies") + } + d.client.ImportCredential(cr) + } else { + return errors.New("missing cookie or qrcode account") + } + + return d.client.LoginCheck() +} diff --git a/drivers/123/driver.go b/drivers/123/driver.go index 6f7fec1bd43..cf221fee6d8 100644 --- a/drivers/123/driver.go +++ b/drivers/123/driver.go @@ -2,14 +2,22 @@ package _123 import ( "context" - "crypto/md5" "encoding/base64" - "encoding/hex" "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/time/rate" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" @@ -17,14 +25,13 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" - "io" - "net/http" - "net/url" ) type Pan123 struct { model.Storage Addition + apiRateLimit sync.Map + safeBoxUnlocked sync.Map } func (d *Pan123) Config() driver.Config { @@ -36,21 +43,38 @@ func (d *Pan123) GetAddition() driver.Additional { } func (d *Pan123) Init(ctx context.Context) error { - _, err := d.request(UserInfo, http.MethodGet, nil, nil) + _, err := d.Request(UserInfo, http.MethodGet, nil, nil) return err } func (d *Pan123) Drop(ctx context.Context) error { - _, _ = d.request(Logout, http.MethodPost, func(req *resty.Request) { + _, _ = d.Request(Logout, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{}) }, nil) return nil } func (d *Pan123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files, err := d.getFiles(dir.GetID()) + if f, ok := dir.(File); ok && f.IsLock { + if err := d.unlockSafeBox(f.FileId); err != nil { + return nil, err + } + } + files, err := d.getFiles(ctx, dir.GetID(), dir.GetName()) if err != nil { - return nil, err + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "safe box") || strings.Contains(err.Error(), "保险箱") { + if id, e := strconv.ParseInt(dir.GetID(), 10, 64); e == nil { + if e = d.unlockSafeBox(id); e == nil { + files, err = d.getFiles(ctx, dir.GetID(), dir.GetName()) + } else { + return nil, e + } + } + } + if err != nil { + return nil, err + } } return utils.SliceConvert(files, func(src File) (model.Obj, error) { return src, nil @@ -76,7 +100,8 @@ func (d *Pan123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) "size": f.Size, "type": f.Type, } - resp, err := d.request(DownloadInfo, http.MethodPost, func(req *resty.Request) { + resp, err := d.Request(DownloadInfo, http.MethodPost, func(req *resty.Request) { + req.SetBody(data).SetHeaders(headers) }, nil) if err != nil { @@ -129,7 +154,7 @@ func (d *Pan123) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin "size": 0, "type": 1, } - _, err := d.request(Mkdir, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(Mkdir, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err @@ -140,7 +165,7 @@ func (d *Pan123) Move(ctx context.Context, srcObj, dstDir model.Obj) error { "fileIdList": []base.Json{{"FileId": srcObj.GetID()}}, "parentFileId": dstDir.GetID(), } - _, err := d.request(Move, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(Move, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err @@ -152,7 +177,7 @@ func (d *Pan123) Rename(ctx context.Context, srcObj model.Obj, newName string) e "fileId": srcObj.GetID(), "fileName": newName, } - _, err := d.request(Rename, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(Rename, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err @@ -169,7 +194,7 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error { "operation": true, "fileTrashInfoList": []File{f}, } - _, err := d.request(Trash, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(Trash, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) return err @@ -178,36 +203,26 @@ func (d *Pan123) Remove(ctx context.Context, obj model.Obj) error { } } -func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - // const DEFAULT int64 = 10485760 - h := md5.New() - // need to calculate md5 of the full content - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - defer func() { - _ = tempFile.Close() - }() - if _, err = io.Copy(h, tempFile); err != nil { - return err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err +func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + etag := file.GetHash().GetHash(utils.MD5) + var err error + if len(etag) < utils.MD5.Width { + _, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } } - etag := hex.EncodeToString(h.Sum(nil)) data := base.Json{ "driveId": 0, "duplicate": 2, // 2->覆盖 1->重命名 0->默认 "etag": etag, - "fileName": stream.GetName(), + "fileName": file.GetName(), "parentFileId": dstDir.GetID(), - "size": stream.GetSize(), + "size": file.GetSize(), "type": 0, } var resp UploadResp - res, err := d.request(UploadRequest, http.MethodPost, func(req *resty.Request) { + res, err := d.Request(UploadRequest, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &resp) if err != nil { @@ -218,7 +233,7 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return nil } if resp.Data.AccessKeyId == "" || resp.Data.SecretAccessKey == "" || resp.Data.SessionToken == "" { - err = d.newUpload(ctx, &resp, stream, tempFile, up) + err = d.newUpload(ctx, &resp, file, up) return err } else { cfg := &aws.Config{ @@ -232,17 +247,23 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } uploader := s3manager.NewUploader(s) + if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) + } input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Key, - Body: tempFile, + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }), } _, err = uploader.UploadWithContext(ctx, input) + if err != nil { + return err + } } - if err != nil { - return err - } - _, err = d.request(UploadComplete, http.MethodPost, func(req *resty.Request) { + _, err = d.Request(UploadComplete, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "fileId": resp.Data.FileId, }).SetContext(ctx) @@ -250,4 +271,12 @@ func (d *Pan123) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return err } +func (d *Pan123) APIRateLimit(ctx context.Context, api string) error { + value, _ := d.apiRateLimit.LoadOrStore(api, + rate.NewLimiter(rate.Every(700*time.Millisecond), 1)) + limiter := value.(*rate.Limiter) + + return limiter.Wait(ctx) +} + var _ driver.Driver = (*Pan123)(nil) diff --git a/drivers/123/meta.go b/drivers/123/meta.go index 0c3c6a2dffb..6c5f013ad4a 100644 --- a/drivers/123/meta.go +++ b/drivers/123/meta.go @@ -6,17 +6,19 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + SafePassword string `json:"safe_password"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - AccessToken string + //OrderBy string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"` + //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + AccessToken string } var config = driver.Config{ Name: "123Pan", DefaultRoot: "0", + LocalSort: true, } func init() { diff --git a/drivers/123/types.go b/drivers/123/types.go index b79be12e201..962e5fbdac0 100644 --- a/drivers/123/types.go +++ b/drivers/123/types.go @@ -20,6 +20,7 @@ type File struct { Etag string `json:"Etag"` S3KeyFlag string `json:"S3KeyFlag"` DownloadUrl string `json:"DownloadUrl"` + IsLock bool `json:"IsLock"` } func (f File) CreateTime() time.Time { @@ -87,8 +88,9 @@ var _ model.Thumb = (*File)(nil) type Files struct { //BaseResp Data struct { - InfoList []File `json:"InfoList"` Next string `json:"Next"` + Total int `json:"Total"` + InfoList []File `json:"InfoList"` } `json:"data"` } diff --git a/drivers/123/upload.go b/drivers/123/upload.go index ae28d6aa519..b0482a9f4c9 100644 --- a/drivers/123/upload.go +++ b/drivers/123/upload.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "math" "net/http" "strconv" @@ -25,7 +24,7 @@ func (d *Pan123) getS3PreSignedUrls(ctx context.Context, upReq *UploadResp, star "StorageNode": upReq.Data.StorageNode, } var s3PreSignedUrls S3PreSignedURLs - _, err := d.request(S3PreSignedUrls, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(S3PreSignedUrls, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &s3PreSignedUrls) if err != nil { @@ -44,7 +43,7 @@ func (d *Pan123) getS3Auth(ctx context.Context, upReq *UploadResp, start, end in "uploadId": upReq.Data.UploadId, } var s3PreSignedUrls S3PreSignedURLs - _, err := d.request(S3Auth, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(S3Auth, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, &s3PreSignedUrls) if err != nil { @@ -63,21 +62,31 @@ func (d *Pan123) completeS3(ctx context.Context, upReq *UploadResp, file model.F "key": upReq.Data.Key, "uploadId": upReq.Data.UploadId, } - _, err := d.request(UploadCompleteV2, http.MethodPost, func(req *resty.Request) { + _, err := d.Request(UploadCompleteV2, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetContext(ctx) }, nil) return err } -func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, reader io.Reader, up driver.UpdateProgress) error { - chunkSize := int64(1024 * 1024 * 16) +func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.FileStreamer, up driver.UpdateProgress) error { + tmpF, err := file.CacheFullInTempFile() + if err != nil { + return err + } // fetch s3 pre signed urls - chunkCount := int(math.Ceil(float64(file.GetSize()) / float64(chunkSize))) + size := file.GetSize() + chunkSize := min(size, 16*utils.MB) + chunkCount := int(size / chunkSize) + lastChunkSize := size % chunkSize + if lastChunkSize > 0 { + chunkCount++ + } else { + lastChunkSize = chunkSize + } // only 1 batch is allowed - isMultipart := chunkCount > 1 batchSize := 1 getS3UploadUrl := d.getS3Auth - if isMultipart { + if chunkCount > 1 { batchSize = 10 getS3UploadUrl = d.getS3PreSignedUrls } @@ -86,10 +95,7 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi return ctx.Err() } start := i - end := i + batchSize - if end > chunkCount+1 { - end = chunkCount + 1 - } + end := min(i+batchSize, chunkCount+1) s3PreSignedUrls, err := getS3UploadUrl(ctx, upReq, start, end) if err != nil { return err @@ -101,25 +107,25 @@ func (d *Pan123) newUpload(ctx context.Context, upReq *UploadResp, file model.Fi } curSize := chunkSize if j == chunkCount { - curSize = file.GetSize() - (int64(chunkCount)-1)*chunkSize + curSize = lastChunkSize } - err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.LimitReader(reader, chunkSize), curSize, false, getS3UploadUrl) + err = d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, j, end, io.NewSectionReader(tmpF, chunkSize*int64(j-1), curSize), curSize, false, getS3UploadUrl) if err != nil { return err } - up(j * 100 / chunkCount) + up(float64(j) * 100 / float64(chunkCount)) } } // complete s3 upload return d.completeS3(ctx, upReq, file, chunkCount > 1) } -func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSignedUrls *S3PreSignedURLs, cur, end int, reader io.Reader, curSize int64, retry bool, getS3UploadUrl func(ctx context.Context, upReq *UploadResp, start int, end int) (*S3PreSignedURLs, error)) error { +func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSignedUrls *S3PreSignedURLs, cur, end int, reader *io.SectionReader, curSize int64, retry bool, getS3UploadUrl func(ctx context.Context, upReq *UploadResp, start int, end int) (*S3PreSignedURLs, error)) error { uploadUrl := s3PreSignedUrls.Data.PreSignedUrls[strconv.Itoa(cur)] if uploadUrl == "" { return fmt.Errorf("upload url is empty, s3PreSignedUrls: %+v", s3PreSignedUrls) } - req, err := http.NewRequest("PUT", uploadUrl, reader) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } @@ -142,6 +148,7 @@ func (d *Pan123) uploadS3Chunk(ctx context.Context, upReq *UploadResp, s3PreSign } s3PreSignedUrls.Data.PreSignedUrls = newS3PreSignedUrls.Data.PreSignedUrls // retry + reader.Seek(0, io.SeekStart) return d.uploadS3Chunk(ctx, upReq, s3PreSignedUrls, cur, end, reader, curSize, true, getS3UploadUrl) } if res.StatusCode != http.StatusOK { diff --git a/drivers/123/util.go b/drivers/123/util.go index e9eb63375d1..bca54b599f4 100644 --- a/drivers/123/util.go +++ b/drivers/123/util.go @@ -1,15 +1,23 @@ package _123 import ( + "context" "errors" "fmt" + "hash/crc32" + "math" + "math/rand" "net/http" + "net/url" "strconv" + "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" + log "github.com/sirupsen/logrus" ) // do others that not defined in Driver interface @@ -18,8 +26,9 @@ const ( Api = "https://www.123pan.com/api" AApi = "https://www.123pan.com/a/api" BApi = "https://www.123pan.com/b/api" - MainApi = Api - SignIn = MainApi + "/user/sign_in" + LoginApi = "https://login.123pan.com/api" + MainApi = BApi + SignIn = LoginApi + "/user/sign_in" Logout = MainApi + "/user/logout" UserInfo = MainApi + "/user/info" FileList = MainApi + "/file/list/new" @@ -34,9 +43,108 @@ const ( S3Auth = MainApi + "/file/s3_upload_object/auth" UploadCompleteV2 = MainApi + "/file/upload_complete/v2" S3Complete = MainApi + "/file/s3_complete_multipart_upload" + SafeBoxUnlock = MainApi + "/restful/goapi/v1/file/safe_box/auth/unlockbox" //AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) +func signPath(path string, os string, version string) (k string, v string) { + table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'} + random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64())) + now := time.Now().In(time.FixedZone("CST", 8*3600)) + timestamp := fmt.Sprint(now.Unix()) + nowStr := []byte(now.Format("200601021504")) + for i := 0; i < len(nowStr); i++ { + nowStr[i] = table[nowStr[i]-48] + } + timeSign := fmt.Sprint(crc32.ChecksumIEEE(nowStr)) + data := strings.Join([]string{timestamp, random, path, os, version, timeSign}, "|") + dataSign := fmt.Sprint(crc32.ChecksumIEEE([]byte(data))) + return timeSign, strings.Join([]string{timestamp, random, dataSign}, "-") +} + +func GetApi(rawUrl string) string { + u, _ := url.Parse(rawUrl) + query := u.Query() + query.Add(signPath(u.Path, "web", "3")) + u.RawQuery = query.Encode() + return u.String() +} + +//func GetApi(url string) string { +// vm := js.New() +// vm.Set("url", url[22:]) +// r, err := vm.RunString(` +// (function(e){ +// function A(t, e) { +// e = 1 < arguments.length && void 0 !== e ? e : 10; +// for (var n = function() { +// for (var t = [], e = 0; e < 256; e++) { +// for (var n = e, r = 0; r < 8; r++) +// n = 1 & n ? 3988292384 ^ n >>> 1 : n >>> 1; +// t[e] = n +// } +// return t +// }(), r = function(t) { +// t = t.replace(/\\r\\n/g, "\\n"); +// for (var e = "", n = 0; n < t.length; n++) { +// var r = t.charCodeAt(n); +// r < 128 ? e += String.fromCharCode(r) : e = 127 < r && r < 2048 ? (e += String.fromCharCode(r >> 6 | 192)) + String.fromCharCode(63 & r | 128) : (e = (e += String.fromCharCode(r >> 12 | 224)) + String.fromCharCode(r >> 6 & 63 | 128)) + String.fromCharCode(63 & r | 128) +// } +// return e +// }(t), a = -1, i = 0; i < r.length; i++) +// a = a >>> 8 ^ n[255 & (a ^ r.charCodeAt(i))]; +// return (a = (-1 ^ a) >>> 0).toString(e) +// } +// +// function v(t) { +// return (v = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function(t) { +// return typeof t +// } +// : function(t) { +// return t && "function" == typeof Symbol && t.constructor === Symbol && t !== Symbol.prototype ? "symbol" : typeof t +// } +// )(t) +// } +// +// for (p in a = Math.round(1e7 * Math.random()), +// o = Math.round(((new Date).getTime() + 60 * (new Date).getTimezoneOffset() * 1e3 + 288e5) / 1e3).toString(), +// m = ["a", "d", "e", "f", "g", "h", "l", "m", "y", "i", "j", "n", "o", "p", "k", "q", "r", "s", "t", "u", "b", "c", "v", "w", "s", "z"], +// u = function(t, e, n) { +// var r; +// n = 2 < arguments.length && void 0 !== n ? n : 8; +// return 0 === arguments.length ? null : (r = "object" === v(t) ? t : (10 === "".concat(t).length && (t = 1e3 * Number.parseInt(t)), +// new Date(t)), +// t += 6e4 * new Date(t).getTimezoneOffset(), +// { +// y: (r = new Date(t + 36e5 * n)).getFullYear(), +// m: r.getMonth() + 1 < 10 ? "0".concat(r.getMonth() + 1) : r.getMonth() + 1, +// d: r.getDate() < 10 ? "0".concat(r.getDate()) : r.getDate(), +// h: r.getHours() < 10 ? "0".concat(r.getHours()) : r.getHours(), +// f: r.getMinutes() < 10 ? "0".concat(r.getMinutes()) : r.getMinutes() +// }) +// }(o), +// h = u.y, +// g = u.m, +// l = u.d, +// c = u.h, +// u = u.f, +// d = [h, g, l, c, u].join(""), +// f = [], +// d) +// f.push(m[Number(d[p])]); +// return h = A(f.join("")), +// g = A("".concat(o, "|").concat(a, "|").concat(e, "|").concat("web", "|").concat("3", "|").concat(h)), +// "".concat(h, "=").concat(o, "-").concat(a, "-").concat(g); +// })(url) +// `) +// if err != nil { +// fmt.Println(err) +// return url +// } +// v, _ := r.Export().(string) +// return url + "?" + v +//} + func (d *Pan123) login() error { var body base.Json if utils.IsEmailFormat(d.Username) { @@ -54,12 +162,12 @@ func (d *Pan123) login() error { } res, err := base.RestyClient.R(). SetHeaders(map[string]string{ - "origin": "https://www.123pan.com", - "referer": "https://www.123pan.com/", - "user-agent": "Dart/2.19(dart:io)", - "platform": "android", - "app-version": "36", - //"user-agent": base.UserAgent, + "origin": "https://www.123pan.com", + "referer": "https://www.123pan.com/", + //"user-agent": "Dart/2.19(dart:io)-alist", + "platform": "web", + "app-version": "3", + "user-agent": base.UserAgent, }). SetBody(body).Post(SignIn) if err != nil { @@ -87,15 +195,17 @@ func (d *Pan123) login() error { // return &authKey, nil //} -func (d *Pan123) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { +func (d *Pan123) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + isRetry := false +do: req := base.RestyClient.R() req.SetHeaders(map[string]string{ "origin": "https://www.123pan.com", "referer": "https://www.123pan.com/", "authorization": "Bearer " + d.AccessToken, - "user-agent": "Dart/2.19(dart:io)", - "platform": "android", - "app-version": "36", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)", + "platform": "web", + "app-version": "3", //"user-agent": base.UserAgent, }) if callback != nil { @@ -109,51 +219,92 @@ func (d *Pan123) request(url string, method string, callback base.ReqCallback, r // return nil, err //} //req.SetQueryParam("auth-key", *authKey) - res, err := req.Execute(method, url) + res, err := req.Execute(method, GetApi(url)) if err != nil { return nil, err } body := res.Body() code := utils.Json.Get(body, "code").ToInt() if code != 0 { - if code == 401 { + if !isRetry && code == 401 { err := d.login() if err != nil { return nil, err } - return d.request(url, method, callback, resp) + isRetry = true + goto do } return nil, errors.New(jsoniter.Get(body, "message").ToString()) } return body, nil } -func (d *Pan123) getFiles(parentId string) ([]File, error) { +func (d *Pan123) unlockSafeBox(fileId int64) error { + if _, ok := d.safeBoxUnlocked.Load(fileId); ok { + return nil + } + data := base.Json{"password": d.SafePassword} + url := fmt.Sprintf("%s?fileId=%d", SafeBoxUnlock, fileId) + _, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + if err != nil { + return err + } + d.safeBoxUnlocked.Store(fileId, true) + return nil +} + +func (d *Pan123) getFiles(ctx context.Context, parentId string, name string) ([]File, error) { page := 1 + total := 0 res := make([]File, 0) + // 2024-02-06 fix concurrency by 123pan for { + if err := d.APIRateLimit(ctx, FileList); err != nil { + return nil, err + } var resp Files query := map[string]string{ - "driveId": "0", - "limit": "100", - "next": "0", - "orderBy": d.OrderBy, - "orderDirection": d.OrderDirection, - "parentFileId": parentId, - "trashed": "false", - "Page": strconv.Itoa(page), + "driveId": "0", + "limit": "100", + "next": "0", + "orderBy": "file_id", + "orderDirection": "desc", + "parentFileId": parentId, + "trashed": "false", + "SearchData": "", + "Page": strconv.Itoa(page), + "OnlyLookAbnormalFile": "0", + "event": "homeListFile", + "operateType": "4", + "inDirectSpace": "false", } - _, err := d.request(FileList, http.MethodGet, func(req *resty.Request) { + _res, err := d.Request(FileList, http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "safe box") || strings.Contains(err.Error(), "保险箱") { + if fid, e := strconv.ParseInt(parentId, 10, 64); e == nil { + if e = d.unlockSafeBox(fid); e == nil { + return d.getFiles(ctx, parentId, name) + } + return nil, e + } + } return nil, err } + log.Debug(string(_res)) page++ res = append(res, resp.Data.InfoList...) + total = resp.Data.Total if len(resp.Data.InfoList) == 0 || resp.Data.Next == "-1" { break } } + if len(res) != total { + log.Warnf("incorrect file count from remote at %s: expected %d, got %d", name, total, len(res)) + } return res, nil } diff --git a/drivers/123_open/api.go b/drivers/123_open/api.go new file mode 100644 index 00000000000..1d2a6f164ff --- /dev/null +++ b/drivers/123_open/api.go @@ -0,0 +1,191 @@ +package _123Open + +import ( + "fmt" + "github.com/go-resty/resty/v2" + "net/http" +) + +const ( + // baseurl + ApiBaseURL = "https://open-api.123pan.com" + + // auth + ApiToken = "/api/v1/access_token" + + // file list + ApiFileList = "/api/v2/file/list" + + // direct link + ApiGetDirectLink = "/api/v1/direct-link/url" + + // mkdir + ApiMakeDir = "/upload/v1/file/mkdir" + + // remove + ApiRemove = "/api/v1/file/trash" + + // upload + ApiUploadDomainURL = "/upload/v2/file/domain" + ApiSingleUploadURL = "/upload/v2/file/single/create" + ApiCreateUploadURL = "/upload/v2/file/create" + ApiUploadSliceURL = "/upload/v2/file/slice" + ApiUploadCompleteURL = "/upload/v2/file/upload_complete" + + // move + ApiMove = "/api/v1/file/move" + + // rename + ApiRename = "/api/v1/file/name" +) + +type Response[T any] struct { + Code int `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type TokenResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data TokenData `json:"data"` +} + +type TokenData struct { + AccessToken string `json:"accessToken"` + ExpiredAt string `json:"expiredAt"` +} + +type FileListResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileListData `json:"data"` +} + +type FileListData struct { + LastFileId int64 `json:"lastFileId"` + FileList []File `json:"fileList"` +} + +type DirectLinkResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data DirectLinkData `json:"data"` +} + +type DirectLinkData struct { + URL string `json:"url"` +} + +type MakeDirRequest struct { + Name string `json:"name"` + ParentID int64 `json:"parentID"` +} + +type MakeDirResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data MakeDirData `json:"data"` +} + +type MakeDirData struct { + DirID int64 `json:"dirID"` +} + +type RemoveRequest struct { + FileIDs []int64 `json:"fileIDs"` +} + +type UploadCreateResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadCreateData `json:"data"` +} + +type UploadCreateData struct { + FileID int64 `json:"fileId"` + Reuse bool `json:"reuse"` + PreuploadID string `json:"preuploadId"` + SliceSize int64 `json:"sliceSize"` + Servers []string `json:"servers"` +} + +type UploadUrlResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadUrlData `json:"data"` +} + +type UploadUrlData struct { + PresignedURL string `json:"presignedUrl"` +} + +type UploadCompleteResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data UploadCompleteData `json:"data"` +} + +type UploadCompleteData struct { + FileID int `json:"fileID"` + Completed bool `json:"completed"` +} + +func (d *Open123) Request(endpoint string, method string, setup func(*resty.Request), result any) (*resty.Response, error) { + client := resty.New() + token, err := d.tm.getToken() + if err != nil { + return nil, err + } + + req := client.R(). + SetHeader("Authorization", "Bearer "+token). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", "application/json"). + SetResult(result) + + if setup != nil { + setup(req) + } + + switch method { + case http.MethodGet: + return req.Get(ApiBaseURL + endpoint) + case http.MethodPost: + return req.Post(ApiBaseURL + endpoint) + case http.MethodPut: + return req.Put(ApiBaseURL + endpoint) + default: + return nil, fmt.Errorf("unsupported method: %s", method) + } +} + +func (d *Open123) RequestTo(fullURL string, method string, setup func(*resty.Request), result any) (*resty.Response, error) { + client := resty.New() + + token, err := d.tm.getToken() + if err != nil { + return nil, err + } + + req := client.R(). + SetHeader("Authorization", "Bearer "+token). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", "application/json"). + SetResult(result) + + if setup != nil { + setup(req) + } + + switch method { + case http.MethodGet: + return req.Get(fullURL) + case http.MethodPost: + return req.Post(fullURL) + case http.MethodPut: + return req.Put(fullURL) + default: + return nil, fmt.Errorf("unsupported method: %s", method) + } +} diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go new file mode 100644 index 00000000000..ace86bb981a --- /dev/null +++ b/drivers/123_open/driver.go @@ -0,0 +1,294 @@ +package _123Open + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "net/http" + "strconv" + "time" +) + +type Open123 struct { + model.Storage + Addition + + UploadThread int + tm *tokenManager +} + +func (d *Open123) Config() driver.Config { + return config +} + +func (d *Open123) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Open123) Init(ctx context.Context) error { + d.tm = newTokenManager(d.ClientID, d.ClientSecret) + + if _, err := d.tm.getToken(); err != nil { + return fmt.Errorf("token 初始化失败: %w", err) + } + + return nil +} + +func (d *Open123) Drop(ctx context.Context) error { + return nil +} + +func (d *Open123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + parentFileId, err := strconv.ParseInt(dir.GetID(), 10, 64) + if err != nil { + return nil, err + } + + fileLastId := int64(0) + var results []File + + for fileLastId != -1 { + files, err := d.getFiles(parentFileId, 100, fileLastId) + if err != nil { + return nil, err + } + for _, f := range files.Data.FileList { + if f.Trashed == 0 { + results = append(results, f) + } + } + fileLastId = files.Data.LastFileId + } + + objs := make([]model.Obj, 0, len(results)) + for _, f := range results { + objs = append(objs, f) + } + return objs, nil +} + +func (d *Open123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.LinkIsDir + } + + fileID := file.GetID() + + var result DirectLinkResp + url := fmt.Sprintf("%s?fileID=%s", ApiGetDirectLink, fileID) + _, err := d.Request(url, http.MethodGet, nil, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("get link failed: %s", result.Message) + } + + linkURL := result.Data.URL + if d.PrivateKey != "" { + if d.UID == 0 { + return nil, fmt.Errorf("uid is required when private key is set") + } + duration := time.Duration(d.ValidDuration) + if duration <= 0 { + duration = 30 + } + signedURL, err := SignURL(linkURL, d.PrivateKey, d.UID, duration*time.Minute) + if err != nil { + return nil, err + } + linkURL = signedURL + } + + return &model.Link{ + URL: linkURL, + }, nil +} + +func (d *Open123) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID, err := strconv.ParseInt(parentDir.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid parent ID: %w", err) + } + + var result MakeDirResp + reqBody := MakeDirRequest{ + Name: dirName, + ParentID: parentID, + } + + _, err = d.Request(ApiMakeDir, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("mkdir failed: %s", result.Message) + } + + newDir := File{ + FileId: result.Data.DirID, + FileName: dirName, + Type: 1, + ParentFileId: int(parentID), + Size: 0, + Trashed: 0, + } + return newDir, nil +} + +func (d *Open123) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcID, err := strconv.ParseInt(srcObj.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid src file ID: %w", err) + } + dstID, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid dest dir ID: %w", err) + } + + var result Response[any] + reqBody := map[string]interface{}{ + "fileIDs": []int64{srcID}, + "toParentFileID": dstID, + } + + _, err = d.Request(ApiMove, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("move failed: %s", result.Message) + } + + files, err := d.getFiles(dstID, 100, 0) + if err != nil { + return nil, fmt.Errorf("move succeed but failed to get target dir: %w", err) + } + for _, f := range files.Data.FileList { + if f.FileId == srcID { + return f, nil + } + } + return nil, fmt.Errorf("move succeed but file not found in target dir") +} + +func (d *Open123) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + srcID, err := strconv.ParseInt(srcObj.GetID(), 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid file ID: %w", err) + } + + var result Response[any] + reqBody := map[string]interface{}{ + "fileId": srcID, + "fileName": newName, + } + + _, err = d.Request(ApiRename, http.MethodPut, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("rename failed: %s", result.Message) + } + + parentID := 0 + if file, ok := srcObj.(File); ok { + parentID = file.ParentFileId + } + files, err := d.getFiles(int64(parentID), 100, 0) + if err != nil { + return nil, fmt.Errorf("rename succeed but failed to get parent dir: %w", err) + } + for _, f := range files.Data.FileList { + if f.FileId == srcID { + return f, nil + } + } + return nil, fmt.Errorf("rename succeed but file not found in parent dir") +} + +func (d *Open123) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { + idStr := obj.GetID() + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + return fmt.Errorf("invalid file ID: %w", err) + } + + var result Response[any] + reqBody := RemoveRequest{ + FileIDs: []int64{id}, + } + + _, err = d.Request(ApiRemove, http.MethodPost, func(r *resty.Request) { + r.SetBody(reqBody) + }, &result) + if err != nil { + return err + } + if result.Code != 0 { + return fmt.Errorf("remove failed: %s", result.Message) + } + + return nil +} + +func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + parentFileId, err := strconv.ParseInt(dstDir.GetID(), 10, 64) + etag := file.GetHash().GetHash(utils.MD5) + + if len(etag) < utils.MD5.Width { + up = model.UpdateProgressWithRange(up, 50, 100) + _, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return err + } + if createResp.Data.Reuse { + return nil + } + + return d.Upload(ctx, file, parentFileId, createResp, up) +} + +func (d *Open123) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotSupport +} + +func (d *Open123) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Open123) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotSupport +} + +func (d *Open123) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +//func (d *Open123) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Open123)(nil) diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go new file mode 100644 index 00000000000..d0b117aa7c0 --- /dev/null +++ b/drivers/123_open/meta.go @@ -0,0 +1,36 @@ +package _123Open + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + + ClientID string `json:"client_id" required:"true" label:"Client ID"` + ClientSecret string `json:"client_secret" required:"true" label:"Client Secret"` + PrivateKey string `json:"private_key"` + UID uint64 `json:"uid" type:"number"` + ValidDuration int64 `json:"valid_duration" type:"number" default:"30" help:"minutes"` +} + +var config = driver.Config{ + Name: "123 Open", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Open123{} + }) +} diff --git a/drivers/123_open/sign.go b/drivers/123_open/sign.go new file mode 100644 index 00000000000..549d7ad8fdf --- /dev/null +++ b/drivers/123_open/sign.go @@ -0,0 +1,27 @@ +package _123Open + +import ( + "crypto/md5" + "fmt" + "math/rand" + "net/url" + "time" +) + +func SignURL(originURL, privateKey string, uid uint64, validDuration time.Duration) (string, error) { + if privateKey == "" { + return originURL, nil + } + parsed, err := url.Parse(originURL) + if err != nil { + return "", err + } + ts := time.Now().Add(validDuration).Unix() + randInt := rand.Int() + signature := fmt.Sprintf("%d-%d-%d-%x", ts, randInt, uid, md5.Sum([]byte(fmt.Sprintf("%s-%d-%d-%d-%s", + parsed.Path, ts, randInt, uid, privateKey)))) + query := parsed.Query() + query.Add("auth_key", signature) + parsed.RawQuery = query.Encode() + return parsed.String(), nil +} diff --git a/drivers/123_open/token.go b/drivers/123_open/token.go new file mode 100644 index 00000000000..435c0b0de67 --- /dev/null +++ b/drivers/123_open/token.go @@ -0,0 +1,85 @@ +package _123Open + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "sync" + "time" +) + +const tokenURL = ApiBaseURL + ApiToken + +type tokenManager struct { + clientID string + clientSecret string + + mu sync.Mutex + accessToken string + expireTime time.Time +} + +func newTokenManager(clientID, clientSecret string) *tokenManager { + return &tokenManager{ + clientID: clientID, + clientSecret: clientSecret, + } +} + +func (tm *tokenManager) getToken() (string, error) { + tm.mu.Lock() + defer tm.mu.Unlock() + + if tm.accessToken != "" && time.Now().Before(tm.expireTime.Add(-5*time.Minute)) { + return tm.accessToken, nil + } + + reqBody := map[string]string{ + "clientID": tm.clientID, + "clientSecret": tm.clientSecret, + } + body, _ := json.Marshal(reqBody) + req, err := http.NewRequest("POST", tokenURL, bytes.NewBuffer(body)) + if err != nil { + return "", err + } + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var result TokenResp + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", err + } + + if result.Code != 0 { + return "", fmt.Errorf("get token failed: %s", result.Message) + } + + tm.accessToken = result.Data.AccessToken + expireAt, err := time.Parse(time.RFC3339, result.Data.ExpiredAt) + if err != nil { + return "", fmt.Errorf("parse expire time failed: %w", err) + } + tm.expireTime = expireAt + + return tm.accessToken, nil +} + +func (tm *tokenManager) buildHeaders() (http.Header, error) { + token, err := tm.getToken() + if err != nil { + return nil, err + } + header := http.Header{} + header.Set("Authorization", "Bearer "+token) + header.Set("Platform", "open_platform") + header.Set("Content-Type", "application/json") + return header, nil +} diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go new file mode 100644 index 00000000000..afece279e60 --- /dev/null +++ b/drivers/123_open/types.go @@ -0,0 +1,70 @@ +package _123Open + +import ( + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" + "time" +) + +type File struct { + FileName string `json:"filename"` + Size int64 `json:"size"` + CreateAt string `json:"createAt"` + UpdateAt string `json:"updateAt"` + FileId int64 `json:"fileId"` + Type int `json:"type"` + Etag string `json:"etag"` + S3KeyFlag string `json:"s3KeyFlag"` + ParentFileId int `json:"parentFileId"` + Category int `json:"category"` + Status int `json:"status"` + Trashed int `json:"trashed"` +} + +func (f File) GetID() string { + return fmt.Sprint(f.FileId) +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) IsDir() bool { + return f.Type == 1 +} + +func (f File) GetModified() string { + return f.UpdateAt +} + +func (f File) GetThumb() string { + return "" +} + +func (f File) ModTime() time.Time { + t, err := time.Parse("2006-01-02 15:04:05", f.UpdateAt) + if err != nil { + return time.Time{} + } + return t +} + +func (f File) CreateTime() time.Time { + t, err := time.Parse("2006-01-02 15:04:05", f.CreateAt) + if err != nil { + return time.Time{} + } + return t +} + +func (f File) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.MD5, f.Etag) +} + +func (f File) GetPath() string { + return "" +} diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go new file mode 100644 index 00000000000..76e8ead46f8 --- /dev/null +++ b/drivers/123_open/upload.go @@ -0,0 +1,282 @@ +package _123Open + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "golang.org/x/sync/errgroup" + "io" + "mime/multipart" + "net/http" + "runtime" + "strconv" + "time" +) + +func (d *Open123) create(parentFileID int64, filename, etag string, size int64, duplicate int, containDir bool) (*UploadCreateResp, error) { + var resp UploadCreateResp + + _, err := d.Request(ApiCreateUploadURL, http.MethodPost, func(req *resty.Request) { + body := base.Json{ + "parentFileID": parentFileID, + "filename": filename, + "etag": etag, + "size": size, + } + if duplicate > 0 { + body["duplicate"] = duplicate + } + if containDir { + body["containDir"] = true + } + req.SetBody(body) + }, &resp) + + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Open123) GetUploadDomains() ([]string, error) { + var resp struct { + Code int `json:"code"` + Message string `json:"message"` + Data []string `json:"data"` + } + + _, err := d.Request(ApiUploadDomainURL, http.MethodGet, nil, &resp) + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("get upload domain failed: %s", resp.Message) + } + return resp.Data, nil +} + +func (d *Open123) UploadSingle(ctx context.Context, createResp *UploadCreateResp, file model.FileStreamer, parentID int64) error { + domain := createResp.Data.Servers[0] + + etag := file.GetHash().GetHash(utils.MD5) + if len(etag) < utils.MD5.Width { + _, _, err := stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + + reader, err := file.RangeRead(http_range.Range{Start: 0, Length: file.GetSize()}) + if err != nil { + return err + } + reader = driver.NewLimitedUploadStream(ctx, reader) + + var b bytes.Buffer + mw := multipart.NewWriter(&b) + mw.WriteField("parentFileID", fmt.Sprint(parentID)) + mw.WriteField("filename", file.GetName()) + mw.WriteField("etag", etag) + mw.WriteField("size", fmt.Sprint(file.GetSize())) + fw, _ := mw.CreateFormFile("file", file.GetName()) + _, err = io.Copy(fw, reader) + mw.Close() + + req, err := http.NewRequestWithContext(ctx, "POST", domain+ApiSingleUploadURL, &b) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.tm.accessToken) + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", mw.FormDataContentType()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + var result struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + FileID int64 `json:"fileID"` + Completed bool `json:"completed"` + } `json:"data"` + } + body, _ := io.ReadAll(resp.Body) + if err := json.Unmarshal(body, &result); err != nil { + return fmt.Errorf("unmarshal response error: %v, body: %s", err, string(body)) + } + if result.Code != 0 { + return fmt.Errorf("upload failed: %s", result.Message) + } + if !result.Data.Completed || result.Data.FileID == 0 { + return fmt.Errorf("upload incomplete or missing fileID") + } + return nil +} + +func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, parentID int64, createResp *UploadCreateResp, up driver.UpdateProgress) error { + if cacher, ok := file.(interface{ CacheFullInTempFile() (model.File, error) }); ok { + if _, err := cacher.CacheFullInTempFile(); err != nil { + return err + } + } + + size := file.GetSize() + chunkSize := createResp.Data.SliceSize + uploadNums := (size + chunkSize - 1) / chunkSize + uploadDomain := createResp.Data.Servers[0] + + if d.UploadThread <= 0 { + cpuCores := runtime.NumCPU() + threads := cpuCores * 2 + if threads < 4 { + threads = 4 + } + if threads > 16 { + threads = 16 + } + d.UploadThread = threads + fmt.Printf("[Upload] Auto set upload concurrency: %d (CPU cores=%d)\n", d.UploadThread, cpuCores) + } + + fmt.Printf("[Upload] File size: %d bytes, chunk size: %d bytes, total slices: %d, concurrency: %d\n", + size, chunkSize, uploadNums, d.UploadThread) + + if size <= 1<<30 { + return d.UploadSingle(ctx, createResp, file, parentID) + } + + if createResp.Data.Reuse { + up(100) + return nil + } + + client := resty.New() + semaphore := make(chan struct{}, d.UploadThread) + threadG, _ := errgroup.WithContext(ctx) + + var progressArr = make([]int64, uploadNums) + + for partIndex := int64(0); partIndex < uploadNums; partIndex++ { + partIndex := partIndex + semaphore <- struct{}{} + + threadG.Go(func() error { + defer func() { <-semaphore }() + offset := partIndex * chunkSize + length := min(chunkSize, size-offset) + partNumber := partIndex + 1 + + fmt.Printf("[Slice %d] Starting read from offset %d, length %d\n", partNumber, offset, length) + reader, err := file.RangeRead(http_range.Range{Start: offset, Length: length}) + if err != nil { + return fmt.Errorf("[Slice %d] RangeRead error: %v", partNumber, err) + } + + buf := make([]byte, length) + n, err := io.ReadFull(reader, buf) + if err != nil && err != io.EOF { + return fmt.Errorf("[Slice %d] Read error: %v", partNumber, err) + } + buf = buf[:n] + hash := md5.Sum(buf) + sliceMD5Str := hex.EncodeToString(hash[:]) + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + writer.WriteField("preuploadID", createResp.Data.PreuploadID) + writer.WriteField("sliceNo", strconv.FormatInt(partNumber, 10)) + writer.WriteField("sliceMD5", sliceMD5Str) + partName := fmt.Sprintf("%s.part%d", file.GetName(), partNumber) + fw, _ := writer.CreateFormFile("slice", partName) + fw.Write(buf) + writer.Close() + + resp, err := client.R(). + SetHeader("Authorization", "Bearer "+d.tm.accessToken). + SetHeader("Platform", "open_platform"). + SetHeader("Content-Type", writer.FormDataContentType()). + SetBody(body.Bytes()). + Post(uploadDomain + ApiUploadSliceURL) + + if err != nil { + return fmt.Errorf("[Slice %d] Upload HTTP error: %v", partNumber, err) + } + if resp.StatusCode() != 200 { + return fmt.Errorf("[Slice %d] Upload failed with status: %s, resp: %s", partNumber, resp.Status(), resp.String()) + } + + progressArr[partIndex] = length + var totalUploaded int64 = 0 + for _, v := range progressArr { + totalUploaded += v + } + if up != nil { + percent := float64(totalUploaded) / float64(size) * 100 + up(percent) + } + + fmt.Printf("[Slice %d] MD5: %s\n", partNumber, sliceMD5Str) + fmt.Printf("[Slice %d] Upload finished\n", partNumber) + return nil + }) + } + + if err := threadG.Wait(); err != nil { + return err + } + + var completeResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Completed bool `json:"completed"` + FileID int64 `json:"fileID"` + } `json:"data"` + } + + for { + reqBody := fmt.Sprintf(`{"preuploadID":"%s"}`, createResp.Data.PreuploadID) + req, err := http.NewRequestWithContext(ctx, "POST", uploadDomain+ApiUploadCompleteURL, bytes.NewBufferString(reqBody)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+d.tm.accessToken) + req.Header.Set("Platform", "open_platform") + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + + if err := json.Unmarshal(body, &completeResp); err != nil { + return fmt.Errorf("completion response unmarshal error: %v, body: %s", err, string(body)) + } + if completeResp.Code != 0 { + return fmt.Errorf("completion API returned error code %d: %s", completeResp.Code, completeResp.Message) + } + if completeResp.Data.Completed && completeResp.Data.FileID != 0 { + fmt.Printf("[Upload] Upload completed successfully. FileID: %d\n", completeResp.Data.FileID) + break + } + time.Sleep(time.Second) + } + up(100) + return nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go new file mode 100644 index 00000000000..429a5e5dda8 --- /dev/null +++ b/drivers/123_open/util.go @@ -0,0 +1,20 @@ +package _123Open + +import ( + "fmt" + "net/http" +) + +func (d *Open123) getFiles(parentFileId int64, limit int, lastFileId int64) (*FileListResp, error) { + var result FileListResp + url := fmt.Sprintf("%s?parentFileId=%d&limit=%d&lastFileId=%d", ApiFileList, parentFileId, limit, lastFileId) + + _, err := d.Request(url, http.MethodGet, nil, &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("list error: %s", result.Message) + } + return &result, nil +} diff --git a/drivers/123_share/driver.go b/drivers/123_share/driver.go index b2fd4313331..640fb74967e 100644 --- a/drivers/123_share/driver.go +++ b/drivers/123_share/driver.go @@ -6,7 +6,12 @@ import ( "fmt" "net/http" "net/url" + "sync" + "time" + "golang.org/x/time/rate" + + _123 "github.com/alist-org/alist/v3/drivers/123" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" @@ -19,6 +24,8 @@ import ( type Pan123Share struct { model.Storage Addition + apiRateLimit sync.Map + ref *_123.Pan123 } func (d *Pan123Share) Config() driver.Config { @@ -35,13 +42,23 @@ func (d *Pan123Share) Init(ctx context.Context) error { return nil } +func (d *Pan123Share) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*_123.Pan123) + if ok { + d.ref = refStorage + return nil + } + return fmt.Errorf("ref: storage is not 123Pan") +} + func (d *Pan123Share) Drop(ctx context.Context) error { + d.ref = nil return nil } func (d *Pan123Share) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { // TODO return the files list, required - files, err := d.getFiles(dir.GetID()) + files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err } @@ -146,4 +163,12 @@ func (d *Pan123Share) Put(ctx context.Context, dstDir model.Obj, stream model.Fi // return nil, errs.NotSupport //} +func (d *Pan123Share) APIRateLimit(ctx context.Context, api string) error { + value, _ := d.apiRateLimit.LoadOrStore(api, + rate.NewLimiter(rate.Every(700*time.Millisecond), 1)) + limiter := value.(*rate.Limiter) + + return limiter.Wait(ctx) +} + var _ driver.Driver = (*Pan123Share)(nil) diff --git a/drivers/123_share/meta.go b/drivers/123_share/meta.go index a4bb14a9593..7cbcba27724 100644 --- a/drivers/123_share/meta.go +++ b/drivers/123_share/meta.go @@ -7,10 +7,11 @@ import ( type Addition struct { ShareKey string `json:"sharekey" required:"true"` - SharePwd string `json:"sharepassword" required:"true"` + SharePwd string `json:"sharepassword"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + //OrderBy string `json:"order_by" type:"select" options:"file_name,size,update_at" default:"file_name"` + //OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + AccessToken string `json:"accesstoken" type:"text"` } var config = driver.Config{ diff --git a/drivers/123_share/util.go b/drivers/123_share/util.go index bfce54f3cc0..c2140bf604f 100644 --- a/drivers/123_share/util.go +++ b/drivers/123_share/util.go @@ -1,9 +1,17 @@ package _123Share import ( + "context" "errors" + "fmt" + "hash/crc32" + "math" + "math/rand" "net/http" + "net/url" "strconv" + "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" @@ -15,20 +23,48 @@ const ( Api = "https://www.123pan.com/api" AApi = "https://www.123pan.com/a/api" BApi = "https://www.123pan.com/b/api" - MainApi = Api + MainApi = BApi FileList = MainApi + "/share/get" DownloadInfo = MainApi + "/share/download/info" //AuthKeySalt = "8-8D$sL8gPjom7bk#cY" ) +func signPath(path string, os string, version string) (k string, v string) { + table := []byte{'a', 'd', 'e', 'f', 'g', 'h', 'l', 'm', 'y', 'i', 'j', 'n', 'o', 'p', 'k', 'q', 'r', 's', 't', 'u', 'b', 'c', 'v', 'w', 's', 'z'} + random := fmt.Sprintf("%.f", math.Round(1e7*rand.Float64())) + now := time.Now().In(time.FixedZone("CST", 8*3600)) + timestamp := fmt.Sprint(now.Unix()) + nowStr := []byte(now.Format("200601021504")) + for i := 0; i < len(nowStr); i++ { + nowStr[i] = table[nowStr[i]-48] + } + timeSign := fmt.Sprint(crc32.ChecksumIEEE(nowStr)) + data := strings.Join([]string{timestamp, random, path, os, version, timeSign}, "|") + dataSign := fmt.Sprint(crc32.ChecksumIEEE([]byte(data))) + return timeSign, strings.Join([]string{timestamp, random, dataSign}, "-") +} + +func GetApi(rawUrl string) string { + u, _ := url.Parse(rawUrl) + query := u.Query() + query.Add(signPath(u.Path, "web", "3")) + u.RawQuery = query.Encode() + return u.String() +} + func (d *Pan123Share) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if d.ref != nil { + return d.ref.Request(url, method, callback, resp) + } req := base.RestyClient.R() req.SetHeaders(map[string]string{ - "origin": "https://www.123pan.com", - "referer": "https://www.123pan.com/", - "user-agent": "Dart/2.19(dart:io)", - "platform": "android", - "app-version": "36", + "origin": "https://www.123pan.com", + "referer": "https://www.123pan.com/", + "authorization": "Bearer " + d.AccessToken, + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) alist-client", + "platform": "web", + "app-version": "3", + //"user-agent": base.UserAgent, }) if callback != nil { callback(req) @@ -36,7 +72,7 @@ func (d *Pan123Share) request(url string, method string, callback base.ReqCallba if resp != nil { req.SetResult(resp) } - res, err := req.Execute(method, url) + res, err := req.Execute(method, GetApi(url)) if err != nil { return nil, err } @@ -48,16 +84,19 @@ func (d *Pan123Share) request(url string, method string, callback base.ReqCallba return body, nil } -func (d *Pan123Share) getFiles(parentId string) ([]File, error) { +func (d *Pan123Share) getFiles(ctx context.Context, parentId string) ([]File, error) { page := 1 res := make([]File, 0) for { + if err := d.APIRateLimit(ctx, FileList); err != nil { + return nil, err + } var resp Files query := map[string]string{ "limit": "100", "next": "0", - "orderBy": d.OrderBy, - "orderDirection": d.OrderDirection, + "orderBy": "file_id", + "orderDirection": "desc", "parentFileId": parentId, "Page": strconv.Itoa(page), "shareKey": d.ShareKey, diff --git a/drivers/139/driver.go b/drivers/139/driver.go index 69ab68f705e..10d9d3e9e03 100644 --- a/drivers/139/driver.go +++ b/drivers/139/driver.go @@ -2,25 +2,32 @@ package _139 import ( "context" - "encoding/base64" + "encoding/xml" "fmt" "io" "net/http" + "path" "strconv" - "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + streamPkg "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" log "github.com/sirupsen/logrus" ) type Yun139 struct { model.Storage Addition - Account string + cron *cron.Cron + Account string + ref *Yun139 + PersonalCloudHost string } func (d *Yun139) Config() driver.Config { @@ -32,319 +39,828 @@ func (d *Yun139) GetAddition() driver.Additional { } func (d *Yun139) Init(ctx context.Context) error { - if d.Authorization == "" { - return fmt.Errorf("authorization is empty") + if d.ref == nil { + if len(d.Authorization) == 0 { + return fmt.Errorf("authorization is empty") + } + err := d.refreshToken() + if err != nil { + return err + } + if d.Addition.Type == MetaPersonalNew { + err = d.ensurePersonalCloudHost() + if err != nil { + return err + } + } + + d.cron = cron.NewCron(time.Hour * 12) + d.cron.Do(func() { + err := d.refreshToken() + if err != nil { + log.Errorf("%+v", err) + } + }) } - decode, err := base64.StdEncoding.DecodeString(d.Authorization) - if err != nil { - return err + switch d.Addition.Type { + case MetaPersonalNew: + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = "/" + } + case MetaPersonal: + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = "root" + } + case MetaGroup: + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = d.CloudID + } + case MetaFamily: + case "share": + if len(d.Addition.RootFolderID) == 0 { + d.RootFolderID = "root" + } + default: + return errs.NotImplement } - decodeStr := string(decode) - splits := strings.Split(decodeStr, ":") - if len(splits) < 2 { - return fmt.Errorf("authorization is invalid, splits < 2") + return nil +} + +func (d *Yun139) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*Yun139) + if ok { + d.ref = refStorage + return nil } - d.Account = splits[1] - _, err = d.post("/orchestration/personalCloud/user/v1.0/qryUserExternInfo", base.Json{ - "qryUserExternInfoReq": base.Json{ - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, - }, - }, nil) - return err + return errs.NotSupport } func (d *Yun139) Drop(ctx context.Context) error { + if d.cron != nil { + d.cron.Stop() + } + d.ref = nil return nil } func (d *Yun139) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if d.isFamily() { - return d.familyGetFiles(dir.GetID()) - } else { + log.Infof("[139Share-Debug] List called! Type: %s, DirID: %s", d.Addition.Type, dir.GetID()) + switch d.Addition.Type { + case MetaPersonalNew: + return d.personalGetFiles(dir.GetID()) + case MetaPersonal: return d.getFiles(dir.GetID()) + case MetaFamily: + return d.familyGetFiles(dir.GetID()) + case MetaGroup: + return d.groupGetFiles(dir.GetID()) + case "share": + return d.shareGetFiles(dir.GetID()) + default: + return nil, errs.NotImplement } } func (d *Yun139) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - u, err := d.getLink(file.GetID()) + var url string + var err error + switch d.Addition.Type { + case MetaPersonalNew: + url, err = d.personalGetLink(file.GetID()) + case MetaPersonal: + url, err = d.getLink(file.GetID()) + case MetaFamily: + url, err = d.familyGetLink(file.GetID(), file.GetPath()) + case MetaGroup: + url, err = d.groupGetLink(file.GetID(), file.GetPath()) + case "share": + return d.shareGetLink(file.GetID()) + default: + return nil, errs.NotImplement + } if err != nil { return nil, err } - return &model.Link{URL: u}, nil + return &model.Link{URL: url}, nil } func (d *Yun139) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - data := base.Json{ - "createCatalogExtReq": base.Json{ - "parentCatalogID": parentDir.GetID(), - "newCatalogName": dirName, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, + var err error + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "parentFileId": parentDir.GetID(), + "name": dirName, + "description": "", + "type": "folder", + "fileRenameMode": "force_rename", + } + pathname := "/file/create" + _, err = d.personalPost(pathname, data, nil) + case MetaPersonal: + data := base.Json{ + "createCatalogExtReq": base.Json{ + "parentCatalogID": parentDir.GetID(), + "newCatalogName": dirName, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, }, - }, - } - pathname := "/orchestration/personalCloud/catalog/v1.0/createCatalogExt" - if d.isFamily() { - data = base.Json{ + } + pathname := "/orchestration/personalCloud/catalog/v1.0/createCatalogExt" + _, err = d.post(pathname, data, nil) + case MetaFamily: + data := base.Json{ "cloudID": d.CloudID, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, "docLibName": dirName, + "path": path.Join(parentDir.GetPath(), parentDir.GetID()), + } + pathname := "/orchestration/familyCloud-rebuild/cloudCatalog/v1.0/createCloudDoc" + _, err = d.post(pathname, data, nil) + case MetaGroup: + data := base.Json{ + "catalogName": dirName, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + "groupID": d.CloudID, + "parentFileId": parentDir.GetID(), + "path": path.Join(parentDir.GetPath(), parentDir.GetID()), } - pathname = "/orchestration/familyCloud/cloudCatalog/v1.0/createCloudDoc" + pathname := "/orchestration/group-rebuild/catalog/v1.0/createGroupCatalog" + _, err = d.post(pathname, data, nil) + default: + err = errs.NotImplement } - _, err := d.post(pathname, data, nil) return err } func (d *Yun139) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - if d.isFamily() { - return nil, errs.NotImplement - } - var contentInfoList []string - var catalogInfoList []string - if srcObj.IsDir() { - catalogInfoList = append(catalogInfoList, srcObj.GetID()) - } else { - contentInfoList = append(contentInfoList, srcObj.GetID()) - } - data := base.Json{ - "createBatchOprTaskReq": base.Json{ - "taskType": 3, - "actionType": "304", - "taskInfo": base.Json{ - "contentInfoList": contentInfoList, - "catalogInfoList": catalogInfoList, - "newCatalogID": dstDir.GetID(), - }, + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "fileIds": []string{srcObj.GetID()}, + "toParentFileId": dstDir.GetID(), + } + pathname := "/file/batchMove" + _, err := d.personalPost(pathname, data, nil) + if err != nil { + return nil, err + } + return srcObj, nil + case MetaGroup: + var contentList []string + var catalogList []string + if srcObj.IsDir() { + catalogList = append(catalogList, srcObj.GetID()) + } else { + contentList = append(contentList, srcObj.GetID()) + } + data := base.Json{ + "taskType": 3, + "srcType": 2, + "srcGroupID": d.CloudID, + "destType": 2, + "destGroupID": d.CloudID, + "destPath": dstDir.GetPath(), + "contentList": contentList, + "catalogList": catalogList, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, - }, - } - pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" - _, err := d.post(pathname, data, nil) - if err != nil { - return nil, err + } + pathname := "/orchestration/group-rebuild/task/v1.0/createBatchOprTask" + _, err := d.post(pathname, data, nil) + if err != nil { + return nil, err + } + return srcObj, nil + case MetaPersonal: + var contentInfoList []string + var catalogInfoList []string + if srcObj.IsDir() { + catalogInfoList = append(catalogInfoList, srcObj.GetID()) + } else { + contentInfoList = append(contentInfoList, srcObj.GetID()) + } + data := base.Json{ + "createBatchOprTaskReq": base.Json{ + "taskType": 3, + "actionType": "304", + "taskInfo": base.Json{ + "contentInfoList": contentInfoList, + "catalogInfoList": catalogInfoList, + "newCatalogID": dstDir.GetID(), + }, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + }, + } + pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" + _, err := d.post(pathname, data, nil) + if err != nil { + return nil, err + } + return srcObj, nil + default: + return nil, errs.NotImplement } - return srcObj, nil } func (d *Yun139) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - if d.isFamily() { - return errs.NotImplement - } - var data base.Json - var pathname string - if srcObj.IsDir() { - data = base.Json{ - "catalogID": srcObj.GetID(), - "catalogName": newName, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, + var err error + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "fileId": srcObj.GetID(), + "name": newName, + "description": "", } - pathname = "/orchestration/personalCloud/catalog/v1.0/updateCatalogInfo" - } else { - data = base.Json{ - "contentID": srcObj.GetID(), - "contentName": newName, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, + pathname := "/file/update" + _, err = d.personalPost(pathname, data, nil) + case MetaPersonal: + var data base.Json + var pathname string + if srcObj.IsDir() { + data = base.Json{ + "catalogID": srcObj.GetID(), + "catalogName": newName, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + } + pathname = "/orchestration/personalCloud/catalog/v1.0/updateCatalogInfo" + } else { + data = base.Json{ + "contentID": srcObj.GetID(), + "contentName": newName, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + } + pathname = "/orchestration/personalCloud/content/v1.0/updateContentInfo" + } + _, err = d.post(pathname, data, nil) + case MetaGroup: + var data base.Json + var pathname string + if srcObj.IsDir() { + data = base.Json{ + "groupID": d.CloudID, + "modifyCatalogID": srcObj.GetID(), + "modifyCatalogName": newName, + "path": srcObj.GetPath(), + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + } + pathname = "/orchestration/group-rebuild/catalog/v1.0/modifyGroupCatalog" + } else { + data = base.Json{ + "groupID": d.CloudID, + "contentID": srcObj.GetID(), + "contentName": newName, + "path": srcObj.GetPath(), + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + } + pathname = "/orchestration/group-rebuild/content/v1.0/modifyGroupContent" } - pathname = "/orchestration/personalCloud/content/v1.0/updateContentInfo" + _, err = d.post(pathname, data, nil) + case MetaFamily: + var data base.Json + var pathname string + if srcObj.IsDir() { + // 网页接口不支持重命名家庭云文件夹 + // data = base.Json{ + // "catalogType": 3, + // "catalogID": srcObj.GetID(), + // "catalogName": newName, + // "commonAccountInfo": base.Json{ + // "account": d.getAccount(), + // "accountType": 1, + // }, + // "path": srcObj.GetPath(), + // } + // pathname = "/orchestration/familyCloud-rebuild/photoContent/v1.0/modifyCatalogInfo" + return errs.NotImplement + } else { + data = base.Json{ + "contentID": srcObj.GetID(), + "contentName": newName, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + "path": srcObj.GetPath(), + } + pathname = "/orchestration/familyCloud-rebuild/photoContent/v1.0/modifyContentInfo" + } + _, err = d.post(pathname, data, nil) + default: + err = errs.NotImplement } - _, err := d.post(pathname, data, nil) return err } func (d *Yun139) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - if d.isFamily() { - return errs.NotImplement - } - var contentInfoList []string - var catalogInfoList []string - if srcObj.IsDir() { - catalogInfoList = append(catalogInfoList, srcObj.GetID()) - } else { - contentInfoList = append(contentInfoList, srcObj.GetID()) - } - data := base.Json{ - "createBatchOprTaskReq": base.Json{ - "taskType": 3, - "actionType": 309, - "taskInfo": base.Json{ - "contentInfoList": contentInfoList, - "catalogInfoList": catalogInfoList, - "newCatalogID": dstDir.GetID(), - }, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, + var err error + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "fileIds": []string{srcObj.GetID()}, + "toParentFileId": dstDir.GetID(), + } + pathname := "/file/batchCopy" + _, err := d.personalPost(pathname, data, nil) + return err + case MetaPersonal: + var contentInfoList []string + var catalogInfoList []string + if srcObj.IsDir() { + catalogInfoList = append(catalogInfoList, srcObj.GetID()) + } else { + contentInfoList = append(contentInfoList, srcObj.GetID()) + } + data := base.Json{ + "createBatchOprTaskReq": base.Json{ + "taskType": 3, + "actionType": 309, + "taskInfo": base.Json{ + "contentInfoList": contentInfoList, + "catalogInfoList": catalogInfoList, + "newCatalogID": dstDir.GetID(), + }, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, }, - }, + } + pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" + _, err = d.post(pathname, data, nil) + default: + err = errs.NotImplement } - pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" - _, err := d.post(pathname, data, nil) return err } func (d *Yun139) Remove(ctx context.Context, obj model.Obj) error { - var contentInfoList []string - var catalogInfoList []string - if obj.IsDir() { - catalogInfoList = append(catalogInfoList, obj.GetID()) - } else { - contentInfoList = append(contentInfoList, obj.GetID()) - } - data := base.Json{ - "createBatchOprTaskReq": base.Json{ - "taskType": 2, - "actionType": 201, - "taskInfo": base.Json{ - "newCatalogID": "", - "contentInfoList": contentInfoList, - "catalogInfoList": catalogInfoList, - }, + switch d.Addition.Type { + case MetaPersonalNew: + data := base.Json{ + "fileIds": []string{obj.GetID()}, + } + pathname := "/recyclebin/batchTrash" + _, err := d.personalPost(pathname, data, nil) + return err + case MetaGroup: + var contentList []string + var catalogList []string + // 必须使用完整路径删除 + if obj.IsDir() { + catalogList = append(catalogList, obj.GetPath()) + } else { + contentList = append(contentList, path.Join(obj.GetPath(), obj.GetID())) + } + data := base.Json{ + "taskType": 2, + "srcGroupID": d.CloudID, + "contentList": contentList, + "catalogList": catalogList, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, - }, - } - pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" - if d.isFamily() { - data = base.Json{ - "catalogList": catalogInfoList, - "contentList": contentInfoList, - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, + } + pathname := "/orchestration/group-rebuild/task/v1.0/createBatchOprTask" + _, err := d.post(pathname, data, nil) + return err + case MetaPersonal: + fallthrough + case MetaFamily: + var contentInfoList []string + var catalogInfoList []string + if obj.IsDir() { + catalogInfoList = append(catalogInfoList, obj.GetID()) + } else { + contentInfoList = append(contentInfoList, obj.GetID()) + } + data := base.Json{ + "createBatchOprTaskReq": base.Json{ + "taskType": 2, + "actionType": 201, + "taskInfo": base.Json{ + "newCatalogID": "", + "contentInfoList": contentInfoList, + "catalogInfoList": catalogInfoList, + }, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, }, - "sourceCatalogType": 1002, - "taskType": 2, } - pathname = "/orchestration/familyCloud/batchOprTask/v1.0/createBatchOprTask" + pathname := "/orchestration/personalCloud/batchOprTask/v1.0/createBatchOprTask" + if d.isFamily() { + data = base.Json{ + "catalogList": catalogInfoList, + "contentList": contentInfoList, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + "sourceCloudID": d.CloudID, + "sourceCatalogType": 1002, + "taskType": 2, + "path": obj.GetPath(), + } + pathname = "/orchestration/familyCloud-rebuild/batchOprTask/v1.0/createBatchOprTask" + } + _, err := d.post(pathname, data, nil) + return err + default: + return errs.NotImplement } - _, err := d.post(pathname, data, nil) - return err } -const ( - _ = iota //ignore first value by assigning to blank identifier - KB = 1 << (10 * iota) - MB - GB - TB -) - -func getPartSize(size int64) int64 { +func (d *Yun139) getPartSize(size int64) int64 { + if d.CustomUploadPartSize != 0 { + return d.CustomUploadPartSize + } // 网盘对于分片数量存在上限 - if size/GB > 30 { - return 512 * MB + if size/utils.GB > 30 { + return 512 * utils.MB } - return 100 * MB + return 100 * utils.MB } func (d *Yun139) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - data := base.Json{ - "manualRename": 2, - "operation": 0, - "fileCount": 1, - "totalSize": 0, // 去除上传大小限制 - "uploadContentList": []base.Json{{ - "contentName": stream.GetName(), - "contentSize": 0, // 去除上传大小限制 - // "digest": "5a3231986ce7a6b46e408612d385bafa" - }}, - "parentCatalogID": dstDir.GetID(), - "newCatalogName": "", - "commonAccountInfo": base.Json{ - "account": d.Account, - "accountType": 1, - }, - } - pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" - if d.isFamily() { - data = d.newJson(base.Json{ - "fileCount": 1, - "manualRename": 2, - "operation": 0, - "path": "", - "seqNo": "", - "totalSize": 0, - "uploadContentList": []base.Json{{ - "contentName": stream.GetName(), - "contentSize": 0, - // "digest": "5a3231986ce7a6b46e408612d385bafa" - }}, - }) - pathname = "/orchestration/familyCloud/content/v1.0/getFileUploadURL" - return errs.NotImplement - } - var resp UploadResp - _, err := d.post(pathname, data, &resp) - if err != nil { - return err - } - - // Progress - p := driver.NewProgress(stream.GetSize(), up) + switch d.Addition.Type { + case MetaPersonalNew: + var err error + fullHash := stream.GetHash().GetHash(utils.SHA256) + if len(fullHash) != utils.SHA256.Width { + _, fullHash, err = streamPkg.CacheFullInTempFileAndHash(stream, utils.SHA256) + if err != nil { + return err + } + } - var partSize = getPartSize(stream.GetSize()) - part := (stream.GetSize() + partSize - 1) / partSize - if part == 0 { - part = 1 - } - for i := int64(0); i < part; i++ { - if utils.IsCanceled(ctx) { - return ctx.Err() + size := stream.GetSize() + var partSize = d.getPartSize(size) + part := size / partSize + if size%partSize > 0 { + part++ + } else if part == 0 { + part = 1 + } + partInfos := make([]PartInfo, 0, part) + for i := int64(0); i < part; i++ { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + start := i * partSize + byteSize := size - start + if byteSize > partSize { + byteSize = partSize + } + partNumber := i + 1 + partInfo := PartInfo{ + PartNumber: partNumber, + PartSize: byteSize, + ParallelHashCtx: ParallelHashCtx{ + PartOffset: start, + }, + } + partInfos = append(partInfos, partInfo) } - start := i * partSize - byteSize := stream.GetSize() - start - if byteSize > partSize { - byteSize = partSize + // 筛选出前 100 个 partInfos + firstPartInfos := partInfos + if len(firstPartInfos) > 100 { + firstPartInfos = firstPartInfos[:100] } - limitReader := io.LimitReader(stream, byteSize) - // Update Progress - r := io.TeeReader(limitReader, p) - req, err := http.NewRequest("POST", resp.Data.UploadResult.RedirectionURL, r) + // 创建任务,获取上传信息和前100个分片的上传地址 + data := base.Json{ + "contentHash": fullHash, + "contentHashAlgorithm": "SHA256", + "contentType": "application/octet-stream", + "parallelUpload": false, + "partInfos": firstPartInfos, + "size": size, + "parentFileId": dstDir.GetID(), + "name": stream.GetName(), + "type": "file", + "fileRenameMode": "auto_rename", + } + pathname := "/file/create" + var resp PersonalUploadResp + _, err = d.personalPost(pathname, data, &resp) if err != nil { return err } - req = req.WithContext(ctx) - req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) - req.Header.Set("contentSize", strconv.FormatInt(stream.GetSize(), 10)) - req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) - req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) - req.Header.Set("rangeType", "0") - req.ContentLength = byteSize + // 判断文件是否已存在 + // resp.Data.Exist: true 已存在同名文件且校验相同,云端不会重复增加文件,无需手动处理冲突 + if resp.Data.Exist { + return nil + } + + // 判断文件是否支持快传 + // resp.Data.RapidUpload: true 支持快传,但此处直接检测是否返回分片的上传地址 + // 快传的情况下同样需要手动处理冲突 + if resp.Data.PartInfos != nil { + // 读取前100个分片的上传地址 + uploadPartInfos := resp.Data.PartInfos + + // 获取后续分片的上传地址 + for i := 101; i < len(partInfos); i += 100 { + end := i + 100 + if end > len(partInfos) { + end = len(partInfos) + } + batchPartInfos := partInfos[i:end] + + moredata := base.Json{ + "fileId": resp.Data.FileId, + "uploadId": resp.Data.UploadId, + "partInfos": batchPartInfos, + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + } + pathname := "/file/getUploadUrl" + var moreresp PersonalUploadUrlResp + _, err = d.personalPost(pathname, moredata, &moreresp) + if err != nil { + return err + } + uploadPartInfos = append(uploadPartInfos, moreresp.Data.PartInfos...) + } + + // Progress + p := driver.NewProgress(size, up) + + rateLimited := driver.NewLimitedUploadStream(ctx, stream) + // 上传所有分片 + for _, uploadPartInfo := range uploadPartInfos { + index := uploadPartInfo.PartNumber - 1 + partSize := partInfos[index].PartSize + log.Debugf("[139] uploading part %+v/%+v", index, len(uploadPartInfos)) + limitReader := io.LimitReader(rateLimited, partSize) - res, err := base.HttpClient.Do(req) + // Update Progress + r := io.TeeReader(limitReader, p) + + req, err := http.NewRequest("PUT", uploadPartInfo.UploadUrl, r) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "application/octet-stream") + req.Header.Set("Content-Length", fmt.Sprint(partSize)) + req.Header.Set("Origin", "https://yun.139.com") + req.Header.Set("Referer", "https://yun.139.com/") + req.ContentLength = partSize + + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + _ = res.Body.Close() + log.Debugf("[139] uploaded: %+v", res) + if res.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + } + + data = base.Json{ + "contentHash": fullHash, + "contentHashAlgorithm": "SHA256", + "fileId": resp.Data.FileId, + "uploadId": resp.Data.UploadId, + } + _, err = d.personalPost("/file/complete", data, nil) + if err != nil { + return err + } + } + + // 处理冲突 + if resp.Data.FileName != stream.GetName() { + log.Debugf("[139] conflict detected: %s != %s", resp.Data.FileName, stream.GetName()) + // 给服务器一定时间处理数据,避免无法刷新文件列表 + time.Sleep(time.Millisecond * 500) + // 刷新并获取文件列表 + files, err := d.List(ctx, dstDir, model.ListArgs{Refresh: true}) + if err != nil { + return err + } + // 删除旧文件 + for _, file := range files { + if file.GetName() == stream.GetName() { + log.Debugf("[139] conflict: removing old: %s", file.GetName()) + // 删除前重命名旧文件,避免仍旧冲突 + err = d.Rename(ctx, file, stream.GetName()+random.String(4)) + if err != nil { + return err + } + err = d.Remove(ctx, file) + if err != nil { + return err + } + break + } + } + // 重命名新文件 + for _, file := range files { + if file.GetName() == resp.Data.FileName { + log.Debugf("[139] conflict: renaming new: %s => %s", file.GetName(), stream.GetName()) + err = d.Rename(ctx, file, stream.GetName()) + if err != nil { + return err + } + break + } + } + } + return nil + case MetaPersonal: + fallthrough + case MetaFamily: + // 处理冲突 + // 获取文件列表 + files, err := d.List(ctx, dstDir, model.ListArgs{}) if err != nil { return err } - _ = res.Body.Close() - log.Debugf("%+v", res) - if res.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: %d", res.StatusCode) + // 删除旧文件 + for _, file := range files { + if file.GetName() == stream.GetName() { + log.Debugf("[139] conflict: removing old: %s", file.GetName()) + // 删除前重命名旧文件,避免仍旧冲突 + err = d.Rename(ctx, file, stream.GetName()+random.String(4)) + if err != nil { + return err + } + err = d.Remove(ctx, file) + if err != nil { + return err + } + break + } + } + var reportSize int64 + if d.ReportRealSize { + reportSize = stream.GetSize() + } else { + reportSize = 0 + } + data := base.Json{ + "manualRename": 2, + "operation": 0, + "fileCount": 1, + "totalSize": reportSize, + "uploadContentList": []base.Json{{ + "contentName": stream.GetName(), + "contentSize": reportSize, + // "digest": "5a3231986ce7a6b46e408612d385bafa" + }}, + "parentCatalogID": dstDir.GetID(), + "newCatalogName": "", + "commonAccountInfo": base.Json{ + "account": d.getAccount(), + "accountType": 1, + }, + } + pathname := "/orchestration/personalCloud/uploadAndDownload/v1.0/pcUploadFileRequest" + if d.isFamily() { + data = d.newJson(base.Json{ + "fileCount": 1, + "manualRename": 2, + "operation": 0, + "path": path.Join(dstDir.GetPath(), dstDir.GetID()), + "seqNo": random.String(32), //序列号不能为空 + "totalSize": reportSize, + "uploadContentList": []base.Json{{ + "contentName": stream.GetName(), + "contentSize": reportSize, + // "digest": "5a3231986ce7a6b46e408612d385bafa" + }}, + }) + pathname = "/orchestration/familyCloud-rebuild/content/v1.0/getFileUploadURL" } + var resp UploadResp + _, err = d.post(pathname, data, &resp) + if err != nil { + return err + } + if resp.Data.Result.ResultCode != "0" { + return fmt.Errorf("get file upload url failed with result code: %s, message: %s", resp.Data.Result.ResultCode, resp.Data.Result.ResultDesc) + } + + size := stream.GetSize() + // Progress + p := driver.NewProgress(size, up) + var partSize = d.getPartSize(size) + part := size / partSize + if size%partSize > 0 { + part++ + } else if part == 0 { + part = 1 + } + rateLimited := driver.NewLimitedUploadStream(ctx, stream) + for i := int64(0); i < part; i++ { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + + start := i * partSize + byteSize := min(size-start, partSize) + + limitReader := io.LimitReader(rateLimited, byteSize) + // Update Progress + r := io.TeeReader(limitReader, p) + req, err := http.NewRequest("POST", resp.Data.UploadResult.RedirectionURL, r) + if err != nil { + return err + } + + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "text/plain;name="+unicode(stream.GetName())) + req.Header.Set("contentSize", strconv.FormatInt(size, 10)) + req.Header.Set("range", fmt.Sprintf("bytes=%d-%d", start, start+byteSize-1)) + req.Header.Set("uploadtaskID", resp.Data.UploadResult.UploadTaskID) + req.Header.Set("rangeType", "0") + req.ContentLength = byteSize + + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + if res.StatusCode != http.StatusOK { + res.Body.Close() + return fmt.Errorf("unexpected status code: %d", res.StatusCode) + } + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("error reading response body: %v", err) + } + var result InterLayerUploadResult + err = xml.Unmarshal(bodyBytes, &result) + if err != nil { + return fmt.Errorf("error parsing XML: %v", err) + } + if result.ResultCode != 0 { + return fmt.Errorf("upload failed with result code: %d, message: %s", result.ResultCode, result.Msg) + } + } + return nil + default: + return errs.NotImplement } +} - return nil +func (d *Yun139) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch d.Addition.Type { + case MetaPersonalNew: + var resp base.Json + var uri string + data := base.Json{ + "category": "video", + "fileId": args.Obj.GetID(), + } + switch args.Method { + case "video_preview": + uri = "/videoPreview/getPreviewInfo" + default: + return nil, errs.NotSupport + } + _, err := d.personalPost(uri, data, &resp) + if err != nil { + return nil, err + } + return resp["data"], nil + default: + return nil, errs.NotImplement + } } var _ driver.Driver = (*Yun139)(nil) diff --git a/drivers/139/meta.go b/drivers/139/meta.go index 273ba14876d..ea90df86cf9 100644 --- a/drivers/139/meta.go +++ b/drivers/139/meta.go @@ -6,20 +6,26 @@ import ( ) type Addition struct { - //Account string `json:"account" required:"true"` Authorization string `json:"authorization" type:"text" required:"true"` driver.RootID - Type string `json:"type" type:"select" options:"personal,family" default:"personal"` - CloudID string `json:"cloud_id"` + Type string `json:"type" type:"select" options:"personal_new,family,group,personal,share" default:"personal_new"` + CloudID string `json:"cloud_id"` + LinkID string `json:"link_id"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0"` + ReportRealSize bool `json:"report_real_size" type:"bool" default:"true"` + UseLargeThumbnail bool `json:"use_large_thumbnail" type:"bool" default:"false"` } var config = driver.Config{ - Name: "139Yun", - LocalSort: true, + Name: "139Yun", + LocalSort: true, + ProxyRangeOption: true, } func init() { op.RegisterDriver(func() driver.Driver { - return &Yun139{} + d := &Yun139{} + d.ProxyRange = true + return d }) } diff --git a/drivers/139/types.go b/drivers/139/types.go index 217aeb9f497..03f88aa7aba 100644 --- a/drivers/139/types.go +++ b/drivers/139/types.go @@ -1,5 +1,16 @@ package _139 +import ( + "encoding/xml" +) + +const ( + MetaPersonal string = "personal" + MetaFamily string = "family" + MetaGroup string = "group" + MetaPersonalNew string = "personal_new" +) + type BaseResp struct { Success bool `json:"success"` Code string `json:"code"` @@ -44,6 +55,7 @@ type Content struct { //ContentDesc string `json:"contentDesc"` //ContentType int `json:"contentType"` //ContentOrigin int `json:"contentOrigin"` + CreateTime string `json:"createTime"` UpdateTime string `json:"updateTime"` //CommentCount int `json:"commentCount"` ThumbnailURL string `json:"thumbnailURL"` @@ -131,6 +143,13 @@ type UploadResp struct { } `json:"data"` } +type InterLayerUploadResult struct { + XMLName xml.Name `xml:"result"` + Text string `xml:",chardata"` + ResultCode int `xml:"resultCode"` + Msg string `xml:"msg"` +} + type CloudContent struct { ContentID string `json:"contentID"` //Modifier string `json:"modifier"` @@ -185,3 +204,149 @@ type QueryContentListResp struct { RecallContent interface{} `json:"recallContent"` } `json:"data"` } + +type QueryGroupContentListResp struct { + BaseResp + Data struct { + Result struct { + ResultCode string `json:"resultCode"` + ResultDesc string `json:"resultDesc"` + } `json:"result"` + GetGroupContentResult struct { + ParentCatalogID string `json:"parentCatalogID"` // 根目录是"0" + CatalogList []struct { + Catalog + Path string `json:"path"` + } `json:"catalogList"` + ContentList []Content `json:"contentList"` + NodeCount int `json:"nodeCount"` // 文件+文件夹数量 + CtlgCnt int `json:"ctlgCnt"` // 文件夹数量 + ContCnt int `json:"contCnt"` // 文件数量 + } `json:"getGroupContentResult"` + } `json:"data"` +} + +type ParallelHashCtx struct { + PartOffset int64 `json:"partOffset"` +} + +type PartInfo struct { + PartNumber int64 `json:"partNumber"` + PartSize int64 `json:"partSize"` + ParallelHashCtx ParallelHashCtx `json:"parallelHashCtx"` +} + +type PersonalThumbnail struct { + Style string `json:"style"` + Url string `json:"url"` +} + +type PersonalFileItem struct { + FileId string `json:"fileId"` + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Thumbnails []PersonalThumbnail `json:"thumbnailUrls"` +} + +type PersonalListResp struct { + BaseResp + Data struct { + Items []PersonalFileItem `json:"items"` + NextPageCursor string `json:"nextPageCursor"` + } +} + +type PersonalPartInfo struct { + PartNumber int `json:"partNumber"` + UploadUrl string `json:"uploadUrl"` +} + +type PersonalUploadResp struct { + BaseResp + Data struct { + FileId string `json:"fileId"` + FileName string `json:"fileName"` + PartInfos []PersonalPartInfo `json:"partInfos"` + Exist bool `json:"exist"` + RapidUpload bool `json:"rapidUpload"` + UploadId string `json:"uploadId"` + } +} + +type PersonalUploadUrlResp struct { + BaseResp + Data struct { + FileId string `json:"fileId"` + UploadId string `json:"uploadId"` + PartInfos []PersonalPartInfo `json:"partInfos"` + } +} + +type QueryRoutePolicyResp struct { + Success bool `json:"success"` + Code string `json:"code"` + Message string `json:"message"` + Data struct { + RoutePolicyList []struct { + SiteID string `json:"siteID"` + SiteCode string `json:"siteCode"` + ModName string `json:"modName"` + HttpUrl string `json:"httpUrl"` + HttpsUrl string `json:"httpsUrl"` + EnvID string `json:"envID"` + ExtInfo string `json:"extInfo"` + HashName string `json:"hashName"` + ModAddrType int `json:"modAddrType"` + } `json:"routePolicyList"` + } `json:"data"` +} + +type RefreshTokenResp struct { + XMLName xml.Name `xml:"root"` + Return string `xml:"return"` + Token string `xml:"token"` + Expiretime int32 `xml:"expiretime"` + AccessToken string `xml:"accessToken"` + Desc string `xml:"desc"` +} + +type ShareCatalog struct { + CaID string `json:"caID"` + CaName string `json:"caName"` + UdTime string `json:"udTime"` +} + +type ShareContent struct { + CoID string `json:"coID"` + CoName string `json:"coName"` + CoSize int64 `json:"coSize"` + CoType int `json:"coType"` + UdTime string `json:"udTime"` + CoSuffix string `json:"coSuffix"` +} + +type ShareListResp struct { + Data struct { + CaLst []ShareCatalog `json:"caLst"` + CoLst []ShareContent `json:"coLst"` + } `json:"data"` +} + +type ShareLinkResp struct { + DownloadURL string `json:"downloadURL"` +} +type ShareContentInfo struct { + ContentName string `json:"contentName"` + ContentSize int64 `json:"contentSize"` + PresentURL string `json:"presentURL"` // HLS 播放地址 + DownloadURL string `json:"cdnDownLoadUrl"` // 真正的下载直链 +} + +type ShareContentInfoResp struct { + Data struct { + ContentInfo ShareContentInfo `json:"contentInfo"` + } `json:"data"` +} \ No newline at end of file diff --git a/drivers/139/util.go b/drivers/139/util.go index 0f26b149955..3823569ad70 100644 --- a/drivers/139/util.go +++ b/drivers/139/util.go @@ -1,20 +1,32 @@ package _139 import ( + "bytes" + "compress/gzip" + "context" + "crypto/aes" + "crypto/cipher" + "crypto/rand" "encoding/base64" "errors" "fmt" + "io" "net/http" "net/url" + "path" "sort" "strconv" "strings" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" log "github.com/sirupsen/logrus" @@ -52,6 +64,56 @@ func getTime(t string) time.Time { return stamp } +func (d *Yun139) refreshToken() error { + if d.ref != nil { + return d.ref.refreshToken() + } + decode, err := base64.StdEncoding.DecodeString(d.Authorization) + if err != nil { + return fmt.Errorf("authorization decode failed: %s", err) + } + decodeStr := string(decode) + splits := strings.Split(decodeStr, ":") + if len(splits) < 3 { + return fmt.Errorf("authorization is invalid, splits < 3") + } + d.Account = splits[1] + strs := strings.Split(splits[2], "|") + if len(strs) < 4 { + return fmt.Errorf("authorization is invalid, strs < 4") + } + expiration, err := strconv.ParseInt(strs[3], 10, 64) + if err != nil { + return fmt.Errorf("authorization is invalid") + } + expiration -= time.Now().UnixMilli() + if expiration > 1000*60*60*24*15 { + // Authorization有效期大于15天无需刷新 + return nil + } + if expiration < 0 { + return fmt.Errorf("authorization has expired") + } + + url := "https://aas.caiyun.feixin.10086.cn:443/tellin/authTokenRefresh.do" + var resp RefreshTokenResp + reqBody := "" + splits[2] + "" + splits[1] + "656" + _, err = base.RestyClient.R(). + ForceContentType("application/xml"). + SetBody(reqBody). + SetResult(&resp). + Post(url) + if err != nil { + return err + } + if resp.Return != "0" { + return fmt.Errorf("failed to refresh token: %s", resp.Desc) + } + d.Authorization = base64.StdEncoding.EncodeToString([]byte(splits[0] + ":" + splits[1] + ":" + resp.Token)) + op.MustSaveDriverStorage(d) + return nil +} + func (d *Yun139) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { url := "https://yun.139.com" + pathname req := base.RestyClient.R() @@ -72,21 +134,22 @@ func (d *Yun139) request(pathname string, method string, callback base.ReqCallba req.SetHeaders(map[string]string{ "Accept": "application/json, text/plain, */*", "CMS-DEVICE": "default", - "Authorization": "Basic " + d.Authorization, + "Authorization": "Basic " + d.getAuthorization(), "mcloud-channel": "1000101", "mcloud-client": "10701", //"mcloud-route": "001", "mcloud-sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), //"mcloud-skey":"", - "mcloud-version": "6.6.0", - "Origin": "https://yun.139.com", - "Referer": "https://yun.139.com/w/", - "x-DeviceInfo": "||9|6.6.0|chrome|95.0.4638.69|uwIy75obnsRPIwlJSd7D9GhUvFwG96ce||macos 10.15.2||zh-CN|||", - "x-huawei-channelSrc": "10000034", - "x-inner-ntwk": "2", - "x-m4c-caller": "PC", - "x-m4c-src": "10002", - "x-SvcType": svcType, + "mcloud-version": "7.14.0", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/w/", + "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", + "x-huawei-channelSrc": "10000034", + "x-inner-ntwk": "2", + "x-m4c-caller": "PC", + "x-m4c-src": "10002", + "x-SvcType": svcType, + "Inner-Hcy-Router-Https": "1", }) var e BaseResp @@ -104,6 +167,104 @@ func (d *Yun139) request(pathname string, method string, callback base.ReqCallba } return res.Body(), nil } + +func (d *Yun139) requestRoute(data interface{}, resp interface{}) ([]byte, error) { + url := "https://user-njs.yun.139.com/user/route/qryRoutePolicy" + req := base.RestyClient.R() + randStr := random.String(16) + ts := time.Now().Format("2006-01-02 15:04:05") + callback := func(req *resty.Request) { + req.SetBody(data) + } + if callback != nil { + callback(req) + } + body, err := utils.Json.Marshal(req.Body) + if err != nil { + return nil, err + } + sign := calSign(string(body), ts, randStr) + svcType := "1" + if d.isFamily() { + svcType = "2" + } + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "CMS-DEVICE": "default", + "Authorization": "Basic " + d.getAuthorization(), + "mcloud-channel": "1000101", + "mcloud-client": "10701", + //"mcloud-route": "001", + "mcloud-sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), + //"mcloud-skey":"", + "mcloud-version": "7.14.0", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/w/", + "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", + "x-huawei-channelSrc": "10000034", + "x-inner-ntwk": "2", + "x-m4c-caller": "PC", + "x-m4c-src": "10002", + "x-SvcType": svcType, + "Inner-Hcy-Router-Https": "1", + }) + + var e BaseResp + req.SetResult(&e) + res, err := req.Execute(http.MethodPost, url) + log.Debugln(res.String()) + if !e.Success { + return nil, errors.New(e.Message) + } + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + return nil, err + } + } + return res.Body(), nil +} + +func (d *Yun139) ensurePersonalCloudHost() error { + if d.ref != nil { + return d.ref.ensurePersonalCloudHost() + } + if d.PersonalCloudHost != "" { + return nil + } + if len(d.Authorization) == 0 { + return fmt.Errorf("authorization is empty") + } + if d.Account == "" { + if err := d.refreshToken(); err != nil { + return err + } + } + + var resp QueryRoutePolicyResp + _, err := d.requestRoute(base.Json{ + "userInfo": base.Json{ + "userType": 1, + "accountType": 1, + "accountName": d.Account, + }, + "modAddrType": 1, + }, &resp) + if err != nil { + return err + } + for _, policyItem := range resp.Data.RoutePolicyList { + if policyItem.ModName == "personal" && policyItem.HttpsUrl != "" { + d.PersonalCloudHost = strings.TrimRight(policyItem.HttpsUrl, "/") + break + } + } + if d.PersonalCloudHost == "" { + return fmt.Errorf("personal cloud host is empty") + } + return nil +} + func (d *Yun139) post(pathname string, data interface{}, resp interface{}) ([]byte, error) { return d.request(pathname, http.MethodPost, func(req *resty.Request) { req.SetBody(data) @@ -124,7 +285,7 @@ func (d *Yun139) getFiles(catalogID string) ([]model.Obj, error) { "catalogSortType": 0, "contentSortType": 0, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -172,7 +333,7 @@ func (d *Yun139) newJson(data map[string]interface{}) base.Json { "cloudID": d.CloudID, "cloudType": 1, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -193,10 +354,11 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { "sortDirection": 1, }) var resp QueryContentListResp - _, err := d.post("/orchestration/familyCloud/content/v1.0/queryContentList", data, &resp) + _, err := d.post("/orchestration/familyCloud-rebuild/content/v1.2/queryContentList", data, &resp) if err != nil { return nil, err } + path := resp.Data.Path for _, catalog := range resp.Data.CloudCatalogList { f := model.Object{ ID: catalog.CatalogID, @@ -205,6 +367,7 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { IsFolder: true, Modified: getTime(catalog.LastUpdateTime), Ctime: getTime(catalog.CreateTime), + Path: path, // 文件夹上一级的Path } files = append(files, &f) } @@ -216,13 +379,14 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { Size: content.ContentSize, Modified: getTime(content.LastUpdateTime), Ctime: getTime(content.CreateTime), + Path: path, // 文件所在目录的Path }, Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL}, //Thumbnail: content.BigthumbnailURL, } files = append(files, &f) } - if 100*pageNum > resp.Data.TotalCount { + if resp.Data.TotalCount == 0 { break } pageNum++ @@ -230,12 +394,67 @@ func (d *Yun139) familyGetFiles(catalogID string) ([]model.Obj, error) { return files, nil } +func (d *Yun139) groupGetFiles(catalogID string) ([]model.Obj, error) { + pageNum := 1 + files := make([]model.Obj, 0) + for { + data := d.newJson(base.Json{ + "groupID": d.CloudID, + "catalogID": path.Base(catalogID), + "contentSortType": 0, + "sortDirection": 1, + "startNumber": pageNum, + "endNumber": pageNum + 99, + "path": path.Join(d.RootFolderID, catalogID), + }) + + var resp QueryGroupContentListResp + _, err := d.post("/orchestration/group-rebuild/content/v1.0/queryGroupContentList", data, &resp) + if err != nil { + return nil, err + } + path := resp.Data.GetGroupContentResult.ParentCatalogID + for _, catalog := range resp.Data.GetGroupContentResult.CatalogList { + f := model.Object{ + ID: catalog.CatalogID, + Name: catalog.CatalogName, + Size: 0, + IsFolder: true, + Modified: getTime(catalog.UpdateTime), + Ctime: getTime(catalog.CreateTime), + Path: catalog.Path, // 文件夹的真实Path, root:/开头 + } + files = append(files, &f) + } + for _, content := range resp.Data.GetGroupContentResult.ContentList { + f := model.ObjThumb{ + Object: model.Object{ + ID: content.ContentID, + Name: content.ContentName, + Size: content.ContentSize, + Modified: getTime(content.UpdateTime), + Ctime: getTime(content.CreateTime), + Path: path, // 文件所在目录的Path + }, + Thumbnail: model.Thumbnail{Thumbnail: content.ThumbnailURL}, + //Thumbnail: content.BigthumbnailURL, + } + files = append(files, &f) + } + if (pageNum + 99) > resp.Data.GetGroupContentResult.NodeCount { + break + } + pageNum = pageNum + 100 + } + return files, nil +} + func (d *Yun139) getLink(contentId string) (string, error) { data := base.Json{ "appName": "", "contentID": contentId, "commonAccountInfo": base.Json{ - "account": d.Account, + "account": d.getAccount(), "accountType": 1, }, } @@ -246,9 +465,546 @@ func (d *Yun139) getLink(contentId string) (string, error) { } return jsoniter.Get(res, "data", "downloadURL").ToString(), nil } +func (d *Yun139) familyGetLink(contentId string, path string) (string, error) { + data := d.newJson(base.Json{ + "contentID": contentId, + "path": path, + }) + res, err := d.post("/orchestration/familyCloud-rebuild/content/v1.0/getFileDownLoadURL", + data, nil) + if err != nil { + return "", err + } + return jsoniter.Get(res, "data", "downloadURL").ToString(), nil +} + +func (d *Yun139) groupGetLink(contentId string, path string) (string, error) { + data := d.newJson(base.Json{ + "contentID": contentId, + "groupID": d.CloudID, + "path": path, + }) + res, err := d.post("/orchestration/group-rebuild/groupManage/v1.0/getGroupFileDownLoadURL", + data, nil) + if err != nil { + return "", err + } + return jsoniter.Get(res, "data", "downloadURL").ToString(), nil +} func unicode(str string) string { textQuoted := strconv.QuoteToASCII(str) textUnquoted := textQuoted[1 : len(textQuoted)-1] return textUnquoted } + +func (d *Yun139) personalRequest(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + if err := d.ensurePersonalCloudHost(); err != nil { + return nil, err + } + url := d.getPersonalCloudHost() + pathname + req := base.RestyClient.R() + randStr := random.String(16) + ts := time.Now().Format("2006-01-02 15:04:05") + if callback != nil { + callback(req) + } + body, err := utils.Json.Marshal(req.Body) + if err != nil { + return nil, err + } + sign := calSign(string(body), ts, randStr) + svcType := "1" + if d.isFamily() { + svcType = "2" + } + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "Authorization": "Basic " + d.getAuthorization(), + "Caller": "web", + "Cms-Device": "default", + "Mcloud-Channel": "1000101", + "Mcloud-Client": "10701", + "Mcloud-Route": "001", + "Mcloud-Sign": fmt.Sprintf("%s,%s,%s", ts, randStr, sign), + "Mcloud-Version": "7.14.0", + "x-DeviceInfo": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||", + "x-huawei-channelSrc": "10000034", + "x-inner-ntwk": "2", + "x-m4c-caller": "PC", + "x-m4c-src": "10002", + "x-SvcType": svcType, + "X-Yun-Api-Version": "v1", + "X-Yun-App-Channel": "10000034", + "X-Yun-Channel-Source": "10000034", + "X-Yun-Client-Info": "||9|7.14.0|chrome|120.0.0.0|||windows 10||zh-CN|||dW5kZWZpbmVk||", + "X-Yun-Module-Type": "100", + "X-Yun-Svc-Type": "1", + }) + + var e BaseResp + req.SetResult(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + log.Debugln(res.String()) + if !e.Success { + return nil, errors.New(e.Message) + } + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + return nil, err + } + } + return res.Body(), nil +} +func (d *Yun139) personalPost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + return d.personalRequest(pathname, http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, resp) +} + +func getPersonalTime(t string) time.Time { + stamp, err := time.ParseInLocation("2006-01-02T15:04:05.999-07:00", t, utils.CNLoc) + if err != nil { + panic(err) + } + return stamp +} + +func (d *Yun139) personalGetFiles(fileId string) ([]model.Obj, error) { + files := make([]model.Obj, 0) + nextPageCursor := "" + for { + data := base.Json{ + "imageThumbnailStyleList": []string{"Small", "Large"}, + "orderBy": "updated_at", + "orderDirection": "DESC", + "pageInfo": base.Json{ + "pageCursor": nextPageCursor, + "pageSize": 100, + }, + "parentFileId": fileId, + } + var resp PersonalListResp + _, err := d.personalPost("/file/list", data, &resp) + if err != nil { + return nil, err + } + nextPageCursor = resp.Data.NextPageCursor + for _, item := range resp.Data.Items { + var isFolder = (item.Type == "folder") + var f model.Obj + if isFolder { + f = &model.Object{ + ID: item.FileId, + Name: item.Name, + Size: 0, + Modified: getPersonalTime(item.UpdatedAt), + Ctime: getPersonalTime(item.CreatedAt), + IsFolder: isFolder, + } + } else { + var Thumbnails = item.Thumbnails + var ThumbnailUrl string + if d.UseLargeThumbnail { + for _, thumb := range Thumbnails { + if strings.Contains(thumb.Style, "Large") { + ThumbnailUrl = thumb.Url + break + } + } + } + if ThumbnailUrl == "" && len(Thumbnails) > 0 { + ThumbnailUrl = Thumbnails[len(Thumbnails)-1].Url + } + f = &model.ObjThumb{ + Object: model.Object{ + ID: item.FileId, + Name: item.Name, + Size: item.Size, + Modified: getPersonalTime(item.UpdatedAt), + Ctime: getPersonalTime(item.CreatedAt), + IsFolder: isFolder, + }, + Thumbnail: model.Thumbnail{Thumbnail: ThumbnailUrl}, + } + } + files = append(files, f) + } + if len(nextPageCursor) == 0 { + break + } + } + return files, nil +} + +func (d *Yun139) personalGetLink(fileId string) (string, error) { + data := base.Json{ + "fileId": fileId, + } + res, err := d.personalPost("/file/getDownloadUrl", + data, nil) + if err != nil { + return "", err + } + var cdnUrl = jsoniter.Get(res, "data", "cdnUrl").ToString() + if cdnUrl != "" { + return cdnUrl, nil + } else { + return jsoniter.Get(res, "data", "url").ToString(), nil + } +} + +func (d *Yun139) getAuthorization() string { + if d.ref != nil { + return d.ref.getAuthorization() + } + return d.Authorization +} +func (d *Yun139) getAccount() string { + if d.ref != nil { + return d.ref.getAccount() + } + return d.Account +} +func (d *Yun139) getPersonalCloudHost() string { + if d.ref != nil { + return d.ref.getPersonalCloudHost() + } + return d.PersonalCloudHost +} + +func (d *Yun139) sharePost(pathname string, data interface{}, resp interface{}) ([]byte, error) { + crypto := NewYunCrypto() + encryptedBody, err := crypto.Encrypt(data) + if err != nil { + return nil, err + } + + url := "https://share-kd-njs.yun.139.com" + pathname + req := base.RestyClient.R() + + auth := d.getAuthorization() + if !strings.HasPrefix(auth, "Basic ") { + auth = "Basic " + auth + } + // randStr := random.String(16) + // ts := time.Now().Format("2006-01-02 15:04:05") + // body, err := utils.Json.Marshal(req.Body) + // if err != nil { + // return nil, err + // } + // sign := calSign(string(body), ts, randStr) + // svcType := "1" + // if d.isFamily() { + // svcType = "2" + // } + req.SetHeaders(map[string]string{ + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0", + "Accept": "application/json, text/plain, */*", + "Content-Type": "application/json;charset=UTF-8", + "Authorization": auth, + "X-Deviceinfo": "||9|12.27.0|firefox|140.0|12b780037221ab547c682223327dc9cd||linux unknow|1920X526|zh-CN|||", + "hcy-cool-flag": "1", + "CMS-DEVICE": "default", + "x-m4c-caller": "PC", + "X-Yun-Api-Version": "v1", + "Origin": "https://yun.139.com", + "Referer": "https://yun.139.com/", + }) + req.SetBody(encryptedBody) + + res, err := req.Post(url) + if err != nil { + return nil, err + } + + decryptedText, err := crypto.Decrypt(res.String()) + if err != nil { + log.Errorf("[139Share] Decryption failed, raw response: %s", res.String()) + return nil, fmt.Errorf("decryption failed: %v, raw: %s", err, res.String()) + } + + if resp != nil { + err = utils.Json.Unmarshal([]byte(decryptedText), resp) + if err != nil { + return nil, err + } + } + return []byte(decryptedText), nil +} + +func (d *Yun139) shareGetFiles(pCaID string) ([]model.Obj, error) { + if pCaID == "" { + pCaID = "root" + } + data := base.Json{ + "getOutLinkInfoReq": base.Json{ + "account": d.getAccount(), + "linkID": d.LinkID, + "pCaID": pCaID, + }, + } + var resp ShareListResp + _, err := d.sharePost("/yun-share/richlifeApp/devapp/IOutLink/getOutLinkInfoV6", data, &resp) + if err != nil { + return nil, err + } + files := make([]model.Obj, 0) + // 直接从 Data 中读取 CaLst + for _, catalog := range resp.Data.CaLst { + modTime, _ := time.ParseInLocation("20060102150405", catalog.UdTime, utils.CNLoc) + f := model.Object{ + ID: catalog.CaID, + Name: catalog.CaName, + Modified: modTime, + IsFolder: true, + } + files = append(files, &f) + } + for _, content := range resp.Data.CoLst { + name := content.CoName + size := content.CoSize + // Force .m3u8 suffix for videos and declare 1MB size for padding logic + if content.CoType == 3 || strings.HasSuffix(strings.ToLower(name), ".mp4") { + if !strings.HasSuffix(name, ".m3u8") { + name += ".m3u8" + } + size = 1024 * 1024 // Key: declare 1MB to match RangeReadCloser padding + } + modTime, _ := time.ParseInLocation("20060102150405", content.UdTime, utils.CNLoc) + f := model.Object{ + ID: content.CoID, + Name: name, + Size: size, + Modified: modTime, + } + files = append(files, &f) + } + + return files, nil +} + + + +type YunCrypto struct { + Key []byte + BlockSize int +} + +func NewYunCrypto() *YunCrypto { + return &YunCrypto{ + Key: []byte("PVGDwmcvfs1uV3d1"), + BlockSize: aes.BlockSize, + } +} + +func (y *YunCrypto) PKCS7Padding(ciphertext []byte, blockSize int) []byte { + padding := blockSize - len(ciphertext)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(ciphertext, padtext...) +} + +func (y *YunCrypto) PKCS7UnPadding(origData []byte) ([]byte, error) { + length := len(origData) + if length == 0 { + return nil, errors.New("data is empty") + } + unpadding := int(origData[length-1]) + if length < unpadding { + return nil, errors.New("unpadding error") + } + return origData[:(length - unpadding)], nil +} + +func (y *YunCrypto) Encrypt(data interface{}) (string, error) { + jsonData, err := utils.Json.Marshal(data) + if err != nil { + return "", err + } + iv := make([]byte, y.BlockSize) + if _, err := io.ReadFull(rand.Reader, iv); err != nil { + return "", err + } + block, err := aes.NewCipher(y.Key) + if err != nil { + return "", err + } + content := y.PKCS7Padding(jsonData, y.BlockSize) + ciphertext := make([]byte, len(content)) + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(ciphertext, content) + result := append(iv, ciphertext...) + return base64.StdEncoding.EncodeToString(result), nil +} + +func (y *YunCrypto) Decrypt(b64Data string) (string, error) { + b64Data = strings.Join(strings.Fields(b64Data), "") + raw, err := base64.StdEncoding.DecodeString(b64Data) + if err != nil { + return "", err + } + if len(raw) < y.BlockSize { + return "", errors.New("data too short") + } + iv := raw[:y.BlockSize] + ciphertext := raw[y.BlockSize:] + block, err := aes.NewCipher(y.Key) + if err != nil { + return "", err + } + decrypted := make([]byte, len(ciphertext)) + mode := cipher.NewCBCDecrypter(block, iv) + mode.CryptBlocks(decrypted, ciphertext) + if len(decrypted) > 2 && decrypted[0] == 0x1f && decrypted[1] == 0x8b { + reader, err := gzip.NewReader(bytes.NewReader(decrypted)) + if err == nil { + defer reader.Close() + unzipped, err := io.ReadAll(reader) + if err == nil { + return string(unzipped), nil + } + } + } + unpadded, err := y.PKCS7UnPadding(decrypted) + if err != nil { + return strings.TrimSpace(string(decrypted)), nil + } + return string(unpadded), nil +} + +func (d *Yun139) rewriteM3U8(masterURL string) (string, error) { + client := resty.New().SetTimeout(10 * time.Second) + headers := map[string]string{ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Referer": "https://yun.139.com/", + } + + // 1. Get Master M3U8 + resp, err := client.R().SetHeaders(headers).Get(masterURL) + if err != nil { + return "", err + } + masterContent := resp.String() + + // 2. Find sub-playlist path + var subRelPath string + lines := strings.Split(masterContent, "\n") + for i, line := range lines { + if strings.Contains(line, "RESOLUTION=") { + if i+1 < len(lines) { + subRelPath = strings.TrimSpace(lines[i+1]) + if strings.Contains(line, "1920x1080") { + break + } + } + } + } + if subRelPath == "" { + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line != "" && !strings.HasPrefix(line, "#") { + subRelPath = line + break + } + } + } + if subRelPath == "" { + return "", fmt.Errorf("sub playlist not found in master m3u8") + } + + // 3. Get sub-playlist content + base, _ := url.Parse(masterURL) + ref, _ := url.Parse(subRelPath) + subURL := base.ResolveReference(ref).String() + + resp, err = client.R().SetHeaders(headers).Get(subURL) + if err != nil { + return "", err + } + subContent := resp.String() + + // 4. Resolve relative TS paths to absolute URLs + subBase, _ := url.Parse(subURL) + subLines := strings.Split(subContent, "\n") + var finalLines []string + for _, line := range subLines { + cleanLine := strings.TrimSpace(line) + if cleanLine != "" && !strings.HasPrefix(cleanLine, "#") { + if !strings.HasPrefix(cleanLine, "http") { + tsRef, _ := url.Parse(cleanLine) + finalLines = append(finalLines, subBase.ResolveReference(tsRef).String()) + } else { + finalLines = append(finalLines, cleanLine) + } + } else { + finalLines = append(finalLines, line) + } + } + + finalM3U8 := strings.Join(finalLines, "\n") + + return finalM3U8, nil +} + +func (d *Yun139) Proxy(c *gin.Context, obj model.Obj) error { + return nil +} + +func (d *Yun139) shareGetLink(coID string) (*model.Link, error) { + data := base.Json{ + "getContentInfoFromOutLinkReq": base.Json{ + "contentId": coID, + "linkID": d.LinkID, + "account": d.getAccount(), + }, + } + var resp ShareContentInfoResp + _, err := d.sharePost("/yun-share/richlifeApp/devapp/IOutLink/getContentInfoFromOutLink", data, &resp) + if err != nil { + return nil, err + } + + res := resp.Data.ContentInfo + if res.PresentURL != "" { + m3u8Content, err := d.rewriteM3U8(res.PresentURL) + if err != nil { + return nil, err + } + + // Core logic: pad to 1MB to ensure compatibility with AList's size validation + targetSize := int64(1024 * 1024) + contentBytes := []byte(m3u8Content) + if int64(len(contentBytes)) < targetSize { + padding := bytes.Repeat([]byte(" "), int(targetSize-int64(len(contentBytes)))) + contentBytes = append(contentBytes, padding...) + } else { + // Truncate if M3U8 exceeds 1MB (extremely rare) + contentBytes = contentBytes[:targetSize] + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, range_ http_range.Range) (io.ReadCloser, error) { + reader := bytes.NewReader(contentBytes) + // Handle AList Range requests + _, _ = reader.Seek(range_.Start, io.SeekStart) + // Wrap as ReadCloser + return io.NopCloser(reader), nil + }, + }, + Header: http.Header{ + "Content-Type": []string{"application/vnd.apple.mpegurl"}, + }, + }, nil + } + + if res.DownloadURL != "" { + return &model.Link{URL: res.DownloadURL}, nil + } + + return nil, fmt.Errorf("failed to get link") +} diff --git a/drivers/189/driver.go b/drivers/189/driver.go index 6fc4932640c..0c8db1134ee 100644 --- a/drivers/189/driver.go +++ b/drivers/189/driver.go @@ -80,9 +80,10 @@ func (d *Cloud189) Link(ctx context.Context, file model.Obj, args model.LinkArgs } func (d *Cloud189) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + safeName := d.sanitizeName(dirName) form := map[string]string{ "parentFolderId": parentDir.GetID(), - "folderName": dirName, + "folderName": safeName, } _, err := d.request("https://cloud.189.cn/api/open/file/createFolder.action", http.MethodPost, func(req *resty.Request) { req.SetFormData(form) @@ -126,9 +127,10 @@ func (d *Cloud189) Rename(ctx context.Context, srcObj model.Obj, newName string) idKey = "folderId" nameKey = "destFolderName" } + safeName := d.sanitizeName(newName) form := map[string]string{ idKey: srcObj.GetID(), - nameKey: newName, + nameKey: safeName, } _, err := d.request(url, http.MethodPost, func(req *resty.Request) { req.SetFormData(form) diff --git a/drivers/189/meta.go b/drivers/189/meta.go index ad621fb440d..da81406e683 100644 --- a/drivers/189/meta.go +++ b/drivers/189/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + Cookie string `json:"cookie" help:"Fill in the cookie if need captcha"` + StripEmoji bool `json:"strip_emoji" help:"Remove four-byte characters (e.g., emoji) before upload"` driver.RootID } diff --git a/drivers/189/util.go b/drivers/189/util.go index 680ce252133..ee2b5061dd8 100644 --- a/drivers/189/util.go +++ b/drivers/189/util.go @@ -11,9 +11,11 @@ import ( "io" "math" "net/http" + "path" "strconv" "strings" "time" + "unicode/utf8" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -222,13 +224,37 @@ func (d *Cloud189) getFiles(fileId string) ([]model.Obj, error) { return res, nil } +func (d *Cloud189) sanitizeName(name string) string { + if !d.StripEmoji { + return name + } + b := strings.Builder{} + for _, r := range name { + if utf8.RuneLen(r) == 4 { + continue + } + b.WriteRune(r) + } + sanitized := b.String() + if sanitized == "" { + ext := path.Ext(name) + if ext != "" { + sanitized = "file" + ext + } else { + sanitized = "file" + } + } + return sanitized +} + func (d *Cloud189) oldUpload(dstDir model.Obj, file model.FileStreamer) error { + safeName := d.sanitizeName(file.GetName()) res, err := d.client.R().SetMultipartFormData(map[string]string{ "parentId": dstDir.GetID(), "sessionKey": "??", "opertype": "1", - "fname": file.GetName(), - }).SetMultipartField("Filedata", file.GetName(), file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") + "fname": safeName, + }).SetMultipartField("Filedata", safeName, file.GetMimetype(), file).Post("https://hb02.upload.cloud.189.cn/v1/DCIWebUploadAction") if err != nil { return err } @@ -313,9 +339,10 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F const DEFAULT int64 = 10485760 var count = int64(math.Ceil(float64(file.GetSize()) / float64(DEFAULT))) + safeName := d.sanitizeName(file.GetName()) res, err := d.uploadRequest("/person/initMultiUpload", map[string]string{ "parentFolderId": dstDir.GetID(), - "fileName": encode(file.GetName()), + "fileName": encode(safeName), "fileSize": strconv.FormatInt(file.GetSize(), 10), "sliceSize": strconv.FormatInt(DEFAULT, 10), "lazyCheck": "1", @@ -365,7 +392,7 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F log.Debugf("uploadData: %+v", uploadData) requestURL := uploadData.RequestURL uploadHeaders := strings.Split(decodeURIComponent(uploadData.RequestHeader), "&") - req, err := http.NewRequest(http.MethodPut, requestURL, bytes.NewReader(byteData)) + req, err := http.NewRequest(http.MethodPut, requestURL, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { return err } @@ -375,12 +402,12 @@ func (d *Cloud189) newUpload(ctx context.Context, dstDir model.Obj, file model.F req.Header.Set(v[0:i], v[i+1:]) } r, err := base.HttpClient.Do(req) - log.Debugf("%+v %+v", r, r.Request.Header) - r.Body.Close() if err != nil { return err } - up(int(i * 100 / count)) + log.Debugf("%+v %+v", r, r.Request.Header) + _ = r.Body.Close() + up(float64(i) * 100 / float64(count)) } fileMd5 := hex.EncodeToString(md5Sum.Sum(nil)) sliceMd5 := fileMd5 diff --git a/drivers/189pc/driver.go b/drivers/189pc/driver.go index f0977995105..9462cef6662 100644 --- a/drivers/189pc/driver.go +++ b/drivers/189pc/driver.go @@ -2,6 +2,7 @@ package _189pc import ( "context" + "fmt" "net/http" "strconv" "strings" @@ -13,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + "github.com/google/uuid" ) type Cloud189PC struct { @@ -28,7 +30,11 @@ type Cloud189PC struct { uploadThread int + familyTransferFolder *Cloud189Folder + cleanFamilyTransferFile func() + storageConfig driver.Config + ref *Cloud189PC } func (y *Cloud189PC) Config() driver.Config { @@ -43,16 +49,24 @@ func (y *Cloud189PC) GetAddition() driver.Additional { } func (y *Cloud189PC) Init(ctx context.Context) (err error) { - // 兼容旧上传接口 - y.storageConfig.NoOverwriteUpload = y.isFamily() && (y.Addition.RapidUpload || y.Addition.UploadMethod == "old") - + y.storageConfig = config + if y.isFamily() { + // 兼容旧上传接口 + if y.Addition.RapidUpload || y.Addition.UploadMethod == "old" { + y.storageConfig.NoOverwriteUpload = true + } + } else { + // 家庭云转存,不支持覆盖上传 + if y.Addition.FamilyTransfer { + y.storageConfig.NoOverwriteUpload = true + } + } // 处理个人云和家庭云参数 if y.isFamily() && y.RootFolderID == "-11" { y.RootFolderID = "" } if !y.isFamily() && y.RootFolderID == "" { y.RootFolderID = "-11" - y.FamilyID = "" } // 限制上传线程数 @@ -61,38 +75,64 @@ func (y *Cloud189PC) Init(ctx context.Context) (err error) { y.uploadThread, y.UploadThread = 3, "3" } - // 初始化请求客户端 - if y.client == nil { - y.client = base.NewRestyClient().SetHeaders(map[string]string{ - "Accept": "application/json;charset=UTF-8", - "Referer": WEB_URL, - }) - } + if y.ref == nil { + // 初始化请求客户端 + if y.client == nil { + y.client = base.NewRestyClient().SetHeaders(map[string]string{ + "Accept": "application/json;charset=UTF-8", + "Referer": WEB_URL, + }) + } - // 避免重复登陆 - identity := utils.GetMD5EncodeStr(y.Username + y.Password) - if !y.isLogin() || y.identity != identity { - y.identity = identity - if err = y.login(); err != nil { - return + // 避免重复登陆 + identity := utils.GetMD5EncodeStr(y.Username + y.Password) + if !y.isLogin() || y.identity != identity { + y.identity = identity + if err = y.login(); err != nil { + return + } } } // 处理家庭云ID - if y.isFamily() && y.FamilyID == "" { + if y.FamilyID == "" { if y.FamilyID, err = y.getFamilyID(); err != nil { return err } } + + // 创建中转文件夹 + if y.FamilyTransfer { + if err := y.createFamilyTransferFolder(); err != nil { + return err + } + } + + // 清理转存文件节流 + y.cleanFamilyTransferFile = utils.NewThrottle2(time.Minute, func() { + if err := y.cleanFamilyTransfer(context.TODO()); err != nil { + utils.Log.Errorf("cleanFamilyTransferFolderError:%s", err) + } + }) return } +func (d *Cloud189PC) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*Cloud189PC) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + func (y *Cloud189PC) Drop(ctx context.Context) error { + y.ref = nil return nil } func (y *Cloud189PC) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - return y.getFiles(ctx, dir.GetID()) + return y.getFiles(ctx, dir.GetID(), y.isFamily()) } func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -100,8 +140,9 @@ func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkAr URL string `json:"fileDownloadUrl"` } + isFamily := y.isFamily() fullUrl := API_URL - if y.isFamily() { + if isFamily { fullUrl += "/family/file" } fullUrl += "/getFileDownloadUrl.action" @@ -109,7 +150,7 @@ func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkAr _, err := y.get(fullUrl, func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParam("fileId", file.GetID()) - if y.isFamily() { + if isFamily { r.SetQueryParams(map[string]string{ "familyId": y.FamilyID, }) @@ -119,7 +160,7 @@ func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkAr "flag": "1", }) } - }, &downloadUrl) + }, &downloadUrl, isFamily) if err != nil { return nil, err } @@ -156,20 +197,22 @@ func (y *Cloud189PC) Link(ctx context.Context, file model.Obj, args model.LinkAr } func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + isFamily := y.isFamily() fullUrl := API_URL - if y.isFamily() { + if isFamily { fullUrl += "/family/file" } fullUrl += "/createFolder.action" var newFolder Cloud189Folder + safeName := y.sanitizeName(dirName) _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) req.SetQueryParams(map[string]string{ - "folderName": dirName, + "folderName": safeName, "relativePath": "", }) - if y.isFamily() { + if isFamily { req.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "parentId": parentDir.GetID(), @@ -179,35 +222,23 @@ func (y *Cloud189PC) MakeDir(ctx context.Context, parentDir model.Obj, dirName s "parentFolderId": parentDir.GetID(), }) } - }, &newFolder) + }, &newFolder, isFamily) if err != nil { return nil, err } + newFolder.Name = safeName return &newFolder, nil } func (y *Cloud189PC) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - var resp CreateBatchTaskResp - _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { - req.SetContext(ctx) - req.SetFormData(map[string]string{ - "type": "MOVE", - "taskInfos": MustString(utils.Json.MarshalToString( - []BatchTaskInfo{ - { - FileId: srcObj.GetID(), - FileName: srcObj.GetName(), - IsFolder: BoolToNumber(srcObj.IsDir()), - }, - })), - "targetFolderId": dstDir.GetID(), - }) - if y.isFamily() { - req.SetFormData(map[string]string{ - "familyId": y.FamilyID, - }) - } - }, &resp) + isFamily := y.isFamily() + other := map[string]string{"targetFileName": dstDir.GetName()} + + resp, err := y.CreateBatchTask("MOVE", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), other, BatchTaskInfo{ + FileId: srcObj.GetID(), + FileName: srcObj.GetName(), + IsFolder: BoolToNumber(srcObj.IsDir()), + }) if err != nil { return nil, err } @@ -218,34 +249,43 @@ func (y *Cloud189PC) Move(ctx context.Context, srcObj, dstDir model.Obj) (model. } func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + isFamily := y.isFamily() queryParam := make(map[string]string) fullUrl := API_URL method := http.MethodPost - if y.isFamily() { + if isFamily { fullUrl += "/family/file" method = http.MethodGet queryParam["familyId"] = y.FamilyID } var newObj model.Obj + safeName := y.sanitizeName(newName) switch f := srcObj.(type) { case *Cloud189File: fullUrl += "/renameFile.action" queryParam["fileId"] = srcObj.GetID() - queryParam["destFileName"] = newName + queryParam["destFileName"] = safeName newObj = &Cloud189File{Icon: f.Icon} // 复用预览 case *Cloud189Folder: fullUrl += "/renameFolder.action" queryParam["folderId"] = srcObj.GetID() - queryParam["destFolderName"] = newName + queryParam["destFolderName"] = safeName newObj = &Cloud189Folder{} default: return nil, errs.NotSupport } + switch obj := newObj.(type) { + case *Cloud189File: + obj.Name = safeName + case *Cloud189Folder: + obj.Name = safeName + } + _, err := y.request(fullUrl, method, func(req *resty.Request) { req.SetContext(ctx).SetQueryParams(queryParam) - }, nil, newObj) + }, nil, newObj, isFamily) if err != nil { return nil, err } @@ -253,28 +293,15 @@ func (y *Cloud189PC) Rename(ctx context.Context, srcObj model.Obj, newName strin } func (y *Cloud189PC) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - var resp CreateBatchTaskResp - _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { - req.SetContext(ctx) - req.SetFormData(map[string]string{ - "type": "COPY", - "taskInfos": MustString(utils.Json.MarshalToString( - []BatchTaskInfo{ - { - FileId: srcObj.GetID(), - FileName: srcObj.GetName(), - IsFolder: BoolToNumber(srcObj.IsDir()), - }, - })), - "targetFolderId": dstDir.GetID(), - "targetFileName": dstDir.GetName(), - }) - if y.isFamily() { - req.SetFormData(map[string]string{ - "familyId": y.FamilyID, - }) - } - }, &resp) + isFamily := y.isFamily() + other := map[string]string{"targetFileName": dstDir.GetName()} + + resp, err := y.CreateBatchTask("COPY", IF(isFamily, y.FamilyID, ""), dstDir.GetID(), other, BatchTaskInfo{ + FileId: srcObj.GetID(), + FileName: srcObj.GetName(), + IsFolder: BoolToNumber(srcObj.IsDir()), + }) + if err != nil { return err } @@ -282,27 +309,13 @@ func (y *Cloud189PC) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { - var resp CreateBatchTaskResp - _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { - req.SetContext(ctx) - req.SetFormData(map[string]string{ - "type": "DELETE", - "taskInfos": MustString(utils.Json.MarshalToString( - []*BatchTaskInfo{ - { - FileId: obj.GetID(), - FileName: obj.GetName(), - IsFolder: BoolToNumber(obj.IsDir()), - }, - })), - }) + isFamily := y.isFamily() - if y.isFamily() { - req.SetFormData(map[string]string{ - "familyId": y.FamilyID, - }) - } - }, &resp) + resp, err := y.CreateBatchTask("DELETE", IF(isFamily, y.FamilyID, ""), "", nil, BatchTaskInfo{ + FileId: obj.GetID(), + FileName: obj.GetName(), + IsFolder: BoolToNumber(obj.IsDir()), + }) if err != nil { return err } @@ -310,25 +323,87 @@ func (y *Cloud189PC) Remove(ctx context.Context, obj model.Obj) error { return y.WaitBatchTask("DELETE", resp.TaskID, time.Millisecond*200) } -func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (y *Cloud189PC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (newObj model.Obj, err error) { + overwrite := true + isFamily := y.isFamily() + // 响应时间长,按需启用 - if y.Addition.RapidUpload { - if newObj, err := y.RapidUpload(ctx, dstDir, stream); err == nil { + if y.Addition.RapidUpload && !stream.IsForceStreamUpload() { + if newObj, err := y.RapidUpload(ctx, dstDir, stream, isFamily, overwrite); err == nil { return newObj, nil } } - switch y.UploadMethod { - case "old": - return y.OldUpload(ctx, dstDir, stream, up) + uploadMethod := y.UploadMethod + if stream.IsForceStreamUpload() { + uploadMethod = "stream" + } + + // 旧版上传家庭云也有限制 + if uploadMethod == "old" { + return y.OldUpload(ctx, dstDir, stream, up, isFamily, overwrite) + } + + // 开启家庭云转存 + if !isFamily && y.FamilyTransfer { + // 修改上传目标为家庭云文件夹 + transferDstDir := dstDir + dstDir = y.familyTransferFolder + + // 使用临时文件名 + srcName := stream.GetName() + stream = &WrapFileStreamer{ + FileStreamer: stream, + Name: fmt.Sprintf("0%s.transfer", uuid.NewString()), + } + + // 使用家庭云上传 + isFamily = true + overwrite = false + + defer func() { + if newObj != nil { + // 转存家庭云文件到个人云 + err = y.SaveFamilyFileToPersonCloud(context.TODO(), y.FamilyID, newObj, transferDstDir, true) + // 删除家庭云源文件 + go y.Delete(context.TODO(), y.FamilyID, newObj) + // 批量任务有概率删不掉 + go y.cleanFamilyTransferFile() + // 转存失败返回错误 + if err != nil { + return + } + + // 查找转存文件 + var file *Cloud189File + file, err = y.findFileByName(context.TODO(), newObj.GetName(), transferDstDir.GetID(), false) + if err != nil { + if err == errs.ObjectNotFound { + err = fmt.Errorf("unknown error: No transfer file obtained %s", newObj.GetName()) + } + return + } + + // 重命名转存文件 + newObj, err = y.Rename(context.TODO(), file, srcName) + if err != nil { + // 重命名失败删除源文件 + _ = y.Delete(context.TODO(), "", file) + } + return + } + }() + } + + switch uploadMethod { case "rapid": - return y.FastUpload(ctx, dstDir, stream, up) + return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) case "stream": if stream.GetSize() == 0 { - return y.FastUpload(ctx, dstDir, stream, up) + return y.FastUpload(ctx, dstDir, stream, up, isFamily, overwrite) } fallthrough default: - return y.StreamUpload(ctx, dstDir, stream, up) + return y.StreamUpload(ctx, dstDir, stream, up, isFamily, overwrite) } } diff --git a/drivers/189pc/help.go b/drivers/189pc/help.go index ba1f3f08e39..bac8880a897 100644 --- a/drivers/189pc/help.go +++ b/drivers/189pc/help.go @@ -18,6 +18,7 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils/random" ) @@ -192,3 +193,28 @@ func partSize(size int64) int64 { } return DEFAULT } + +func isBool(bs ...bool) bool { + for _, b := range bs { + if b { + return true + } + } + return false +} + +func IF[V any](o bool, t V, f V) V { + if o { + return t + } + return f +} + +type WrapFileStreamer struct { + model.FileStreamer + Name string +} + +func (w *WrapFileStreamer) GetName() string { + return w.Name +} diff --git a/drivers/189pc/meta.go b/drivers/189pc/meta.go index 079ac7cc2b9..d6edc063593 100644 --- a/drivers/189pc/meta.go +++ b/drivers/189pc/meta.go @@ -6,9 +6,10 @@ import ( ) type Addition struct { - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - VCode string `json:"validate_code"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + VCode string `json:"validate_code"` + StripEmoji bool `json:"strip_emoji" help:"Remove four-byte characters (e.g., emoji) before upload"` driver.RootID OrderBy string `json:"order_by" type:"select" options:"filename,filesize,lastOpTime" default:"filename"` OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` @@ -16,6 +17,7 @@ type Addition struct { FamilyID string `json:"family_id"` UploadMethod string `json:"upload_method" type:"select" options:"stream,rapid,old" default:"stream"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + FamilyTransfer bool `json:"family_transfer"` RapidUpload bool `json:"rapid_upload"` NoUseOcr bool `json:"no_use_ocr"` } diff --git a/drivers/189pc/types.go b/drivers/189pc/types.go index d779659ed16..2e9ed4c203d 100644 --- a/drivers/189pc/types.go +++ b/drivers/189pc/types.go @@ -3,10 +3,11 @@ package _189pc import ( "encoding/xml" "fmt" - "github.com/alist-org/alist/v3/pkg/utils" "sort" "strings" "time" + + "github.com/alist-org/alist/v3/pkg/utils" ) // 居然有四种返回方式 @@ -142,7 +143,7 @@ type FamilyInfoListResp struct { type FamilyInfoResp struct { Count int `json:"count"` CreateTime string `json:"createTime"` - FamilyID int `json:"familyId"` + FamilyID int64 `json:"familyId"` RemarkName string `json:"remarkName"` Type int `json:"type"` UseFlag int `json:"useFlag"` @@ -242,7 +243,12 @@ type BatchTaskInfo struct { // IsFolder 是否是文件夹,0-否,1-是 IsFolder int `json:"isFolder"` // SrcParentId 文件所在父目录ID - //SrcParentId string `json:"srcParentId"` + SrcParentId string `json:"srcParentId,omitempty"` + + /* 冲突管理 */ + // 1 -> 跳过 2 -> 保留 3 -> 覆盖 + DealWay int `json:"dealWay,omitempty"` + IsConflict int `json:"isConflict,omitempty"` } /* 上传部分 */ @@ -355,6 +361,14 @@ type BatchTaskStateResp struct { TaskStatus int `json:"taskStatus"` //1 初始化 2 存在冲突 3 执行中,4 完成 } +type BatchTaskConflictTaskInfoResp struct { + SessionKey string `json:"sessionKey"` + TargetFolderID int `json:"targetFolderId"` + TaskID string `json:"taskId"` + TaskInfos []BatchTaskInfo + TaskType int `json:"taskType"` +} + /* query 加密参数*/ type Params map[string]string diff --git a/drivers/189pc/utils.go b/drivers/189pc/utils.go index 1868aeb25ff..ca89251e278 100644 --- a/drivers/189pc/utils.go +++ b/drivers/189pc/utils.go @@ -3,28 +3,33 @@ package _189pc import ( "bytes" "context" - "crypto/md5" "encoding/base64" "encoding/hex" "encoding/xml" "fmt" "io" - "math" "net/http" "net/http/cookiejar" "net/url" + "os" + "path" "regexp" "sort" "strconv" "strings" "time" + "unicode/utf8" + + "golang.org/x/sync/semaphore" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/errgroup" "github.com/alist-org/alist/v3/pkg/utils" @@ -54,13 +59,36 @@ const ( CHANNEL_ID = "web_cloud.189.cn" ) -func (y *Cloud189PC) SignatureHeader(url, method, params string) map[string]string { +func (y *Cloud189PC) sanitizeName(name string) string { + if !y.StripEmoji { + return name + } + b := strings.Builder{} + for _, r := range name { + if utf8.RuneLen(r) == 4 { + continue + } + b.WriteRune(r) + } + sanitized := b.String() + if sanitized == "" { + ext := path.Ext(name) + if ext != "" { + sanitized = "file" + ext + } else { + sanitized = "file" + } + } + return sanitized +} + +func (y *Cloud189PC) SignatureHeader(url, method, params string, isFamily bool) map[string]string { dateOfGmt := getHttpDateStr() - sessionKey := y.tokenInfo.SessionKey - sessionSecret := y.tokenInfo.SessionSecret - if y.isFamily() { - sessionKey = y.tokenInfo.FamilySessionKey - sessionSecret = y.tokenInfo.FamilySessionSecret + sessionKey := y.getTokenInfo().SessionKey + sessionSecret := y.getTokenInfo().SessionSecret + if isFamily { + sessionKey = y.getTokenInfo().FamilySessionKey + sessionSecret = y.getTokenInfo().FamilySessionSecret } header := map[string]string{ @@ -72,10 +100,10 @@ func (y *Cloud189PC) SignatureHeader(url, method, params string) map[string]stri return header } -func (y *Cloud189PC) EncryptParams(params Params) string { - sessionSecret := y.tokenInfo.SessionSecret - if y.isFamily() { - sessionSecret = y.tokenInfo.FamilySessionSecret +func (y *Cloud189PC) EncryptParams(params Params, isFamily bool) string { + sessionSecret := y.getTokenInfo().SessionSecret + if isFamily { + sessionSecret = y.getTokenInfo().FamilySessionSecret } if params != nil { return AesECBEncrypt(params.Encode(), sessionSecret[:16]) @@ -83,17 +111,17 @@ func (y *Cloud189PC) EncryptParams(params Params) string { return "" } -func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, params Params, resp interface{}) ([]byte, error) { - req := y.client.R().SetQueryParams(clientSuffix()) +func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, params Params, resp interface{}, isFamily ...bool) ([]byte, error) { + req := y.getClient().R().SetQueryParams(clientSuffix()) // 设置params - paramsData := y.EncryptParams(params) + paramsData := y.EncryptParams(params, isBool(isFamily...)) if paramsData != "" { req.SetQueryParam("params", paramsData) } // Signature - req.SetHeaders(y.SignatureHeader(url, method, paramsData)) + req.SetHeaders(y.SignatureHeader(url, method, paramsData, isBool(isFamily...))) var erron RespErr req.SetError(&erron) @@ -113,31 +141,33 @@ func (y *Cloud189PC) request(url, method string, callback base.ReqCallback, para if err = y.refreshSession(); err != nil { return nil, err } - return y.request(url, method, callback, params, resp) + return y.request(url, method, callback, params, resp, isFamily...) + } + + // if erron.ErrorCode == "InvalidSessionKey" || erron.Code == "InvalidSessionKey" { + if strings.Contains(res.String(), "InvalidSessionKey") { + if err = y.refreshSession(); err != nil { + return nil, err + } + return y.request(url, method, callback, params, resp, isFamily...) } // 处理错误 if erron.HasError() { - if erron.ErrorCode == "InvalidSessionKey" { - if err = y.refreshSession(); err != nil { - return nil, err - } - return y.request(url, method, callback, params, resp) - } return nil, &erron } return res.Body(), nil } -func (y *Cloud189PC) get(url string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - return y.request(url, http.MethodGet, callback, nil, resp) +func (y *Cloud189PC) get(url string, callback base.ReqCallback, resp interface{}, isFamily ...bool) ([]byte, error) { + return y.request(url, http.MethodGet, callback, nil, resp, isFamily...) } -func (y *Cloud189PC) post(url string, callback base.ReqCallback, resp interface{}) ([]byte, error) { - return y.request(url, http.MethodPost, callback, nil, resp) +func (y *Cloud189PC) post(url string, callback base.ReqCallback, resp interface{}, isFamily ...bool) ([]byte, error) { + return y.request(url, http.MethodPost, callback, nil, resp, isFamily...) } -func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]string, sign bool, file io.Reader) ([]byte, error) { +func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]string, sign bool, file io.Reader, isFamily bool) ([]byte, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPut, url, file) if err != nil { return nil, err @@ -154,7 +184,7 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str } if sign { - for key, value := range y.SignatureHeader(url, http.MethodPut, "") { + for key, value := range y.SignatureHeader(url, http.MethodPut, "", isFamily) { req.Header.Add(key, value) } } @@ -171,8 +201,8 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str } var erron RespErr - jsoniter.Unmarshal(body, &erron) - xml.Unmarshal(body, &erron) + _ = jsoniter.Unmarshal(body, &erron) + _ = xml.Unmarshal(body, &erron) if erron.HasError() { return nil, &erron } @@ -181,40 +211,10 @@ func (y *Cloud189PC) put(ctx context.Context, url string, headers map[string]str } return body, nil } -func (y *Cloud189PC) getFiles(ctx context.Context, fileId string) ([]model.Obj, error) { - fullUrl := API_URL - if y.isFamily() { - fullUrl += "/family/file" - } - fullUrl += "/listFiles.action" - - res := make([]model.Obj, 0, 130) +func (y *Cloud189PC) getFiles(ctx context.Context, fileId string, isFamily bool) ([]model.Obj, error) { + res := make([]model.Obj, 0, 100) for pageNum := 1; ; pageNum++ { - var resp Cloud189FilesResp - _, err := y.get(fullUrl, func(r *resty.Request) { - r.SetContext(ctx) - r.SetQueryParams(map[string]string{ - "folderId": fileId, - "fileType": "0", - "mediaAttr": "0", - "iconOption": "5", - "pageNum": fmt.Sprint(pageNum), - "pageSize": "130", - }) - if y.isFamily() { - r.SetQueryParams(map[string]string{ - "familyId": y.FamilyID, - "orderBy": toFamilyOrderBy(y.OrderBy), - "descending": toDesc(y.OrderDirection), - }) - } else { - r.SetQueryParams(map[string]string{ - "recursive": "0", - "orderBy": y.OrderBy, - "descending": toDesc(y.OrderDirection), - }) - } - }, &resp) + resp, err := y.getFilesWithPage(ctx, fileId, isFamily, pageNum, 1000, y.OrderBy, y.OrderDirection) if err != nil { return nil, err } @@ -233,6 +233,63 @@ func (y *Cloud189PC) getFiles(ctx context.Context, fileId string) ([]model.Obj, return res, nil } +func (y *Cloud189PC) getFilesWithPage(ctx context.Context, fileId string, isFamily bool, pageNum int, pageSize int, orderBy string, orderDirection string) (*Cloud189FilesResp, error) { + fullUrl := API_URL + if isFamily { + fullUrl += "/family/file" + } + fullUrl += "/listFiles.action" + + var resp Cloud189FilesResp + _, err := y.get(fullUrl, func(r *resty.Request) { + r.SetContext(ctx) + r.SetQueryParams(map[string]string{ + "folderId": fileId, + "fileType": "0", + "mediaAttr": "0", + "iconOption": "5", + "pageNum": fmt.Sprint(pageNum), + "pageSize": fmt.Sprint(pageSize), + }) + if isFamily { + r.SetQueryParams(map[string]string{ + "familyId": y.FamilyID, + "orderBy": toFamilyOrderBy(orderBy), + "descending": toDesc(orderDirection), + }) + } else { + r.SetQueryParams(map[string]string{ + "recursive": "0", + "orderBy": orderBy, + "descending": toDesc(orderDirection), + }) + } + }, &resp, isFamily) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (y *Cloud189PC) findFileByName(ctx context.Context, searchName string, folderId string, isFamily bool) (*Cloud189File, error) { + for pageNum := 1; ; pageNum++ { + resp, err := y.getFilesWithPage(ctx, folderId, isFamily, pageNum, 10, "filename", "asc") + if err != nil { + return nil, err + } + // 获取完毕跳出 + if resp.FileListAO.Count == 0 { + return nil, errs.ObjectNotFound + } + for i := 0; i < len(resp.FileListAO.FileList); i++ { + file := resp.FileListAO.FileList[i] + if file.Name == searchName { + return &file, nil + } + } + } +} + func (y *Cloud189PC) login() (err error) { // 初始化登陆所需参数 if y.loginParam == nil { @@ -292,7 +349,7 @@ func (y *Cloud189PC) login() (err error) { _, err = y.client.R(). SetResult(&tokenInfo).SetError(&erron). SetQueryParams(clientSuffix()). - SetQueryParam("redirectURL", url.QueryEscape(loginresp.ToUrl)). + SetQueryParam("redirectURL", loginresp.ToUrl). Post(API_URL + "/getSessionForPC.action") if err != nil { return @@ -400,6 +457,9 @@ func (y *Cloud189PC) initLoginParam() error { // 刷新会话 func (y *Cloud189PC) refreshSession() (err error) { + if y.ref != nil { + return y.ref.refreshSession() + } var erron RespErr var userSessionResp UserSessionResp _, err = y.client.R(). @@ -437,24 +497,21 @@ func (y *Cloud189PC) refreshSession() (err error) { // 普通上传 // 无法上传大小为0的文件 -func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - var sliceSize = partSize(file.GetSize()) - count := int(math.Ceil(float64(file.GetSize()) / float64(sliceSize))) - lastPartSize := file.GetSize() % sliceSize - if file.GetSize() > 0 && lastPartSize == 0 { - lastPartSize = sliceSize - } +func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { + size := file.GetSize() + sliceSize := partSize(size) + safeName := y.sanitizeName(file.GetName()) params := Params{ "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(file.GetName()), + "fileName": url.QueryEscape(safeName), "fileSize": fmt.Sprint(file.GetSize()), "sliceSize": fmt.Sprint(sliceSize), "lazyCheck": "1", } fullUrl := UPLOAD_URL - if y.isFamily() { + if isFamily { params.Set("familyId", y.FamilyID) fullUrl += "/family" } else { @@ -466,7 +523,7 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo var initMultiUpload InitMultiUploadResp _, err := y.request(fullUrl+"/initMultiUpload", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, params, &initMultiUpload) + }, params, &initMultiUpload, isFamily) if err != nil { return nil, err } @@ -475,24 +532,32 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) + sem := semaphore.NewWeighted(3) - fileMd5 := md5.New() - silceMd5 := md5.New() + count := int(size / sliceSize) + lastPartSize := size % sliceSize + if lastPartSize > 0 { + count++ + } else { + lastPartSize = sliceSize + } + fileMd5 := utils.MD5.NewFunc() + silceMd5 := utils.MD5.NewFunc() silceMd5Hexs := make([]string, 0, count) - + teeReader := io.TeeReader(file, io.MultiWriter(fileMd5, silceMd5)) + byteSize := sliceSize for i := 1; i <= count; i++ { if utils.IsCanceled(upCtx) { break } - - byteData := make([]byte, sliceSize) if i == count { - byteData = byteData[:lastPartSize] + byteSize = lastPartSize } - + byteData := make([]byte, byteSize) // 读取块 silceMd5.Reset() - if _, err := io.ReadFull(io.TeeReader(file, io.MultiWriter(fileMd5, silceMd5)), byteData); err != io.EOF && err != nil { + if _, err := io.ReadFull(teeReader, byteData); err != io.EOF && err != nil { + sem.Release(1) return nil, err } @@ -502,18 +567,23 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo partInfo := fmt.Sprintf("%d-%s", i, base64.StdEncoding.EncodeToString(md5Bytes)) threadG.Go(func(ctx context.Context) error { - uploadUrls, err := y.GetMultiUploadUrls(ctx, initMultiUpload.Data.UploadFileID, partInfo) + if err = sem.Acquire(ctx, 1); err != nil { + return err + } + defer sem.Release(1) + uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, initMultiUpload.Data.UploadFileID, partInfo) if err != nil { return err } // step.4 上传切片 uploadUrl := uploadUrls[0] - _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, bytes.NewReader(byteData)) + _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData)), isFamily) if err != nil { return err } - up(int(threadG.Success()) * 100 / count) + up(float64(threadG.Success()) * 100 / float64(count)) return nil }) } @@ -538,21 +608,22 @@ func (y *Cloud189PC) StreamUpload(ctx context.Context, dstDir model.Obj, file mo "sliceMd5": sliceMd5Hex, "lazyCheck": "1", "isLog": "0", - "opertype": "3", - }, &resp) + "opertype": IF(overwrite, "3", "1"), + }, &resp, isFamily) if err != nil { return nil, err } return resp.toFile(), nil } -func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer) (model.Obj, error) { +func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, isFamily bool, overwrite bool) (model.Obj, error) { fileMd5 := stream.GetHash().GetHash(utils.MD5) if len(fileMd5) < utils.MD5.Width { return nil, errors.New("invalid hash") } - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, stream.GetName(), fmt.Sprint(stream.GetSize())) + safeName := y.sanitizeName(stream.GetName()) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, safeName, fmt.Sprint(stream.GetSize()), isFamily) if err != nil { return nil, err } @@ -561,29 +632,49 @@ func (y *Cloud189PC) RapidUpload(ctx context.Context, dstDir model.Obj, stream m return nil, errors.New("rapid upload fail") } - return y.OldUploadCommit(ctx, uploadInfo.FileCommitUrl, uploadInfo.UploadFileId) + return y.OldUploadCommit(ctx, uploadInfo.FileCommitUrl, uploadInfo.UploadFileId, isFamily, overwrite) } // 快传 -func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - tempFile, err := file.CacheFullInTempFile() - if err != nil { - return nil, err - } - - var sliceSize = partSize(file.GetSize()) - count := int(math.Ceil(float64(file.GetSize()) / float64(sliceSize))) - lastSliceSize := file.GetSize() % sliceSize - if file.GetSize() > 0 && lastSliceSize == 0 { +func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { + var ( + cache = file.GetFile() + tmpF *os.File + err error + ) + safeName := y.sanitizeName(file.GetName()) + size := file.GetSize() + if _, ok := cache.(io.ReaderAt); !ok && size > 0 { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF + } + sliceSize := partSize(size) + count := int(size / sliceSize) + lastSliceSize := size % sliceSize + if lastSliceSize > 0 { + count++ + } else { lastSliceSize = sliceSize } //step.1 优先计算所需信息 byteSize := sliceSize - fileMd5 := md5.New() - silceMd5 := md5.New() - silceMd5Hexs := make([]string, 0, count) + fileMd5 := utils.MD5.NewFunc() + sliceMd5 := utils.MD5.NewFunc() + sliceMd5Hexs := make([]string, 0, count) partInfos := make([]string, 0, count) + writers := []io.Writer{fileMd5, sliceMd5} + if tmpF != nil { + writers = append(writers, tmpF) + } + written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() @@ -593,23 +684,35 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode byteSize = lastSliceSize } - silceMd5.Reset() - if _, err := io.CopyN(io.MultiWriter(fileMd5, silceMd5), tempFile, byteSize); err != nil && err != io.EOF { + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), file, byteSize) + written += n + if err != nil && err != io.EOF { return nil, err } - md5Byte := silceMd5.Sum(nil) - silceMd5Hexs = append(silceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte))) + md5Byte := sliceMd5.Sum(nil) + sliceMd5Hexs = append(sliceMd5Hexs, strings.ToUpper(hex.EncodeToString(md5Byte))) partInfos = append(partInfos, fmt.Sprint(i, "-", base64.StdEncoding.EncodeToString(md5Byte))) + sliceMd5.Reset() + } + + if tmpF != nil { + if size > 0 && written != size { + return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, size) + } + _, err = tmpF.Seek(0, io.SeekStart) + if err != nil { + return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") + } } fileMd5Hex := strings.ToUpper(hex.EncodeToString(fileMd5.Sum(nil))) sliceMd5Hex := fileMd5Hex - if file.GetSize() > sliceSize { - sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(silceMd5Hexs, "\n"))) + if size > sliceSize { + sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(sliceMd5Hexs, "\n"))) } fullUrl := UPLOAD_URL - if y.isFamily() { + if isFamily { fullUrl += "/family" } else { //params.Set("extend", `{"opScene":"1","relativepath":"","rootfolderid":""}`) @@ -617,24 +720,24 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } // 尝试恢复进度 - uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.tokenInfo.SessionKey, fileMd5Hex) + uploadProgress, ok := base.GetUploadProgress[*UploadProgress](y, y.getTokenInfo().SessionKey, fileMd5Hex) if !ok { //step.2 预上传 params := Params{ "parentFolderId": dstDir.GetID(), - "fileName": url.QueryEscape(file.GetName()), + "fileName": url.QueryEscape(safeName), "fileSize": fmt.Sprint(file.GetSize()), "fileMd5": fileMd5Hex, "sliceSize": fmt.Sprint(sliceSize), "sliceMd5": sliceMd5Hex, } - if y.isFamily() { + if isFamily { params.Set("familyId", y.FamilyID) } var uploadInfo InitMultiUploadResp _, err = y.request(fullUrl+"/initMultiUpload", http.MethodGet, func(req *resty.Request) { req.SetContext(ctx) - }, params, &uploadInfo) + }, params, &uploadInfo, isFamily) if err != nil { return nil, err } @@ -659,7 +762,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode i, uploadPart := i, uploadPart threadG.Go(func(ctx context.Context) error { // step.3 获取上传链接 - uploadUrls, err := y.GetMultiUploadUrls(ctx, uploadInfo.UploadFileID, uploadPart) + uploadUrls, err := y.GetMultiUploadUrls(ctx, isFamily, uploadInfo.UploadFileID, uploadPart) if err != nil { return err } @@ -671,12 +774,12 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode } // step.4 上传切片 - _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, io.NewSectionReader(tempFile, offset, byteSize)) + _, err = y.put(ctx, uploadUrl.RequestURL, uploadUrl.Headers, false, io.NewSectionReader(cache, offset, byteSize), isFamily) if err != nil { return err } - up(int(threadG.Success()) * 100 / len(uploadUrls)) + up(float64(threadG.Success()) * 100 / float64(len(uploadUrls))) uploadProgress.UploadParts[i] = "" return nil }) @@ -684,7 +787,7 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { uploadProgress.UploadParts = utils.SliceFilter(uploadProgress.UploadParts, func(s string) bool { return s != "" }) - base.SaveUploadProgress(y, uploadProgress, y.tokenInfo.SessionKey, fileMd5Hex) + base.SaveUploadProgress(y, uploadProgress, y.getTokenInfo().SessionKey, fileMd5Hex) } return nil, err } @@ -698,8 +801,8 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode }, Params{ "uploadFileId": uploadInfo.UploadFileID, "isLog": "0", - "opertype": "3", - }, &resp) + "opertype": IF(overwrite, "3", "1"), + }, &resp, isFamily) if err != nil { return nil, err } @@ -708,9 +811,9 @@ func (y *Cloud189PC) FastUpload(ctx context.Context, dstDir model.Obj, file mode // 获取上传切片信息 // 对http body有大小限制,分片信息太多会出错 -func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, uploadFileId string, partInfo ...string) ([]UploadUrlInfo, error) { +func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, isFamily bool, uploadFileId string, partInfo ...string) ([]UploadUrlInfo, error) { fullUrl := UPLOAD_URL - if y.isFamily() { + if isFamily { fullUrl += "/family" } else { fullUrl += "/person" @@ -723,7 +826,7 @@ func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, uploadFileId string }, Params{ "uploadFileId": uploadFileId, "partInfo": strings.Join(partInfo, ","), - }, &uploadUrlsResp) + }, &uploadUrlsResp, isFamily) if err != nil { return nil, err } @@ -752,18 +855,16 @@ func (y *Cloud189PC) GetMultiUploadUrls(ctx context.Context, uploadFileId string } // 旧版本上传,家庭云不支持覆盖 -func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - tempFile, err := file.CacheFullInTempFile() - if err != nil { - return nil, err - } - fileMd5, err := utils.HashFile(utils.MD5, tempFile) +func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, isFamily bool, overwrite bool) (model.Obj, error) { + tempFile, fileMd5, err := stream.CacheFullInTempFileAndHash(file, utils.MD5) if err != nil { return nil, err } + rateLimited := driver.NewLimitedUploadStream(ctx, io.NopCloser(tempFile)) + safeName := y.sanitizeName(file.GetName()) // 创建上传会话 - uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, file.GetName(), fmt.Sprint(file.GetSize())) + uploadInfo, err := y.OldUploadCreate(ctx, dstDir.GetID(), fileMd5, safeName, fmt.Sprint(file.GetSize()), isFamily) if err != nil { return nil, err } @@ -780,14 +881,14 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model "Expect": "100-continue", } - if y.isFamily() { + if isFamily { header["FamilyId"] = fmt.Sprint(y.FamilyID) header["UploadFileId"] = fmt.Sprint(status.UploadFileId) } else { header["Edrive-UploadFileId"] = fmt.Sprint(status.UploadFileId) } - _, err := y.put(ctx, status.FileUploadUrl, header, true, io.NopCloser(tempFile)) + _, err := y.put(ctx, status.FileUploadUrl, header, true, rateLimited, isFamily) if err, ok := err.(*RespErr); ok && err.Code != "InputStreamReadError" { return nil, err } @@ -802,33 +903,33 @@ func (y *Cloud189PC) OldUpload(ctx context.Context, dstDir model.Obj, file model "uploadFileId": fmt.Sprint(status.UploadFileId), "resumePolicy": "1", }) - if y.isFamily() { + if isFamily { req.SetQueryParam("familyId", fmt.Sprint(y.FamilyID)) } - }, &status) + }, &status, isFamily) if err != nil { return nil, err } if _, err := tempFile.Seek(status.GetSize(), io.SeekStart); err != nil { return nil, err } - up(int(status.GetSize()/file.GetSize()) * 100) + up(float64(status.GetSize()) / float64(file.GetSize()) * 100) } - return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId) + return y.OldUploadCommit(ctx, status.FileCommitUrl, status.UploadFileId, isFamily, overwrite) } // 创建上传会话 -func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileMd5, fileName, fileSize string) (*CreateUploadFileResp, error) { +func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileMd5, fileName, fileSize string, isFamily bool) (*CreateUploadFileResp, error) { var uploadInfo CreateUploadFileResp fullUrl := API_URL + "/createUploadFile.action" - if y.isFamily() { + if isFamily { fullUrl = API_URL + "/family/file/createFamilyFile.action" } _, err := y.post(fullUrl, func(req *resty.Request) { req.SetContext(ctx) - if y.isFamily() { + if isFamily { req.SetQueryParams(map[string]string{ "familyId": y.FamilyID, "parentId": parentID, @@ -849,7 +950,7 @@ func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileM "isLog": "0", }) } - }, &uploadInfo) + }, &uploadInfo, isFamily) if err != nil { return nil, err @@ -858,11 +959,11 @@ func (y *Cloud189PC) OldUploadCreate(ctx context.Context, parentID string, fileM } // 提交上传文件 -func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string, uploadFileID int64) (model.Obj, error) { +func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string, uploadFileID int64, isFamily bool, overwrite bool) (model.Obj, error) { var resp OldCommitUploadFileResp _, err := y.post(fileCommitUrl, func(req *resty.Request) { req.SetContext(ctx) - if y.isFamily() { + if isFamily { req.SetHeaders(map[string]string{ "ResumePolicy": "1", "UploadFileId": fmt.Sprint(uploadFileID), @@ -870,13 +971,13 @@ func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string, }) } else { req.SetFormData(map[string]string{ - "opertype": "3", + "opertype": IF(overwrite, "3", "1"), "resumePolicy": "1", "uploadFileId": fmt.Sprint(uploadFileID), "isLog": "0", }) } - }, &resp) + }, &resp, isFamily) if err != nil { return nil, err } @@ -895,10 +996,79 @@ func (y *Cloud189PC) isLogin() bool { return err == nil } +// 创建家庭云中转文件夹 +func (y *Cloud189PC) createFamilyTransferFolder() error { + var rootFolder Cloud189Folder + _, err := y.post(API_URL+"/family/file/createFolder.action", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "folderName": "FamilyTransferFolder", + "familyId": y.FamilyID, + }) + }, &rootFolder, true) + if err != nil { + return err + } + y.familyTransferFolder = &rootFolder + return nil +} + +// 清理中转文件夹 +func (y *Cloud189PC) cleanFamilyTransfer(ctx context.Context) error { + transferFolderId := y.familyTransferFolder.GetID() + for pageNum := 1; ; pageNum++ { + resp, err := y.getFilesWithPage(ctx, transferFolderId, true, pageNum, 100, "lastOpTime", "asc") + if err != nil { + return err + } + // 获取完毕跳出 + if resp.FileListAO.Count == 0 { + break + } + + var tasks []BatchTaskInfo + for i := 0; i < len(resp.FileListAO.FolderList); i++ { + folder := resp.FileListAO.FolderList[i] + tasks = append(tasks, BatchTaskInfo{ + FileId: folder.GetID(), + FileName: folder.GetName(), + IsFolder: BoolToNumber(folder.IsDir()), + }) + } + for i := 0; i < len(resp.FileListAO.FileList); i++ { + file := resp.FileListAO.FileList[i] + tasks = append(tasks, BatchTaskInfo{ + FileId: file.GetID(), + FileName: file.GetName(), + IsFolder: BoolToNumber(file.IsDir()), + }) + } + + if len(tasks) > 0 { + // 删除 + resp, err := y.CreateBatchTask("DELETE", y.FamilyID, "", nil, tasks...) + if err != nil { + return err + } + err = y.WaitBatchTask("DELETE", resp.TaskID, time.Second) + if err != nil { + return err + } + // 永久删除 + resp, err = y.CreateBatchTask("CLEAR_RECYCLE", y.FamilyID, "", nil, tasks...) + if err != nil { + return err + } + err = y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) + return err + } + } + return nil +} + // 获取家庭云所有用户信息 func (y *Cloud189PC) getFamilyInfoList() ([]FamilyInfoResp, error) { var resp FamilyInfoListResp - _, err := y.get(API_URL+"/family/manage/getFamilyList.action", nil, &resp) + _, err := y.get(API_URL+"/family/manage/getFamilyList.action", nil, &resp, true) if err != nil { return nil, err } @@ -915,13 +1085,108 @@ func (y *Cloud189PC) getFamilyID() (string, error) { return "", fmt.Errorf("cannot get automatically,please input family_id") } for _, info := range infos { - if strings.Contains(y.tokenInfo.LoginName, info.RemarkName) { + if strings.Contains(y.getTokenInfo().LoginName, info.RemarkName) { return fmt.Sprint(info.FamilyID), nil } } return fmt.Sprint(infos[0].FamilyID), nil } +// 保存家庭云中的文件到个人云 +func (y *Cloud189PC) SaveFamilyFileToPersonCloud(ctx context.Context, familyId string, srcObj, dstDir model.Obj, overwrite bool) error { + // _, err := y.post(API_URL+"/family/file/saveFileToMember.action", func(req *resty.Request) { + // req.SetQueryParams(map[string]string{ + // "channelId": "home", + // "familyId": familyId, + // "destParentId": destParentId, + // "fileIdList": familyFileId, + // }) + // }, nil) + // return err + + task := BatchTaskInfo{ + FileId: srcObj.GetID(), + FileName: srcObj.GetName(), + IsFolder: BoolToNumber(srcObj.IsDir()), + } + resp, err := y.CreateBatchTask("COPY", familyId, dstDir.GetID(), map[string]string{ + "groupId": "null", + "copyType": "2", + "shareId": "null", + }, task) + if err != nil { + return err + } + + for { + state, err := y.CheckBatchTask("COPY", resp.TaskID) + if err != nil { + return err + } + switch state.TaskStatus { + case 2: + task.DealWay = IF(overwrite, 3, 2) + // 冲突时覆盖文件 + if err := y.ManageBatchTask("COPY", resp.TaskID, dstDir.GetID(), task); err != nil { + return err + } + case 4: + return nil + } + time.Sleep(time.Millisecond * 400) + } +} + +// 永久删除文件 +func (y *Cloud189PC) Delete(ctx context.Context, familyId string, srcObj model.Obj) error { + task := BatchTaskInfo{ + FileId: srcObj.GetID(), + FileName: srcObj.GetName(), + IsFolder: BoolToNumber(srcObj.IsDir()), + } + // 删除源文件 + resp, err := y.CreateBatchTask("DELETE", familyId, "", nil, task) + if err != nil { + return err + } + err = y.WaitBatchTask("DELETE", resp.TaskID, time.Second) + if err != nil { + return err + } + // 清除回收站 + resp, err = y.CreateBatchTask("CLEAR_RECYCLE", familyId, "", nil, task) + if err != nil { + return err + } + err = y.WaitBatchTask("CLEAR_RECYCLE", resp.TaskID, time.Second) + if err != nil { + return err + } + return nil +} + +func (y *Cloud189PC) CreateBatchTask(aType string, familyID string, targetFolderId string, other map[string]string, taskInfos ...BatchTaskInfo) (*CreateBatchTaskResp, error) { + var resp CreateBatchTaskResp + _, err := y.post(API_URL+"/batch/createBatchTask.action", func(req *resty.Request) { + req.SetFormData(map[string]string{ + "type": aType, + "taskInfos": MustString(utils.Json.MarshalToString(taskInfos)), + }) + if targetFolderId != "" { + req.SetFormData(map[string]string{"targetFolderId": targetFolderId}) + } + if familyID != "" { + req.SetFormData(map[string]string{"familyId": familyID}) + } + req.SetFormData(other) + }, &resp, familyID != "") + if err != nil { + return nil, err + } + return &resp, nil +} + +// 检测任务状态 func (y *Cloud189PC) CheckBatchTask(aType string, taskID string) (*BatchTaskStateResp, error) { var resp BatchTaskStateResp _, err := y.post(API_URL+"/batch/checkBatchTask.action", func(req *resty.Request) { @@ -936,6 +1201,37 @@ func (y *Cloud189PC) CheckBatchTask(aType string, taskID string) (*BatchTaskStat return &resp, nil } +// 获取冲突的任务信息 +func (y *Cloud189PC) GetConflictTaskInfo(aType string, taskID string) (*BatchTaskConflictTaskInfoResp, error) { + var resp BatchTaskConflictTaskInfoResp + _, err := y.post(API_URL+"/batch/getConflictTaskInfo.action", func(req *resty.Request) { + req.SetFormData(map[string]string{ + "type": aType, + "taskId": taskID, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +// 处理冲突 +func (y *Cloud189PC) ManageBatchTask(aType string, taskID string, targetFolderId string, taskInfos ...BatchTaskInfo) error { + _, err := y.post(API_URL+"/batch/manageBatchTask.action", func(req *resty.Request) { + req.SetFormData(map[string]string{ + "targetFolderId": targetFolderId, + "type": aType, + "taskId": taskID, + "taskInfos": MustString(utils.Json.MarshalToString(taskInfos)), + }) + }, nil) + return err +} + +var ErrIsConflict = errors.New("there is a conflict with the target object") + +// 等待任务完成 func (y *Cloud189PC) WaitBatchTask(aType string, taskID string, t time.Duration) error { for { state, err := y.CheckBatchTask(aType, taskID) @@ -944,10 +1240,24 @@ func (y *Cloud189PC) WaitBatchTask(aType string, taskID string, t time.Duration) } switch state.TaskStatus { case 2: - return errors.New("there is a conflict with the target object") + return ErrIsConflict case 4: return nil } time.Sleep(t) } } + +func (y *Cloud189PC) getTokenInfo() *AppSessionResp { + if y.ref != nil { + return y.ref.getTokenInfo() + } + return y.tokenInfo +} + +func (y *Cloud189PC) getClient() *resty.Client { + if y.ref != nil { + return y.ref.getClient() + } + return y.client +} diff --git a/drivers/alias/driver.go b/drivers/alias/driver.go index 271096b3e46..e292a62816f 100644 --- a/drivers/alias/driver.go +++ b/drivers/alias/driver.go @@ -3,10 +3,12 @@ package alias import ( "context" "errors" + stdpath "path" "strings" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" ) @@ -45,6 +47,9 @@ func (d *Alias) Init(ctx context.Context) error { d.oneKey = k } d.autoFlatten = true + } else { + d.oneKey = "" + d.autoFlatten = false } return nil } @@ -87,8 +92,9 @@ func (d *Alias) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ return nil, errs.ObjectNotFound } var objs []model.Obj + fsArgs := &fs.ListArgs{NoLog: true, Refresh: args.Refresh} for _, dst := range dsts { - tmp, err := d.list(ctx, dst, sub) + tmp, err := d.list(ctx, dst, sub, fsArgs) if err == nil { objs = append(objs, tmp...) } @@ -105,10 +111,211 @@ func (d *Alias) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( for _, dst := range dsts { link, err := d.link(ctx, dst, sub, args) if err == nil { + if !args.Redirect && len(link.URL) > 0 { + // 正常情况下 多并发 仅支持返回URL的驱动 + // alias套娃alias 可以让crypt、mega等驱动(不返回URL的) 支持并发 + if d.DownloadConcurrency > 0 { + link.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + link.PartSize = d.DownloadPartSize * utils.KB + } + } return link, nil } } return nil, errs.ObjectNotFound } +func (d *Alias) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, parentDir, true) + if err == nil { + return fs.MakeDir(ctx, stdpath.Join(*reqPath, dirName)) + } + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot make sub-dir") + } + return err +} + +func (d *Alias) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.Writable { + return errs.PermissionDenied + } + srcPath, err := d.getReqPath(ctx, srcObj, false) + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be moved") + } + if err != nil { + return err + } + dstPath, err := d.getReqPath(ctx, dstDir, true) + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot be moved to") + } + if err != nil { + return err + } + return fs.Move(ctx, *srcPath, *dstPath) +} + +func (d *Alias) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, srcObj, false) + if err == nil { + return fs.Rename(ctx, *reqPath, newName) + } + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be Rename") + } + return err +} + +func (d *Alias) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.Writable { + return errs.PermissionDenied + } + srcPath, err := d.getReqPath(ctx, srcObj, false) + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be copied") + } + if err != nil { + return err + } + dstPath, err := d.getReqPath(ctx, dstDir, true) + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot be copied to") + } + if err != nil { + return err + } + _, err = fs.Copy(ctx, *srcPath, *dstPath) + return err +} + +func (d *Alias) Remove(ctx context.Context, obj model.Obj) error { + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, obj, false) + if err == nil { + return fs.Remove(ctx, *reqPath) + } + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be Delete") + } + return err +} + +func (d *Alias) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, dstDir, true) + if err == nil { + return fs.PutDirectly(ctx, *reqPath, s) + } + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot be Put") + } + return err +} + +func (d *Alias) PutURL(ctx context.Context, dstDir model.Obj, name, url string) error { + if !d.Writable { + return errs.PermissionDenied + } + reqPath, err := d.getReqPath(ctx, dstDir, true) + if err == nil { + return fs.PutURL(ctx, *reqPath, name, url) + } + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot offline download") + } + return err +} + +func (d *Alias) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + root, sub := d.getRootAndPath(obj.GetPath()) + dsts, ok := d.pathMap[root] + if !ok { + return nil, errs.ObjectNotFound + } + for _, dst := range dsts { + meta, err := d.getArchiveMeta(ctx, dst, sub, args) + if err == nil { + return meta, nil + } + } + return nil, errs.NotImplement +} + +func (d *Alias) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + root, sub := d.getRootAndPath(obj.GetPath()) + dsts, ok := d.pathMap[root] + if !ok { + return nil, errs.ObjectNotFound + } + for _, dst := range dsts { + l, err := d.listArchive(ctx, dst, sub, args) + if err == nil { + return l, nil + } + } + return nil, errs.NotImplement +} + +func (d *Alias) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // alias的两个驱动,一个支持驱动提取,一个不支持,如何兼容? + // 如果访问的是不支持驱动提取的驱动内的压缩文件,GetArchiveMeta就会返回errs.NotImplement,提取URL前缀就会是/ae,Extract就不会被调用 + // 如果访问的是支持驱动提取的驱动内的压缩文件,GetArchiveMeta就会返回有效值,提取URL前缀就会是/ad,Extract就会被调用 + root, sub := d.getRootAndPath(obj.GetPath()) + dsts, ok := d.pathMap[root] + if !ok { + return nil, errs.ObjectNotFound + } + for _, dst := range dsts { + link, err := d.extract(ctx, dst, sub, args) + if err == nil { + if !args.Redirect && len(link.URL) > 0 { + if d.DownloadConcurrency > 0 { + link.Concurrency = d.DownloadConcurrency + } + if d.DownloadPartSize > 0 { + link.PartSize = d.DownloadPartSize * utils.KB + } + } + return link, nil + } + } + return nil, errs.NotImplement +} + +func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { + if !d.Writable { + return errs.PermissionDenied + } + srcPath, err := d.getReqPath(ctx, srcObj, false) + if errs.IsNotImplement(err) { + return errors.New("same-name files cannot be decompressed") + } + if err != nil { + return err + } + dstPath, err := d.getReqPath(ctx, dstDir, true) + if errs.IsNotImplement(err) { + return errors.New("same-name dirs cannot be decompressed to") + } + if err != nil { + return err + } + _, err = fs.ArchiveDecompress(ctx, *srcPath, *dstPath, args) + return err +} + var _ driver.Driver = (*Alias)(nil) diff --git a/drivers/alias/meta.go b/drivers/alias/meta.go index 6611e1dc302..70dc59f0b73 100644 --- a/drivers/alias/meta.go +++ b/drivers/alias/meta.go @@ -9,19 +9,28 @@ type Addition struct { // Usually one of two // driver.RootPath // define other - Paths string `json:"paths" required:"true" type:"text"` + Paths string `json:"paths" required:"true" type:"text"` + ProtectSameName bool `json:"protect_same_name" default:"true" required:"false" help:"Protects same-name files from Delete or Rename"` + DownloadConcurrency int `json:"download_concurrency" default:"0" required:"false" type:"number" help:"Need to enable proxy"` + DownloadPartSize int `json:"download_part_size" default:"0" type:"number" required:"false" help:"Need to enable proxy. Unit: KB"` + Writable bool `json:"writable" type:"bool" default:"false"` } var config = driver.Config{ - Name: "Alias", - LocalSort: true, - NoCache: true, - NoUpload: true, - DefaultRoot: "/", + Name: "Alias", + LocalSort: true, + NoCache: true, + NoUpload: false, + DefaultRoot: "/", + ProxyRangeOption: true, } func init() { op.RegisterDriver(func() driver.Driver { - return &Alias{} + return &Alias{ + Addition: Addition{ + ProtectSameName: true, + }, + } }) } diff --git a/drivers/alias/util.go b/drivers/alias/util.go index 4e3d6bf0f5c..ffb0b84f605 100644 --- a/drivers/alias/util.go +++ b/drivers/alias/util.go @@ -3,11 +3,15 @@ package alias import ( "context" "fmt" + "net/url" stdpath "path" "strings" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -15,7 +19,7 @@ import ( func (d *Alias) listRoot() []model.Obj { var objs []model.Obj - for k, _ := range d.pathMap { + for k := range d.pathMap { obj := model.Object{ Name: k, IsFolder: true, @@ -61,11 +65,12 @@ func (d *Alias) get(ctx context.Context, path string, dst, sub string) (model.Ob Size: obj.GetSize(), Modified: obj.ModTime(), IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), }, nil } -func (d *Alias) list(ctx context.Context, dst, sub string) ([]model.Obj, error) { - objs, err := fs.List(ctx, stdpath.Join(dst, sub), &fs.ListArgs{NoLog: true}) +func (d *Alias) list(ctx context.Context, dst, sub string, args *fs.ListArgs) ([]model.Obj, error) { + objs, err := fs.List(ctx, stdpath.Join(dst, sub), args) // the obj must implement the model.SetPath interface // return objs, err if err != nil { @@ -93,22 +98,128 @@ func (d *Alias) list(ctx context.Context, dst, sub string) ([]model.Obj, error) func (d *Alias) link(ctx context.Context, dst, sub string, args model.LinkArgs) (*model.Link, error) { reqPath := stdpath.Join(dst, sub) - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + // 参考 crypt 驱动 + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) if err != nil { return nil, err } + if _, ok := storage.(*Alias); !ok && !args.Redirect { + link, _, err := op.Link(ctx, storage, reqActualPath, args) + return link, err + } _, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) if err != nil { return nil, err } if common.ShouldProxy(storage, stdpath.Base(sub)) { - return &model.Link{ + link := &model.Link{ URL: fmt.Sprintf("%s/p%s?sign=%s", common.GetApiUrl(args.HttpReq), utils.EncodePath(reqPath, true), sign.Sign(reqPath)), - }, nil + } + if args.HttpReq != nil && d.ProxyRange { + link.RangeReadCloser = common.NoProxyRange + } + return link, nil } - link, _, err := fs.Link(ctx, reqPath, args) + link, _, err := op.Link(ctx, storage, reqActualPath, args) return link, err } + +func (d *Alias) getReqPath(ctx context.Context, obj model.Obj, isParent bool) (*string, error) { + root, sub := d.getRootAndPath(obj.GetPath()) + if sub == "" && !isParent { + return nil, errs.NotSupport + } + dsts, ok := d.pathMap[root] + if !ok { + return nil, errs.ObjectNotFound + } + var reqPath *string + for _, dst := range dsts { + path := stdpath.Join(dst, sub) + _, err := fs.Get(ctx, path, &fs.GetArgs{NoLog: true}) + if err != nil { + continue + } + if !d.ProtectSameName { + return &path, nil + } + if ok { + ok = false + } else { + return nil, errs.NotImplement + } + reqPath = &path + } + if reqPath == nil { + return nil, errs.ObjectNotFound + } + return reqPath, nil +} + +func (d *Alias) getArchiveMeta(ctx context.Context, dst, sub string, args model.ArchiveArgs) (model.ArchiveMeta, error) { + reqPath := stdpath.Join(dst, sub) + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + return nil, err + } + if _, ok := storage.(driver.ArchiveReader); ok { + return op.GetArchiveMeta(ctx, storage, reqActualPath, model.ArchiveMetaArgs{ + ArchiveArgs: args, + Refresh: true, + }) + } + return nil, errs.NotImplement +} + +func (d *Alias) listArchive(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) ([]model.Obj, error) { + reqPath := stdpath.Join(dst, sub) + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + return nil, err + } + if _, ok := storage.(driver.ArchiveReader); ok { + return op.ListArchive(ctx, storage, reqActualPath, model.ArchiveListArgs{ + ArchiveInnerArgs: args, + Refresh: true, + }) + } + return nil, errs.NotImplement +} + +func (d *Alias) extract(ctx context.Context, dst, sub string, args model.ArchiveInnerArgs) (*model.Link, error) { + reqPath := stdpath.Join(dst, sub) + storage, reqActualPath, err := op.GetStorageAndActualPath(reqPath) + if err != nil { + return nil, err + } + if _, ok := storage.(driver.ArchiveReader); ok { + if _, ok := storage.(*Alias); !ok && !args.Redirect { + link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) + return link, err + } + _, err = fs.Get(ctx, reqPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err + } + if common.ShouldProxy(storage, stdpath.Base(sub)) { + link := &model.Link{ + URL: fmt.Sprintf("%s/ap%s?inner=%s&pass=%s&sign=%s", + common.GetApiUrl(args.HttpReq), + utils.EncodePath(reqPath, true), + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + sign.SignArchive(reqPath)), + } + if args.HttpReq != nil && d.ProxyRange { + link.RangeReadCloser = common.NoProxyRange + } + return link, nil + } + link, _, err := op.DriverExtract(ctx, storage, reqActualPath, args) + return link, err + } + return nil, errs.NotImplement +} diff --git a/drivers/alist_v3/driver.go b/drivers/alist_v3/driver.go index e2deabacccb..56f9c01e124 100644 --- a/drivers/alist_v3/driver.go +++ b/drivers/alist_v3/driver.go @@ -5,17 +5,19 @@ import ( "fmt" "io" "net/http" + "net/url" "path" - "strconv" "strings" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) type AListV3 struct { @@ -34,29 +36,29 @@ func (d *AListV3) GetAddition() driver.Additional { func (d *AListV3) Init(ctx context.Context) error { d.Addition.Address = strings.TrimSuffix(d.Addition.Address, "/") var resp common.Resp[MeResp] - _, err := d.request("/me", http.MethodGet, func(req *resty.Request) { + _, _, err := d.request("/me", http.MethodGet, func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { return err } // if the username is not empty and the username is not the same as the current username, then login again - if d.Username != "" && d.Username != resp.Data.Username { + if d.Username != resp.Data.Username { err = d.login() if err != nil { return err } } // re-get the user info - _, err = d.request("/me", http.MethodGet, func(req *resty.Request) { + _, _, err = d.request("/me", http.MethodGet, func(req *resty.Request) { req.SetResult(&resp) }) if err != nil { return err } - if resp.Data.Role == model.GUEST { - url := d.Address + "/api/public/settings" - res, err := base.RestyClient.R().Get(url) + if utils.SliceContains(resp.Data.Role, model.GUEST) { + u := d.Address + "/api/public/settings" + res, err := base.RestyClient.R().Get(u) if err != nil { return err } @@ -74,7 +76,7 @@ func (d *AListV3) Drop(ctx context.Context) error { func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { var resp common.Resp[FsListResp] - _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/list", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(ListReq{ PageReq: model.PageReq{ Page: 1, @@ -108,11 +110,19 @@ func (d *AListV3) List(ctx context.Context, dir model.Obj, args model.ListArgs) func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { var resp common.Resp[FsGetResp] - _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { + // if PassUAToUpsteam is true, then pass the user-agent to the upstream + userAgent := base.UserAgent + if d.PassUAToUpsteam { + userAgent = args.Header.Get("user-agent") + if userAgent == "" { + userAgent = base.UserAgent + } + } + _, _, err := d.request("/fs/get", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(FsGetReq{ Path: file.GetPath(), Password: d.MetaPassword, - }) + }).SetHeader("user-agent", userAgent) }) if err != nil { return nil, err @@ -123,7 +133,7 @@ func (d *AListV3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *AListV3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.request("/fs/mkdir", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/mkdir", http.MethodPost, func(req *resty.Request) { req.SetBody(MkdirOrLinkReq{ Path: path.Join(parentDir.GetPath(), dirName), }) @@ -132,7 +142,7 @@ func (d *AListV3) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri } func (d *AListV3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("/fs/move", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/move", http.MethodPost, func(req *resty.Request) { req.SetBody(MoveCopyReq{ SrcDir: path.Dir(srcObj.GetPath()), DstDir: dstDir.GetPath(), @@ -143,7 +153,7 @@ func (d *AListV3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *AListV3) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err := d.request("/fs/rename", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/rename", http.MethodPost, func(req *resty.Request) { req.SetBody(RenameReq{ Path: srcObj.GetPath(), Name: newName, @@ -153,7 +163,7 @@ func (d *AListV3) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *AListV3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("/fs/copy", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(MoveCopyReq{ SrcDir: path.Dir(srcObj.GetPath()), DstDir: dstDir.GetPath(), @@ -164,7 +174,7 @@ func (d *AListV3) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { - _, err := d.request("/fs/remove", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/fs/remove", http.MethodPost, func(req *resty.Request) { req.SetBody(RemoveReq{ Dir: path.Dir(obj.GetPath()), Names: []string{obj.GetName()}, @@ -173,13 +183,174 @@ func (d *AListV3) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - _, err := d.request("/fs/put", http.MethodPut, func(req *resty.Request) { - req.SetHeader("File-Path", path.Join(dstDir.GetPath(), stream.GetName())). - SetHeader("Password", d.MetaPassword). - SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). - SetContentLength(true). - SetBody(io.ReadCloser(stream)) +func (d *AListV3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPut, d.Address+"/api/fs/put", reader) + if err != nil { + return err + } + req.Header.Set("Authorization", d.Token) + req.Header.Set("File-Path", path.Join(dstDir.GetPath(), s.GetName())) + req.Header.Set("Password", d.MetaPassword) + if md5 := s.GetHash().GetHash(utils.MD5); len(md5) > 0 { + req.Header.Set("X-File-Md5", md5) + } + if sha1 := s.GetHash().GetHash(utils.SHA1); len(sha1) > 0 { + req.Header.Set("X-File-Sha1", sha1) + } + if sha256 := s.GetHash().GetHash(utils.SHA256); len(sha256) > 0 { + req.Header.Set("X-File-Sha256", sha256) + } + + req.ContentLength = s.GetSize() + // client := base.NewHttpClient() + // client.Timeout = time.Hour * 6 + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + + bytes, err := io.ReadAll(res.Body) + if err != nil { + return err + } + log.Debugf("[alist_v3] response body: %s", string(bytes)) + if res.StatusCode >= 400 { + return fmt.Errorf("request failed, status: %s", res.Status) + } + code := utils.Json.Get(bytes, "code").ToInt() + if code != 200 { + if code == 401 || code == 403 { + err = d.login() + if err != nil { + return err + } + } + return fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(bytes, "message").ToString()) + } + return nil +} + +func (d *AListV3) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotImplement + } + var resp common.Resp[ArchiveMetaResp] + _, code, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }) + }) + if code == 202 { + return nil, errs.WrongArchivePassword + } + if err != nil { + return nil, err + } + var tree []model.ObjTree + if resp.Data.Content != nil { + tree = make([]model.ObjTree, 0, len(resp.Data.Content)) + for _, content := range resp.Data.Content { + tree = append(tree, &content) + } + } + return &model.ArchiveMetaInfo{ + Comment: resp.Data.Comment, + Encrypted: resp.Data.Encrypted, + Tree: tree, + }, nil +} + +func (d *AListV3) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotImplement + } + var resp common.Resp[ArchiveListResp] + _, code, err := d.request("/fs/archive/list", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveListReq{ + ArchiveMetaReq: ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }, + PageReq: model.PageReq{ + Page: 1, + PerPage: 0, + }, + InnerPath: args.InnerPath, + }) + }) + if code == 202 { + return nil, errs.WrongArchivePassword + } + if err != nil { + return nil, err + } + var files []model.Obj + for _, f := range resp.Data.Content { + file := model.ObjThumb{ + Object: model.Object{ + Name: f.Name, + Modified: f.Modified, + Ctime: f.Created, + Size: f.Size, + IsFolder: f.IsDir, + HashInfo: utils.FromString(f.HashInfo), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumb}, + } + files = append(files, &file) + } + return files, nil +} + +func (d *AListV3) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + if !d.ForwardArchiveReq { + return nil, errs.NotSupport + } + var resp common.Resp[ArchiveMetaResp] + _, _, err := d.request("/fs/archive/meta", http.MethodPost, func(req *resty.Request) { + req.SetResult(&resp).SetBody(ArchiveMetaReq{ + ArchivePass: args.Password, + Password: d.MetaPassword, + Path: obj.GetPath(), + Refresh: false, + }) + }) + if err != nil { + return nil, err + } + return &model.Link{ + URL: fmt.Sprintf("%s?inner=%s&pass=%s&sign=%s", + resp.Data.RawURL, + utils.EncodePath(args.InnerPath, true), + url.QueryEscape(args.Password), + resp.Data.Sign), + }, nil +} + +func (d *AListV3) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error { + if !d.ForwardArchiveReq { + return errs.NotImplement + } + dir, name := path.Split(srcObj.GetPath()) + _, _, err := d.request("/fs/archive/decompress", http.MethodPost, func(req *resty.Request) { + req.SetBody(DecompressReq{ + ArchivePass: args.Password, + CacheFull: args.CacheFull, + DstDir: dstDir.GetPath(), + InnerPath: args.InnerPath, + Name: []string{name}, + PutIntoNewDir: args.PutIntoNewDir, + SrcDir: dir, + }) }) return err } diff --git a/drivers/alist_v3/meta.go b/drivers/alist_v3/meta.go index bb3d35aea22..1e8b3c53c0a 100644 --- a/drivers/alist_v3/meta.go +++ b/drivers/alist_v3/meta.go @@ -7,18 +7,21 @@ import ( type Addition struct { driver.RootPath - Address string `json:"url" required:"true"` - MetaPassword string `json:"meta_password"` - Username string `json:"username"` - Password string `json:"password"` - Token string `json:"token"` + Address string `json:"url" required:"true"` + MetaPassword string `json:"meta_password"` + Username string `json:"username"` + Password string `json:"password"` + Token string `json:"token"` + PassUAToUpsteam bool `json:"pass_ua_to_upsteam" default:"true"` + ForwardArchiveReq bool `json:"forward_archive_requests" default:"true"` } var config = driver.Config{ - Name: "AList V3", - LocalSort: true, - DefaultRoot: "/", - CheckStatus: true, + Name: "AList V3", + LocalSort: true, + DefaultRoot: "/", + CheckStatus: true, + ProxyRangeOption: true, } func init() { diff --git a/drivers/alist_v3/types.go b/drivers/alist_v3/types.go index e517307f3ef..3e8e2f71eac 100644 --- a/drivers/alist_v3/types.go +++ b/drivers/alist_v3/types.go @@ -1,9 +1,11 @@ package alist_v3 import ( + "encoding/json" "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" ) type ListReq struct { @@ -71,13 +73,113 @@ type LoginResp struct { } type MeResp struct { - Id int `json:"id"` - Username string `json:"username"` - Password string `json:"password"` - BasePath string `json:"base_path"` - Role int `json:"role"` - Disabled bool `json:"disabled"` - Permission int `json:"permission"` - SsoId string `json:"sso_id"` - Otp bool `json:"otp"` + Id int `json:"id"` + Username string `json:"username"` + Password string `json:"password"` + BasePath string `json:"base_path"` + Role IntSlice `json:"role"` + Disabled bool `json:"disabled"` + Permission int `json:"permission"` + SsoId string `json:"sso_id"` + Otp bool `json:"otp"` +} + +type ArchiveMetaReq struct { + ArchivePass string `json:"archive_pass"` + Password string `json:"password"` + Path string `json:"path"` + Refresh bool `json:"refresh"` +} + +type TreeResp struct { + ObjResp + Children []TreeResp `json:"children"` + hashCache *utils.HashInfo +} + +func (t *TreeResp) GetSize() int64 { + return t.Size +} + +func (t *TreeResp) GetName() string { + return t.Name +} + +func (t *TreeResp) ModTime() time.Time { + return t.Modified +} + +func (t *TreeResp) CreateTime() time.Time { + return t.Created +} + +func (t *TreeResp) IsDir() bool { + return t.ObjResp.IsDir +} + +func (t *TreeResp) GetHash() utils.HashInfo { + return utils.FromString(t.HashInfo) +} + +func (t *TreeResp) GetID() string { + return "" +} + +func (t *TreeResp) GetPath() string { + return "" +} + +func (t *TreeResp) GetChildren() []model.ObjTree { + ret := make([]model.ObjTree, 0, len(t.Children)) + for _, child := range t.Children { + ret = append(ret, &child) + } + return ret +} + +func (t *TreeResp) Thumb() string { + return t.ObjResp.Thumb +} + +type ArchiveMetaResp struct { + Comment string `json:"comment"` + Encrypted bool `json:"encrypted"` + Content []TreeResp `json:"content"` + RawURL string `json:"raw_url"` + Sign string `json:"sign"` +} + +type ArchiveListReq struct { + model.PageReq + ArchiveMetaReq + InnerPath string `json:"inner_path"` +} + +type ArchiveListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` +} + +type DecompressReq struct { + ArchivePass string `json:"archive_pass"` + CacheFull bool `json:"cache_full"` + DstDir string `json:"dst_dir"` + InnerPath string `json:"inner_path"` + Name []string `json:"name"` + PutIntoNewDir bool `json:"put_into_new_dir"` + SrcDir string `json:"src_dir"` +} + +type IntSlice []int + +func (s *IntSlice) UnmarshalJSON(data []byte) error { + if len(data) > 0 && data[0] == '[' { + return json.Unmarshal(data, (*[]int)(s)) + } + var single int + if err := json.Unmarshal(data, &single); err != nil { + return err + } + *s = []int{single} + return nil } diff --git a/drivers/alist_v3/util.go b/drivers/alist_v3/util.go index bf47c61289a..50c20250313 100644 --- a/drivers/alist_v3/util.go +++ b/drivers/alist_v3/util.go @@ -13,8 +13,11 @@ import ( ) func (d *AListV3) login() error { + if d.Username == "" { + return nil + } var resp common.Resp[LoginResp] - _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { + _, _, err := d.request("/auth/login", http.MethodPost, func(req *resty.Request) { req.SetResult(&resp).SetBody(base.Json{ "username": d.Username, "password": d.Password, @@ -28,7 +31,7 @@ func (d *AListV3) login() error { return nil } -func (d *AListV3) request(api, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { +func (d *AListV3) request(api, method string, callback base.ReqCallback, retry ...bool) ([]byte, int, error) { url := d.Address + "/api" + api req := base.RestyClient.R() req.SetHeader("Authorization", d.Token) @@ -37,22 +40,26 @@ func (d *AListV3) request(api, method string, callback base.ReqCallback, retry . } res, err := req.Execute(method, url) if err != nil { - return nil, err + code := 0 + if res != nil { + code = res.StatusCode() + } + return nil, code, err } log.Debugf("[alist_v3] response body: %s", res.String()) if res.StatusCode() >= 400 { - return nil, fmt.Errorf("request failed, status: %s", res.Status()) + return nil, res.StatusCode(), fmt.Errorf("request failed, status: %s", res.Status()) } code := utils.Json.Get(res.Body(), "code").ToInt() if code != 200 { if (code == 401 || code == 403) && !utils.IsBool(retry...) { err = d.login() if err != nil { - return nil, err + return nil, code, err } return d.request(api, method, callback, true) } - return nil, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) + return nil, code, fmt.Errorf("request failed,code: %d, message: %s", code, utils.Json.Get(res.Body(), "message").ToString()) } - return res.Body(), nil + return res.Body(), 200, nil } diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go index f11452629a4..606ff385e81 100644 --- a/drivers/aliyundrive/driver.go +++ b/drivers/aliyundrive/driver.go @@ -7,7 +7,6 @@ import ( "encoding/base64" "encoding/hex" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "math" "math/big" @@ -20,6 +19,7 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/cron" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -51,11 +51,11 @@ func (d *AliDrive) Init(ctx context.Context) error { return err } // get driver id - res, err, _ := d.request("https://api.aliyundrive.com/v2/user/get", http.MethodPost, nil, nil) + res, err, _ := d.request("https://api.alipan.com/v2/user/get", http.MethodPost, nil, nil) if err != nil { return err } - d.DriveId = utils.Json.Get(res, "default_drive_id").ToString() + d.DriveId = d.Addition.DeviceID d.UserID = utils.Json.Get(res, "user_id").ToString() d.cron = cron.NewCron(time.Hour * 2) d.cron.Do(func() { @@ -105,7 +105,7 @@ func (d *AliDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs "file_id": file.GetID(), "expire_sec": 14400, } - res, err, _ := d.request("https://api.aliyundrive.com/v2/file/get_download_url", http.MethodPost, func(req *resty.Request) { + res, err, _ := d.request("https://api.alipan.com/v2/file/get_download_url", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) if err != nil { @@ -113,14 +113,14 @@ func (d *AliDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs } return &model.Link{ Header: http.Header{ - "Referer": []string{"https://www.aliyundrive.com/"}, + "Referer": []string{"https://www.alipan.com/"}, }, URL: utils.Json.Get(res, "url").ToString(), }, nil } func (d *AliDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err, _ := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "check_name_mode": "refuse", "drive_id": d.DriveId, @@ -138,7 +138,7 @@ func (d *AliDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *AliDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err, _ := d.request("https://api.aliyundrive.com/v3/file/update", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/v3/file/update", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "check_name_mode": "refuse", "drive_id": d.DriveId, @@ -155,7 +155,7 @@ func (d *AliDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *AliDrive) Remove(ctx context.Context, obj model.Obj) error { - _, err, _ := d.request("https://api.aliyundrive.com/v2/recyclebin/trash", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/v2/recyclebin/trash", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": obj.GetID(), @@ -193,7 +193,10 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil } if d.RapidUpload { buf := bytes.NewBuffer(make([]byte, 0, 1024)) - io.CopyN(buf, file, 1024) + _, err := utils.CopyWithBufferN(buf, file, 1024) + if err != nil { + return err + } reqBody["pre_hash"] = utils.HashData(utils.SHA1, buf.Bytes()) if localFile != nil { if _, err := localFile.Seek(0, io.SeekStart); err != nil { @@ -215,7 +218,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil } var resp UploadResp - _, err, e := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + _, err, e := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(reqBody) }, &resp) @@ -269,7 +272,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil n, _ := io.NewSectionReader(localFile, o.Int64(), 8).Read(buf[:8]) reqBody["proof_code"] = base64.StdEncoding.EncodeToString(buf[:n]) - _, err, e := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + _, err, e := d.request("https://api.alipan.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { req.SetBody(reqBody) }, &resp) if err != nil && e.Code != "PreHashMatched" { @@ -285,6 +288,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil file.Reader = localFile } + rateLimited := driver.NewLimitedUploadStream(ctx, file) for i, partInfo := range resp.PartInfoList { if utils.IsCanceled(ctx) { return ctx.Err() @@ -293,7 +297,7 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil if d.InternalUpload { url = partInfo.InternalUploadUrl } - req, err := http.NewRequest("PUT", url, io.LimitReader(file, DEFAULT)) + req, err := http.NewRequest("PUT", url, io.LimitReader(rateLimited, DEFAULT)) if err != nil { return err } @@ -302,13 +306,13 @@ func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, streamer model.Fil if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() if count > 0 { - up(i * 100 / count) + up(float64(i) * 100 / float64(count)) } } var resp2 base.Json - _, err, e = d.request("https://api.aliyundrive.com/v2/file/complete", http.MethodPost, func(req *resty.Request) { + _, err, e = d.request("https://api.alipan.com/v2/file/complete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": resp.FileId, @@ -333,10 +337,10 @@ func (d *AliDrive) Other(ctx context.Context, args model.OtherArgs) (interface{} } switch args.Method { case "doc_preview": - url = "https://api.aliyundrive.com/v2/file/get_office_preview_url" + url = "https://api.alipan.com/v2/file/get_office_preview_url" data["access_token"] = d.AccessToken case "video_preview": - url = "https://api.aliyundrive.com/v2/file/get_video_preview_play_info" + url = "https://api.alipan.com/v2/file/get_video_preview_play_info" data["category"] = "live_transcoding" data["url_expire_sec"] = 14400 default: diff --git a/drivers/aliyundrive/meta.go b/drivers/aliyundrive/meta.go index 9aee856908d..a0ae8a5917d 100644 --- a/drivers/aliyundrive/meta.go +++ b/drivers/aliyundrive/meta.go @@ -7,8 +7,8 @@ import ( type Addition struct { driver.RootID - RefreshToken string `json:"refresh_token" required:"true"` - //DeviceID string `json:"device_id" required:"true"` + RefreshToken string `json:"refresh_token" required:"true"` + DeviceID string `json:"device_id" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` RapidUpload bool `json:"rapid_upload"` diff --git a/drivers/aliyundrive/util.go b/drivers/aliyundrive/util.go index b36900fb964..0e81b082bb9 100644 --- a/drivers/aliyundrive/util.go +++ b/drivers/aliyundrive/util.go @@ -26,7 +26,7 @@ func (d *AliDrive) createSession() error { state.retry = 0 return fmt.Errorf("createSession failed after three retries") } - _, err, _ := d.request("https://api.aliyundrive.com/users/v1/users/device/create_session", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/users/v1/users/device/create_session", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "deviceName": "samsung", "modelName": "SM-G9810", @@ -42,7 +42,7 @@ func (d *AliDrive) createSession() error { } // func (d *AliDrive) renewSession() error { -// _, err, _ := d.request("https://api.aliyundrive.com/users/v1/users/device/renew_session", http.MethodPost, nil, nil) +// _, err, _ := d.request("https://api.alipan.com/users/v1/users/device/renew_session", http.MethodPost, nil, nil) // return err // } @@ -58,7 +58,7 @@ func (d *AliDrive) sign() { // do others that not defined in Driver interface func (d *AliDrive) refreshToken() error { - url := "https://auth.aliyundrive.com/v2/account/token" + url := "https://auth.alipan.com/v2/account/token" var resp base.TokenResp var e RespErr _, err := base.RestyClient.R(). @@ -85,7 +85,7 @@ func (d *AliDrive) request(url, method string, callback base.ReqCallback, resp i req := base.RestyClient.R() state, ok := global.Load(d.UserID) if !ok { - if url == "https://api.aliyundrive.com/v2/user/get" { + if url == "https://api.alipan.com/v2/user/get" { state = &State{} } else { return nil, fmt.Errorf("can't load user state, user_id: %s", d.UserID), RespErr{} @@ -94,8 +94,8 @@ func (d *AliDrive) request(url, method string, callback base.ReqCallback, resp i req.SetHeaders(map[string]string{ "Authorization": "Bearer\t" + d.AccessToken, "content-type": "application/json", - "origin": "https://www.aliyundrive.com", - "Referer": "https://aliyundrive.com/", + "origin": "https://www.alipan.com", + "Referer": "https://alipan.com/", "X-Signature": state.signature, "x-request-id": uuid.NewString(), "X-Canary": "client=Android,app=adrive,version=v4.1.0", @@ -158,7 +158,7 @@ func (d *AliDrive) getFiles(fileId string) ([]File, error) { "video_thumbnail_process": "video/snapshot,t_0,f_jpg,ar_auto,w_300", "url_expire_sec": 14400, } - _, err, _ := d.request("https://api.aliyundrive.com/v2/file/list", http.MethodPost, func(req *resty.Request) { + _, err, _ := d.request("https://api.alipan.com/v2/file/list", http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, &resp) @@ -172,7 +172,7 @@ func (d *AliDrive) getFiles(fileId string) ([]File, error) { } func (d *AliDrive) batch(srcId, dstId string, url string) error { - res, err, _ := d.request("https://api.aliyundrive.com/v3/batch", http.MethodPost, func(req *resty.Request) { + res, err, _ := d.request("https://api.alipan.com/v3/batch", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "requests": []base.Json{ { diff --git a/drivers/aliyundrive_open/driver.go b/drivers/aliyundrive_open/driver.go index bc41e56b6fd..5ef814184e3 100644 --- a/drivers/aliyundrive_open/driver.go +++ b/drivers/aliyundrive_open/driver.go @@ -3,28 +3,27 @@ package aliyundrive_open import ( "context" "errors" - "fmt" "net/http" + "path/filepath" "time" - "github.com/Xhofe/rateg" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" ) type AliyundriveOpen struct { model.Storage Addition - base string DriveId string - limitList func(ctx context.Context, data base.Json) (*Files, error) - limitLink func(ctx context.Context, file model.Obj) (*model.Link, error) + limiter *limiter + ref *AliyundriveOpen } func (d *AliyundriveOpen) Config() driver.Config { @@ -36,47 +35,74 @@ func (d *AliyundriveOpen) GetAddition() driver.Additional { } func (d *AliyundriveOpen) Init(ctx context.Context) error { + d.limiter = getLimiterForUser(globalLimiterUserID) if d.LIVPDownloadFormat == "" { d.LIVPDownloadFormat = "jpeg" } if d.DriveType == "" { d.DriveType = "default" } - res, err := d.request("/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) + res, err := d.request(ctx, limiterOther, "/adrive/v1.0/user/getDriveInfo", http.MethodPost, nil) if err != nil { + d.limiter.free() + d.limiter = nil return err } d.DriveId = utils.Json.Get(res, d.DriveType+"_drive_id").ToString() - d.limitList = rateg.LimitFnCtx(d.list, rateg.LimitFnOption{ - Limit: 4, - Bucket: 1, - }) - d.limitLink = rateg.LimitFnCtx(d.link, rateg.LimitFnOption{ - Limit: 1, - Bucket: 1, - }) + userID := utils.Json.Get(res, "user_id").ToString() + d.limiter.free() + d.limiter = getLimiterForUser(userID) return nil } +func (d *AliyundriveOpen) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*AliyundriveOpen) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + func (d *AliyundriveOpen) Drop(ctx context.Context) error { + d.limiter.free() + d.limiter = nil + d.ref = nil return nil } +// GetRoot implements the driver.GetRooter interface to properly set up the root object +func (d *AliyundriveOpen) GetRoot(ctx context.Context) (model.Obj, error) { + return &model.Object{ + ID: d.RootFolderID, + Path: "/", + Name: "root", + Size: 0, + Modified: d.Modified, + IsFolder: true, + }, nil +} + func (d *AliyundriveOpen) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - if d.limitList == nil { - return nil, fmt.Errorf("driver not init") - } files, err := d.getFiles(ctx, dir.GetID()) if err != nil { return nil, err } - return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil + + objs, err := utils.SliceConvert(files, func(src File) (model.Obj, error) { + obj := fileToObj(src) + // Set the correct path for the object + if dir.GetPath() != "" { + obj.Path = filepath.Join(dir.GetPath(), obj.GetName()) + } + return obj, nil }) + + return objs, err } func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link, error) { - res, err := d.request("/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { + res, err := d.request(ctx, limiterLink, "/adrive/v1.0/openFile/getDownloadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": file.GetID(), @@ -93,7 +119,7 @@ func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link } url = utils.Json.Get(res, "streamsUrl", d.LIVPDownloadFormat).ToString() } - exp := time.Hour + exp := time.Minute return &model.Link{ URL: url, Expiration: &exp, @@ -101,16 +127,13 @@ func (d *AliyundriveOpen) link(ctx context.Context, file model.Obj) (*model.Link } func (d *AliyundriveOpen) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - if d.limitLink == nil { - return nil, fmt.Errorf("driver not init") - } - return d.limitLink(ctx, file) + return d.link(ctx, file) } func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { nowTime, _ := getNowTime() newDir := File{CreatedAt: nowTime, UpdatedAt: nowTime} - _, err := d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "parent_file_id": parentDir.GetID(), @@ -122,30 +145,43 @@ func (d *AliyundriveOpen) MakeDir(ctx context.Context, parentDir model.Obj, dirN if err != nil { return nil, err } - return fileToObj(newDir), nil + obj := fileToObj(newDir) + + // Set the correct Path for the returned directory object + if parentDir.GetPath() != "" { + obj.Path = filepath.Join(parentDir.GetPath(), dirName) + } else { + obj.Path = "/" + dirName + } + + return obj, nil } func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { var resp MoveOrCopyResp - _, err := d.request("/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/move", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), "to_parent_file_id": dstDir.GetID(), - "check_name_mode": "refuse", // optional:ignore,auto_rename,refuse + "check_name_mode": "ignore", // optional:ignore,auto_rename,refuse //"new_name": "newName", // The new name to use when a file of the same name exists }).SetResult(&resp) }) if err != nil { return nil, err } - if resp.Exist { - return nil, errors.New("existence of files with the same name") - } if srcObj, ok := srcObj.(*model.ObjThumb); ok { srcObj.ID = resp.FileID srcObj.Modified = time.Now() + srcObj.Path = filepath.Join(dstDir.GetPath(), srcObj.GetName()) + + // Check for duplicate files in the destination directory + if err := d.removeDuplicateFiles(ctx, dstDir.GetPath(), srcObj.GetName(), srcObj.GetID()); err != nil { + // Only log a warning instead of returning an error since the move operation has already completed successfully + log.Warnf("Failed to remove duplicate files after move: %v", err) + } return srcObj, nil } return nil, nil @@ -153,7 +189,7 @@ func (d *AliyundriveOpen) Move(ctx context.Context, srcObj, dstDir model.Obj) (m func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { var newFile File - _, err := d.request("/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/update", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), @@ -163,19 +199,47 @@ func (d *AliyundriveOpen) Rename(ctx context.Context, srcObj model.Obj, newName if err != nil { return nil, err } - return fileToObj(newFile), nil + + // Check for duplicate files in the parent directory + parentPath := filepath.Dir(srcObj.GetPath()) + if err := d.removeDuplicateFiles(ctx, parentPath, newName, newFile.FileId); err != nil { + // Only log a warning instead of returning an error since the rename operation has already completed successfully + log.Warnf("Failed to remove duplicate files after rename: %v", err) + } + + obj := fileToObj(newFile) + + // Set the correct Path for the renamed object + if parentPath != "" && parentPath != "." { + obj.Path = filepath.Join(parentPath, newName) + } else { + obj.Path = "/" + newName + } + + return obj, nil } func (d *AliyundriveOpen) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { + var resp MoveOrCopyResp + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/copy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": srcObj.GetID(), "to_parent_file_id": dstDir.GetID(), - "auto_rename": true, - }) + "auto_rename": false, + }).SetResult(&resp) }) - return err + if err != nil { + return err + } + + // Check for duplicate files in the destination directory + if err := d.removeDuplicateFiles(ctx, dstDir.GetPath(), srcObj.GetName(), resp.FileID); err != nil { + // Only log a warning instead of returning an error since the copy operation has already completed successfully + log.Warnf("Failed to remove duplicate files after copy: %v", err) + } + + return nil } func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { @@ -183,7 +247,7 @@ func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { if d.RemoveWay == "delete" { uri = "/adrive/v1.0/openFile/delete" } - _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": obj.GetID(), @@ -193,7 +257,18 @@ func (d *AliyundriveOpen) Remove(ctx context.Context, obj model.Obj) error { } func (d *AliyundriveOpen) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - return d.upload(ctx, dstDir, stream, up) + obj, err := d.upload(ctx, dstDir, stream, up) + + // Set the correct Path for the returned file object + if obj != nil && obj.GetPath() == "" { + if dstDir.GetPath() != "" { + if objWithPath, ok := obj.(model.SetPath); ok { + objWithPath.SetPath(filepath.Join(dstDir.GetPath(), obj.GetName())) + } + } + } + + return obj, err } func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { @@ -211,7 +286,7 @@ func (d *AliyundriveOpen) Other(ctx context.Context, args model.OtherArgs) (inte default: return nil, errs.NotSupport } - _, err := d.request(uri, http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, uri, http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { @@ -225,3 +300,4 @@ var _ driver.MkdirResult = (*AliyundriveOpen)(nil) var _ driver.MoveResult = (*AliyundriveOpen)(nil) var _ driver.RenameResult = (*AliyundriveOpen)(nil) var _ driver.PutResult = (*AliyundriveOpen)(nil) +var _ driver.GetRooter = (*AliyundriveOpen)(nil) diff --git a/drivers/aliyundrive_open/limiter.go b/drivers/aliyundrive_open/limiter.go new file mode 100644 index 00000000000..5138fbe2e50 --- /dev/null +++ b/drivers/aliyundrive_open/limiter.go @@ -0,0 +1,97 @@ +package aliyundrive_open + +import ( + "context" + "fmt" + "sync" + + "golang.org/x/time/rate" +) + +// Aliyun Open API rate limits are per user per app, so requests for the same +// user should share one limiter across all storage instances. +type limiterType int + +const ( + limiterList limiterType = iota + limiterLink + limiterOther +) + +const ( + listRateLimit = 3.9 + linkRateLimit = 0.9 + otherRateLimit = 14.9 + globalLimiterUserID = "" +) + +type limiter struct { + usedBy int + list *rate.Limiter + link *rate.Limiter + other *rate.Limiter +} + +var ( + limiters = make(map[string]*limiter) + limitersLock sync.Mutex +) + +func getLimiterForUser(userID string) *limiter { + limitersLock.Lock() + defer limitersLock.Unlock() + defer func() { + for id, lim := range limiters { + if lim.usedBy <= 0 && id != globalLimiterUserID { + delete(limiters, id) + } + } + }() + if lim, ok := limiters[userID]; ok { + lim.usedBy++ + return lim + } + lim := &limiter{ + usedBy: 1, + list: rate.NewLimiter(rate.Limit(listRateLimit), 1), + link: rate.NewLimiter(rate.Limit(linkRateLimit), 1), + other: rate.NewLimiter(rate.Limit(otherRateLimit), 1), + } + limiters[userID] = lim + return lim +} + +func (l *limiter) wait(ctx context.Context, typ limiterType) error { + if l == nil { + return fmt.Errorf("driver not init") + } + switch typ { + case limiterList: + return l.list.Wait(ctx) + case limiterLink: + return l.link.Wait(ctx) + case limiterOther: + return l.other.Wait(ctx) + default: + return fmt.Errorf("unknown limiter type") + } +} + +func (l *limiter) free() { + if l == nil { + return + } + limitersLock.Lock() + defer limitersLock.Unlock() + l.usedBy-- +} + +func (d *AliyundriveOpen) wait(ctx context.Context, typ limiterType) error { + if d == nil { + return fmt.Errorf("driver not init") + } + if d.ref != nil { + return d.ref.wait(ctx, typ) + } + return d.limiter.wait(ctx, typ) +} diff --git a/drivers/aliyundrive_open/meta.go b/drivers/aliyundrive_open/meta.go index bd69211c77e..bb4354ddc11 100644 --- a/drivers/aliyundrive_open/meta.go +++ b/drivers/aliyundrive_open/meta.go @@ -6,12 +6,12 @@ import ( ) type Addition struct { - DriveType string `json:"drive_type" type:"select" options:"default,resource,backup" default:"default"` + DriveType string `json:"drive_type" type:"select" options:"default,resource,backup" default:"resource"` driver.RootID RefreshToken string `json:"refresh_token" required:"true"` OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` - OauthTokenURL string `json:"oauth_token_url" default:"https://api.nn.ci/alist/ali_open/token"` + OauthTokenURL string `json:"oauth_token_url" default:"https://api.alistgo.com/alist/ali_open/token"` ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` @@ -32,11 +32,10 @@ var config = driver.Config{ DefaultRoot: "root", NoOverwriteUpload: true, } +var API_URL = "https://openapi.alipan.com" func init() { op.RegisterDriver(func() driver.Driver { - return &AliyundriveOpen{ - base: "https://openapi.aliyundrive.com", - } + return &AliyundriveOpen{} }) } diff --git a/drivers/aliyundrive_open/upload.go b/drivers/aliyundrive_open/upload.go index e4a0cf7ec0d..19a3c3d0f05 100644 --- a/drivers/aliyundrive_open/upload.go +++ b/drivers/aliyundrive_open/upload.go @@ -1,11 +1,9 @@ package aliyundrive_open import ( - "bytes" "context" "encoding/base64" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" "io" "math" "net/http" @@ -16,6 +14,8 @@ import ( "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + streamPkg "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" "github.com/go-resty/resty/v2" @@ -50,10 +50,10 @@ func calPartSize(fileSize int64) int64 { return partSize } -func (d *AliyundriveOpen) getUploadUrl(count int, fileId, uploadId string) ([]PartInfo, error) { +func (d *AliyundriveOpen) getUploadUrl(ctx context.Context, count int, fileId, uploadId string) ([]PartInfo, error) { partInfoList := makePartInfos(count) var resp CreateResp - _, err := d.request("/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/getUploadUrl", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, @@ -77,17 +77,17 @@ func (d *AliyundriveOpen) uploadPart(ctx context.Context, r io.Reader, partInfo if err != nil { return err } - res.Body.Close() + _ = res.Body.Close() if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusConflict { return fmt.Errorf("upload status: %d", res.StatusCode) } return nil } -func (d *AliyundriveOpen) completeUpload(fileId, uploadId string) (model.Obj, error) { +func (d *AliyundriveOpen) completeUpload(ctx context.Context, fileId, uploadId string) (model.Obj, error) { // 3. complete var newFile File - _, err := d.request("/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/complete", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "drive_id": d.DriveId, "file_id": fileId, @@ -126,21 +126,24 @@ func getProofRange(input string, size int64) (*ProofRange, error) { } func (d *AliyundriveOpen) calProofCode(stream model.FileStreamer) (string, error) { - proofRange, err := getProofRange(d.AccessToken, stream.GetSize()) + proofRange, err := getProofRange(d.getAccessToken(), stream.GetSize()) if err != nil { return "", err } length := proofRange.End - proofRange.Start - buf := bytes.NewBuffer(make([]byte, 0, length)) reader, err := stream.RangeRead(http_range.Range{Start: proofRange.Start, Length: length}) if err != nil { return "", err } - _, err = io.CopyN(buf, reader, length) + buf := make([]byte, length) + n, err := io.ReadFull(reader, buf) + if err == io.ErrUnexpectedEOF { + return "", fmt.Errorf("can't read data, expected=%d, got=%d", len(buf), n) + } if err != nil { return "", err } - return base64.StdEncoding.EncodeToString(buf.Bytes()), nil + return base64.StdEncoding.EncodeToString(buf), nil } func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { @@ -164,7 +167,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m count := int(math.Ceil(float64(stream.GetSize()) / float64(partSize))) createData["part_info_list"] = makePartInfos(count) // rapid upload - rapidUpload := stream.GetSize() > 100*utils.KB && d.RapidUpload + rapidUpload := !stream.IsForceStreamUpload() && stream.GetSize() > 100*utils.KB && d.RapidUpload if rapidUpload { log.Debugf("[aliyundrive_open] start cal pre_hash") // read 1024 bytes to calculate pre hash @@ -180,28 +183,21 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m createData["pre_hash"] = hash } var createResp CreateResp - _, err, e := d.requestReturnErrResp("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err, e := d.requestReturnErrResp(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) - var tmpF model.File if err != nil { if e.Code != "PreHashMatched" || !rapidUpload { return nil, err } log.Debugf("[aliyundrive_open] pre_hash matched, start rapid upload") - hi := stream.GetHash() - hash := hi.GetHash(utils.SHA1) - if len(hash) <= 0 { - tmpF, err = stream.CacheFullInTempFile() - if err != nil { - return nil, err - } - hash, err = utils.HashFile(utils.SHA1, tmpF) + hash := stream.GetHash().GetHash(utils.SHA1) + if len(hash) != utils.SHA1.Width { + _, hash, err = streamPkg.CacheFullInTempFileAndHash(stream, utils.SHA1) if err != nil { return nil, err } - } delete(createData, "pre_hash") @@ -212,7 +208,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m if err != nil { return nil, fmt.Errorf("cal proof code error: %s", err.Error()) } - _, err = d.request("/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { + _, err = d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) { req.SetBody(createData).SetResult(&createResp) }) if err != nil { @@ -233,7 +229,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m } // refresh upload url if 50 minutes passed if time.Since(preTime) > 50*time.Minute { - createResp.PartInfoList, err = d.getUploadUrl(count, createResp.FileId, createResp.UploadId) + createResp.PartInfoList, err = d.getUploadUrl(ctx, count, createResp.FileId, createResp.UploadId) if err != nil { return nil, err } @@ -242,14 +238,18 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m if remain := stream.GetSize() - offset; length > remain { length = remain } - //rd := utils.NewMultiReadable(io.LimitReader(stream, partSize)) - rd, err := stream.RangeRead(http_range.Range{Start: offset, Length: length}) - if err != nil { - return nil, err + rd := utils.NewMultiReadable(io.LimitReader(stream, partSize)) + if rapidUpload { + srd, err := stream.RangeRead(http_range.Range{Start: offset, Length: length}) + if err != nil { + return nil, err + } + rd = utils.NewMultiReadable(srd) } err = retry.Do(func() error { - //rd.Reset() - return d.uploadPart(ctx, rd, createResp.PartInfoList[i]) + _ = rd.Reset() + rateLimitedRd := driver.NewLimitedUploadStream(ctx, rd) + return d.uploadPart(ctx, rateLimitedRd, createResp.PartInfoList[i]) }, retry.Attempts(3), retry.DelayType(retry.BackOffDelay), @@ -258,6 +258,7 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m return nil, err } offset += partSize + up(float64(i*100) / float64(count)) } } else { log.Debugf("[aliyundrive_open] rapid upload success, file id: %s", createResp.FileId) @@ -265,5 +266,5 @@ func (d *AliyundriveOpen) upload(ctx context.Context, dstDir model.Obj, stream m log.Debugf("[aliyundrive_open] create file success, resp: %+v", createResp) // 3. complete - return d.completeUpload(createResp.FileId, createResp.UploadId) + return d.completeUpload(ctx, createResp.FileId, createResp.UploadId) } diff --git a/drivers/aliyundrive_open/util.go b/drivers/aliyundrive_open/util.go index 331e6400c97..544c8bdd10b 100644 --- a/drivers/aliyundrive_open/util.go +++ b/drivers/aliyundrive_open/util.go @@ -10,6 +10,7 @@ import ( "time" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -18,13 +19,94 @@ import ( // do others that not defined in Driver interface -func (d *AliyundriveOpen) _refreshToken() (string, string, error) { - url := d.base + "/oauth/access_token" +const legacyOauthTokenURL = "https://api.alistgo.com/alist/ali_open/token" + +type refreshRateLimitError struct { + message string + retryAfter time.Duration +} + +func (e *refreshRateLimitError) Error() string { + if e.retryAfter > 0 { + return fmt.Sprintf("%s, retry after %s", e.message, e.retryAfter.Round(time.Second)) + } + return e.message +} + +func (d *AliyundriveOpen) _refreshToken(ctx context.Context) (string, string, error) { if d.OauthTokenURL != "" && d.ClientID == "" { - url = d.OauthTokenURL + return d.refreshTokenWithOnlineAPI(ctx) + } + + if d.ClientID == "" || d.ClientSecret == "" { + return "", "", fmt.Errorf("empty ClientID or ClientSecret") + } + return d.refreshTokenWithPost(ctx, API_URL+"/oauth/access_token") +} + +func (d *AliyundriveOpen) refreshTokenWithOnlineAPI(ctx context.Context) (string, string, error) { + // New hosted renew endpoint uses a GET API and returns {"refresh_token","access_token","text"}. + // Older hosted endpoints still expect the legacy POST payload, so we fall back when we detect that shape. + var resp struct { + RefreshToken string `json:"refresh_token"` + AccessToken string `json:"access_token"` + ErrorMessage string `json:"text"` + } + var e ErrResp + if err := d.wait(ctx, limiterOther); err != nil { + return "", "", err + } + res, err := base.RestyClient.R(). + SetResult(&resp). + SetError(&e). + SetQueryParams(map[string]string{ + "refresh_ui": d.RefreshToken, + "server_use": "true", + "driver_txt": "alicloud_qr", + }). + Get(d.OauthTokenURL) + if err != nil { + return "", "", err } + if resp.RefreshToken != "" && resp.AccessToken != "" { + return resp.RefreshToken, resp.AccessToken, nil + } + if resp.ErrorMessage != "" { + if res != nil && res.StatusCode() == http.StatusTooManyRequests { + return d.refreshTokenWithLegacyFallback(ctx, resp.ErrorMessage, retryAfterFromResponse(res)) + } + return "", "", fmt.Errorf("failed to refresh token: %s", resp.ErrorMessage) + } + if res != nil && res.StatusCode() == http.StatusTooManyRequests { + return d.refreshTokenWithLegacyFallback(ctx, http.StatusText(http.StatusTooManyRequests), retryAfterFromResponse(res)) + } + if e.Code != "" || e.Message != "" { + return d.refreshTokenWithPost(ctx, d.OauthTokenURL) + } + return "", "", fmt.Errorf("empty token returned from online API") +} + +func (d *AliyundriveOpen) refreshTokenWithLegacyFallback(ctx context.Context, message string, retryAfter time.Duration) (string, string, error) { + if d.OauthTokenURL != legacyOauthTokenURL { + log.Warnf("[ali_open] online refresh API is rate-limited, trying legacy fallback: %s", legacyOauthTokenURL) + if refresh, access, err := d.refreshTokenWithPost(ctx, legacyOauthTokenURL); err == nil { + return refresh, access, nil + } else if _, ok := err.(*refreshRateLimitError); !ok { + return "", "", err + } + } + return "", "", &refreshRateLimitError{ + message: fmt.Sprintf("failed to refresh token: %s", message), + retryAfter: retryAfter, + } +} + +func (d *AliyundriveOpen) refreshTokenWithPost(ctx context.Context, url string) (string, string, error) { //var resp base.TokenResp var e ErrResp + if err := d.wait(ctx, limiterOther); err != nil { + return "", "", err + } res, err := base.RestyClient.R(). //ForceContentType("application/json"). SetBody(base.Json{ @@ -40,6 +122,16 @@ func (d *AliyundriveOpen) _refreshToken() (string, string, error) { return "", "", err } log.Debugf("[ali_open] refresh token response: %s", res.String()) + if res.StatusCode() == http.StatusTooManyRequests { + message := e.Message + if message == "" { + message = http.StatusText(http.StatusTooManyRequests) + } + return "", "", &refreshRateLimitError{ + message: fmt.Sprintf("failed to refresh token: %s", message), + retryAfter: retryAfterFromResponse(res), + } + } if e.Code != "" { return "", "", fmt.Errorf("failed to refresh token: %s", e.Message) } @@ -73,15 +165,29 @@ func getSub(token string) (string, error) { return utils.Json.Get(bs, "sub").ToString(), nil } -func (d *AliyundriveOpen) refreshToken() error { - refresh, access, err := d._refreshToken() +func (d *AliyundriveOpen) refreshToken(ctx context.Context) error { + if d.ref != nil { + return d.ref.refreshToken(ctx) + } + refresh, access, err := d._refreshToken(ctx) for i := 0; i < 3; i++ { if err == nil { break + } + if rateLimitErr, ok := err.(*refreshRateLimitError); ok { + wait := rateLimitErr.retryAfter + if wait <= 0 { + wait = time.Duration(i+1) * 2 * time.Second + } + if wait > 15*time.Second { + wait = 15 * time.Second + } + log.Warnf("[ali_open] %s", rateLimitErr.Error()) + time.Sleep(wait) } else { log.Errorf("[ali_open] failed to refresh token: %s", err) } - refresh, access, err = d._refreshToken() + refresh, access, err = d._refreshToken(ctx) } if err != nil { return err @@ -92,15 +198,32 @@ func (d *AliyundriveOpen) refreshToken() error { return nil } -func (d *AliyundriveOpen) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { - b, err, _ := d.requestReturnErrResp(uri, method, callback, retry...) +func retryAfterFromResponse(res *resty.Response) time.Duration { + if res == nil { + return 0 + } + retryAfter := strings.TrimSpace(res.Header().Get("Retry-After")) + if retryAfter == "" { + return 0 + } + if seconds, err := time.ParseDuration(retryAfter + "s"); err == nil { + return seconds + } + if t, err := http.ParseTime(retryAfter); err == nil { + return time.Until(t) + } + return 0 +} + +func (d *AliyundriveOpen) request(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { + b, err, _ := d.requestReturnErrResp(ctx, limitTy, uri, method, callback, retry...) return b, err } -func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { +func (d *AliyundriveOpen) requestReturnErrResp(ctx context.Context, limitTy limiterType, uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error, *ErrResp) { req := base.RestyClient.R() // TODO check whether access_token is expired - req.SetHeader("Authorization", "Bearer "+d.AccessToken) + req.SetHeader("Authorization", "Bearer "+d.getAccessToken()) if method == http.MethodPost { req.SetHeader("Content-Type", "application/json") } @@ -109,7 +232,10 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base } var e ErrResp req.SetError(&e) - res, err := req.Execute(method, d.base+uri) + if err := d.wait(ctx, limitTy); err != nil { + return nil, err, nil + } + res, err := req.Execute(method, API_URL+uri) if err != nil { if res != nil { log.Errorf("[aliyundrive_open] request error: %s", res.String()) @@ -118,12 +244,12 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base } isRetry := len(retry) > 0 && retry[0] if e.Code != "" { - if !isRetry && (utils.SliceContains([]string{"AccessTokenInvalid", "AccessTokenExpired", "I400JD"}, e.Code) || d.AccessToken == "") { - err = d.refreshToken() + if !isRetry && (utils.SliceContains([]string{"AccessTokenInvalid", "AccessTokenExpired", "I400JD"}, e.Code) || d.getAccessToken() == "") { + err = d.refreshToken(ctx) if err != nil { return nil, err, nil } - return d.requestReturnErrResp(uri, method, callback, true) + return d.requestReturnErrResp(ctx, limitTy, uri, method, callback, true) } return nil, fmt.Errorf("%s:%s", e.Code, e.Message), &e } @@ -132,7 +258,7 @@ func (d *AliyundriveOpen) requestReturnErrResp(uri, method string, callback base func (d *AliyundriveOpen) list(ctx context.Context, data base.Json) (*Files, error) { var resp Files - _, err := d.request("/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { + _, err := d.request(ctx, limiterList, "/adrive/v1.0/openFile/list", http.MethodPost, func(req *resty.Request) { req.SetBody(data).SetResult(&resp) }) if err != nil { @@ -161,7 +287,7 @@ func (d *AliyundriveOpen) getFiles(ctx context.Context, fileId string) ([]File, //"video_thumbnail_width": 480, //"image_thumbnail_width": 480, } - resp, err := d.limitList(ctx, data) + resp, err := d.list(ctx, data) if err != nil { return nil, err } @@ -176,3 +302,43 @@ func getNowTime() (time.Time, string) { nowTimeStr := nowTime.Format("2006-01-02T15:04:05.000Z") return nowTime, nowTimeStr } + +func (d *AliyundriveOpen) getAccessToken() string { + if d.ref != nil { + return d.ref.getAccessToken() + } + return d.AccessToken +} + +// Remove duplicate files with the same name in the given directory path, +// preserving the file with the given skipID if provided +func (d *AliyundriveOpen) removeDuplicateFiles(ctx context.Context, parentPath string, fileName string, skipID string) error { + // Handle empty path (root directory) case + if parentPath == "" { + parentPath = "/" + } + + // List all files in the parent directory + files, err := op.List(ctx, d, parentPath, model.ListArgs{}) + if err != nil { + return err + } + + // Find all files with the same name + var duplicates []model.Obj + for _, file := range files { + if file.GetName() == fileName && file.GetID() != skipID { + duplicates = append(duplicates, file) + } + } + + // Remove all duplicates files, except the file with the given ID + for _, file := range duplicates { + err := d.Remove(ctx, file) + if err != nil { + return err + } + } + + return nil +} diff --git a/drivers/aliyundrive_share/driver.go b/drivers/aliyundrive_share/driver.go index 2e042ceef35..db2ff9ace16 100644 --- a/drivers/aliyundrive_share/driver.go +++ b/drivers/aliyundrive_share/driver.go @@ -105,7 +105,7 @@ func (d *AliyundriveShare) link(ctx context.Context, file model.Obj) (*model.Lin "share_id": d.ShareId, } var resp ShareLinkResp - _, err := d.request("https://api.aliyundrive.com/v2/file/get_share_link_download_url", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api.alipan.com/v2/file/get_share_link_download_url", http.MethodPost, func(req *resty.Request) { req.SetHeader(CanaryHeaderKey, CanaryHeaderValue).SetBody(data).SetResult(&resp) }) if err != nil { @@ -113,7 +113,7 @@ func (d *AliyundriveShare) link(ctx context.Context, file model.Obj) (*model.Lin } return &model.Link{ Header: http.Header{ - "Referer": []string{"https://www.aliyundrive.com/"}, + "Referer": []string{"https://www.alipan.com/"}, }, URL: resp.DownloadUrl, }, nil @@ -128,9 +128,9 @@ func (d *AliyundriveShare) Other(ctx context.Context, args model.OtherArgs) (int } switch args.Method { case "doc_preview": - url = "https://api.aliyundrive.com/v2/file/get_office_preview_url" + url = "https://api.alipan.com/v2/file/get_office_preview_url" case "video_preview": - url = "https://api.aliyundrive.com/v2/file/get_video_preview_play_info" + url = "https://api.alipan.com/v2/file/get_video_preview_play_info" data["category"] = "live_transcoding" default: return nil, errs.NotSupport diff --git a/drivers/aliyundrive_share/util.go b/drivers/aliyundrive_share/util.go index a29f86f5522..899e15cec1b 100644 --- a/drivers/aliyundrive_share/util.go +++ b/drivers/aliyundrive_share/util.go @@ -16,7 +16,7 @@ const ( ) func (d *AliyundriveShare) refreshToken() error { - url := "https://auth.aliyundrive.com/v2/account/token" + url := "https://auth.alipan.com/v2/account/token" var resp base.TokenResp var e ErrorResp _, err := base.RestyClient.R(). @@ -47,7 +47,7 @@ func (d *AliyundriveShare) getShareToken() error { var resp ShareTokenResp _, err := base.RestyClient.R(). SetResult(&resp).SetError(&e).SetBody(data). - Post("https://api.aliyundrive.com/v2/share_link/get_share_token") + Post("https://api.alipan.com/v2/share_link/get_share_token") if err != nil { return err } @@ -116,7 +116,7 @@ func (d *AliyundriveShare) getFiles(fileId string) ([]File, error) { SetHeader("x-share-token", d.ShareToken). SetHeader(CanaryHeaderKey, CanaryHeaderValue). SetResult(&resp).SetError(&e).SetBody(data). - Post("https://api.aliyundrive.com/adrive/v3/file/list") + Post("https://api.alipan.com/adrive/v3/file/list") if err != nil { return nil, err } diff --git a/drivers/all.go b/drivers/all.go index 921a467620d..3dc90424cbe 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -2,8 +2,11 @@ package drivers import ( _ "github.com/alist-org/alist/v3/drivers/115" + _ "github.com/alist-org/alist/v3/drivers/115_open" + _ "github.com/alist-org/alist/v3/drivers/115_share" _ "github.com/alist-org/alist/v3/drivers/123" _ "github.com/alist-org/alist/v3/drivers/123_link" + _ "github.com/alist-org/alist/v3/drivers/123_open" _ "github.com/alist-org/alist/v3/drivers/123_share" _ "github.com/alist-org/alist/v3/drivers/139" _ "github.com/alist-org/alist/v3/drivers/189" @@ -14,41 +17,77 @@ import ( _ "github.com/alist-org/alist/v3/drivers/aliyundrive" _ "github.com/alist-org/alist/v3/drivers/aliyundrive_open" _ "github.com/alist-org/alist/v3/drivers/aliyundrive_share" + _ "github.com/alist-org/alist/v3/drivers/azure_blob" _ "github.com/alist-org/alist/v3/drivers/baidu_netdisk" _ "github.com/alist-org/alist/v3/drivers/baidu_photo" _ "github.com/alist-org/alist/v3/drivers/baidu_share" + _ "github.com/alist-org/alist/v3/drivers/baidu_youth" + _ "github.com/alist-org/alist/v3/drivers/bitqiu" + _ "github.com/alist-org/alist/v3/drivers/chaoxing" + _ "github.com/alist-org/alist/v3/drivers/chunker" _ "github.com/alist-org/alist/v3/drivers/cloudreve" + _ "github.com/alist-org/alist/v3/drivers/cloudreve_v4" _ "github.com/alist-org/alist/v3/drivers/crypt" + _ "github.com/alist-org/alist/v3/drivers/darkibox" + _ "github.com/alist-org/alist/v3/drivers/doubao" + _ "github.com/alist-org/alist/v3/drivers/doubao_new" + _ "github.com/alist-org/alist/v3/drivers/doubao_share" _ "github.com/alist-org/alist/v3/drivers/dropbox" + _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" + _ "github.com/alist-org/alist/v3/drivers/ftps" + _ "github.com/alist-org/alist/v3/drivers/gitee" + _ "github.com/alist-org/alist/v3/drivers/github" + _ "github.com/alist-org/alist/v3/drivers/github_releases" + _ "github.com/alist-org/alist/v3/drivers/gofile" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" + _ "github.com/alist-org/alist/v3/drivers/guangyapan" + _ "github.com/alist-org/alist/v3/drivers/halalcloud" + _ "github.com/alist-org/alist/v3/drivers/ilanzou" _ "github.com/alist-org/alist/v3/drivers/ipfs_api" + _ "github.com/alist-org/alist/v3/drivers/kodbox" _ "github.com/alist-org/alist/v3/drivers/lanzou" + _ "github.com/alist-org/alist/v3/drivers/lenovonas_share" _ "github.com/alist-org/alist/v3/drivers/local" + _ "github.com/alist-org/alist/v3/drivers/mediafire" _ "github.com/alist-org/alist/v3/drivers/mediatrack" _ "github.com/alist-org/alist/v3/drivers/mega" + _ "github.com/alist-org/alist/v3/drivers/misskey" _ "github.com/alist-org/alist/v3/drivers/mopan" + _ "github.com/alist-org/alist/v3/drivers/netease_music" _ "github.com/alist-org/alist/v3/drivers/onedrive" _ "github.com/alist-org/alist/v3/drivers/onedrive_app" + _ "github.com/alist-org/alist/v3/drivers/onedrive_sharelink" + _ "github.com/alist-org/alist/v3/drivers/pcloud" _ "github.com/alist-org/alist/v3/drivers/pikpak" _ "github.com/alist-org/alist/v3/drivers/pikpak_share" + _ "github.com/alist-org/alist/v3/drivers/proton_drive" _ "github.com/alist-org/alist/v3/drivers/quark_uc" + _ "github.com/alist-org/alist/v3/drivers/quark_uc_tv" + _ "github.com/alist-org/alist/v3/drivers/quqi" _ "github.com/alist-org/alist/v3/drivers/s3" _ "github.com/alist-org/alist/v3/drivers/seafile" _ "github.com/alist-org/alist/v3/drivers/sftp" _ "github.com/alist-org/alist/v3/drivers/smb" + _ "github.com/alist-org/alist/v3/drivers/streamtape" + _ "github.com/alist-org/alist/v3/drivers/strm" _ "github.com/alist-org/alist/v3/drivers/teambition" _ "github.com/alist-org/alist/v3/drivers/terabox" _ "github.com/alist-org/alist/v3/drivers/thunder" + _ "github.com/alist-org/alist/v3/drivers/thunder_browser" + _ "github.com/alist-org/alist/v3/drivers/thunderx" _ "github.com/alist-org/alist/v3/drivers/trainbit" _ "github.com/alist-org/alist/v3/drivers/url_tree" _ "github.com/alist-org/alist/v3/drivers/uss" _ "github.com/alist-org/alist/v3/drivers/virtual" + _ "github.com/alist-org/alist/v3/drivers/vtencent" _ "github.com/alist-org/alist/v3/drivers/webdav" _ "github.com/alist-org/alist/v3/drivers/weiyun" _ "github.com/alist-org/alist/v3/drivers/wopan" + _ "github.com/alist-org/alist/v3/drivers/wukong" _ "github.com/alist-org/alist/v3/drivers/yandex_disk" + _ "github.com/alist-org/alist/v3/drivers/yunpan360" ) // All do nothing,just for import diff --git a/drivers/azure_blob/driver.go b/drivers/azure_blob/driver.go new file mode 100644 index 00000000000..6836533a070 --- /dev/null +++ b/drivers/azure_blob/driver.go @@ -0,0 +1,313 @@ +package azure_blob + +import ( + "context" + "fmt" + "io" + "path" + "regexp" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) +// Azure Blob Storage based on the blob APIs +// Link: https://learn.microsoft.com/rest/api/storageservices/blob-service-rest-api +type AzureBlob struct { + model.Storage + Addition + client *azblob.Client + containerClient *container.Client + config driver.Config +} + +// Config returns the driver configuration. +func (d *AzureBlob) Config() driver.Config { + return d.config +} + +// GetAddition returns additional settings specific to Azure Blob Storage. +func (d *AzureBlob) GetAddition() driver.Additional { + return &d.Addition +} + +// Init initializes the Azure Blob Storage client using shared key authentication. +func (d *AzureBlob) Init(ctx context.Context) error { + // Validate the endpoint URL + accountName := extractAccountName(d.Addition.Endpoint) + if !regexp.MustCompile(`^[a-z0-9]+$`).MatchString(accountName) { + return fmt.Errorf("invalid storage account name: must be chars of lowercase letters or numbers only") + } + + credential, err := azblob.NewSharedKeyCredential(accountName, d.Addition.AccessKey) + if err != nil { + return fmt.Errorf("failed to create credential: %w", err) + } + + // Check if Endpoint is just account name + endpoint := d.Addition.Endpoint + if accountName == endpoint { + endpoint = fmt.Sprintf("https://%s.blob.core.windows.net/", accountName) + } + // Initialize Azure Blob client with retry policy + client, err := azblob.NewClientWithSharedKeyCredential(endpoint, credential, + &azblob.ClientOptions{ClientOptions: azcore.ClientOptions{ + Retry: policy.RetryOptions{ + MaxRetries: MaxRetries, + RetryDelay: RetryDelay, + }, + }}) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + d.client = client + + // Ensure container exists or create it + containerName := strings.Trim(d.Addition.ContainerName, "/ \\") + if containerName == "" { + return fmt.Errorf("container name cannot be empty") + } + return d.createContainerIfNotExists(ctx, containerName) +} + +// Drop releases resources associated with the Azure Blob client. +func (d *AzureBlob) Drop(ctx context.Context) error { + d.client = nil + return nil +} + +// List retrieves blobs and directories under the specified path. +func (d *AzureBlob) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + prefix := ensureTrailingSlash(dir.GetPath()) + + pager := d.containerClient.NewListBlobsHierarchyPager("/", &container.ListBlobsHierarchyOptions{ + Prefix: &prefix, + }) + + var objs []model.Obj + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list blobs: %w", err) + } + + // Process directories + for _, blobPrefix := range page.Segment.BlobPrefixes { + objs = append(objs, &model.Object{ + Name: path.Base(strings.TrimSuffix(*blobPrefix.Name, "/")), + Path: *blobPrefix.Name, + Modified: *blobPrefix.Properties.LastModified, + Ctime: *blobPrefix.Properties.CreationTime, + IsFolder: true, + }) + } + + // Process files + for _, blob := range page.Segment.BlobItems { + if strings.HasSuffix(*blob.Name, "/") { + continue + } + objs = append(objs, &model.Object{ + Name: path.Base(*blob.Name), + Path: *blob.Name, + Size: *blob.Properties.ContentLength, + Modified: *blob.Properties.LastModified, + Ctime: *blob.Properties.CreationTime, + IsFolder: false, + }) + } + } + return objs, nil +} + +// Link generates a temporary SAS URL for accessing a blob. +func (d *AzureBlob) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + blobClient := d.containerClient.NewBlobClient(file.GetPath()) + expireDuration := time.Hour * time.Duration(d.SignURLExpire) + + sasURL, err := blobClient.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expireDuration), nil) + if err != nil { + return nil, fmt.Errorf("failed to generate SAS URL: %w", err) + } + return &model.Link{URL: sasURL}, nil +} + +// MakeDir creates a virtual directory by uploading an empty blob as a marker. +func (d *AzureBlob) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + dirPath := path.Join(parentDir.GetPath(), dirName) + if err := d.mkDir(ctx, dirPath); err != nil { + return nil, fmt.Errorf("failed to create directory marker: %w", err) + } + + return &model.Object{ + Path: dirPath, + Name: dirName, + IsFolder: true, + }, nil +} + +// Move relocates an object (file or directory) to a new directory. +func (d *AzureBlob) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcPath := srcObj.GetPath() + dstPath := path.Join(dstDir.GetPath(), srcObj.GetName()) + + if err := d.moveOrRename(ctx, srcPath, dstPath, srcObj.IsDir(), srcObj.GetSize()); err != nil { + return nil, fmt.Errorf("move operation failed: %w", err) + } + + return &model.Object{ + Path: dstPath, + Name: srcObj.GetName(), + Modified: time.Now(), + IsFolder: srcObj.IsDir(), + Size: srcObj.GetSize(), + }, nil +} + +// Rename changes the name of an existing object. +func (d *AzureBlob) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + srcPath := srcObj.GetPath() + dstPath := path.Join(path.Dir(srcPath), newName) + + if err := d.moveOrRename(ctx, srcPath, dstPath, srcObj.IsDir(), srcObj.GetSize()); err != nil { + return nil, fmt.Errorf("rename operation failed: %w", err) + } + + return &model.Object{ + Path: dstPath, + Name: newName, + Modified: time.Now(), + IsFolder: srcObj.IsDir(), + Size: srcObj.GetSize(), + }, nil +} + +// Copy duplicates an object (file or directory) to a specified destination directory. +func (d *AzureBlob) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + dstPath := path.Join(dstDir.GetPath(), srcObj.GetName()) + + // Handle directory copying using flat listing + if srcObj.IsDir() { + srcPrefix := srcObj.GetPath() + srcPrefix = ensureTrailingSlash(srcPrefix) + + // Get all blobs under the source directory + blobs, err := d.flattenListBlobs(ctx, srcPrefix) + if err != nil { + return nil, fmt.Errorf("failed to list source directory contents: %w", err) + } + + // Process each blob - copy to destination + for _, blob := range blobs { + // Skip the directory marker itself + if *blob.Name == srcPrefix { + continue + } + + // Calculate relative path from source + relPath := strings.TrimPrefix(*blob.Name, srcPrefix) + itemDstPath := path.Join(dstPath, relPath) + + if strings.HasSuffix(itemDstPath, "/") || (blob.Metadata["hdi_isfolder"] != nil && *blob.Metadata["hdi_isfolder"] == "true") { + // Create directory marker at destination + err := d.mkDir(ctx, itemDstPath) + if err != nil { + return nil, fmt.Errorf("failed to create directory marker [%s]: %w", itemDstPath, err) + } + } else { + // Copy the blob + if err := d.copyFile(ctx, *blob.Name, itemDstPath); err != nil { + return nil, fmt.Errorf("failed to copy %s: %w", *blob.Name, err) + } + } + + } + + // Create directory marker at destination if needed + if len(blobs) == 0 { + err := d.mkDir(ctx, dstPath) + if err != nil { + return nil, fmt.Errorf("failed to create directory [%s]: %w", dstPath, err) + } + } + + return &model.Object{ + Path: dstPath, + Name: srcObj.GetName(), + Modified: time.Now(), + IsFolder: true, + }, nil + } + + // Copy a single file + if err := d.copyFile(ctx, srcObj.GetPath(), dstPath); err != nil { + return nil, fmt.Errorf("failed to copy blob: %w", err) + } + return &model.Object{ + Path: dstPath, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: time.Now(), + IsFolder: false, + }, nil +} + +// Remove deletes a specified blob or recursively deletes a directory and its contents. +func (d *AzureBlob) Remove(ctx context.Context, obj model.Obj) error { + path := obj.GetPath() + + // Handle recursive directory deletion + if obj.IsDir() { + return d.deleteFolder(ctx, path) + } + + // Delete single file + return d.deleteFile(ctx, path, false) +} + +// Put uploads a file stream to Azure Blob Storage with progress tracking. +func (d *AzureBlob) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + blobPath := path.Join(dstDir.GetPath(), stream.GetName()) + blobClient := d.containerClient.NewBlockBlobClient(blobPath) + + // Determine optimal upload options based on file size + options := optimizedUploadOptions(stream.GetSize()) + + // Track upload progress + progressTracker := &progressTracker{ + total: stream.GetSize(), + updateProgress: up, + } + + // Wrap stream to handle context cancellation and progress tracking + limitedStream := driver.NewLimitedUploadStream(ctx, io.TeeReader(stream, progressTracker)) + + // Upload the stream to Azure Blob Storage + _, err := blobClient.UploadStream(ctx, limitedStream, options) + if err != nil { + return nil, fmt.Errorf("failed to upload file: %w", err) + } + + return &model.Object{ + Path: blobPath, + Name: stream.GetName(), + Size: stream.GetSize(), + Modified: time.Now(), + IsFolder: false, + }, nil +} + +// The following methods related to archive handling are not implemented yet. +// func (d *AzureBlob) GetArchiveMeta(...) {...} +// func (d *AzureBlob) ListArchive(...) {...} +// func (d *AzureBlob) Extract(...) {...} +// func (d *AzureBlob) ArchiveDecompress(...) {...} + +// Ensure AzureBlob implements the driver.Driver interface. +var _ driver.Driver = (*AzureBlob)(nil) diff --git a/drivers/azure_blob/meta.go b/drivers/azure_blob/meta.go new file mode 100644 index 00000000000..b1e021b86d0 --- /dev/null +++ b/drivers/azure_blob/meta.go @@ -0,0 +1,32 @@ +package azure_blob + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + Endpoint string `json:"endpoint" required:"true" default:"https://.blob.core.windows.net/" help:"e.g. https://accountname.blob.core.windows.net/. The full endpoint URL for Azure Storage, including the unique storage account name (3 ~ 24 numbers and lowercase letters only)."` + AccessKey string `json:"access_key" required:"true" help:"The access key for Azure Storage, used for authentication. https://learn.microsoft.com/azure/storage/common/storage-account-keys-manage"` + ContainerName string `json:"container_name" required:"true" help:"The name of the container in Azure Storage (created in the Azure portal). https://learn.microsoft.com/azure/storage/blobs/blob-containers-portal"` + SignURLExpire int `json:"sign_url_expire" type:"number" default:"4" help:"The expiration time for SAS URLs, in hours."` +} + +// implement GetRootId interface +func (r Addition) GetRootId() string { + return r.ContainerName +} + +var config = driver.Config{ + Name: "Azure Blob Storage", + LocalSort: true, + CheckStatus: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &AzureBlob{ + config: config, + } + }) +} diff --git a/drivers/azure_blob/types.go b/drivers/azure_blob/types.go new file mode 100644 index 00000000000..01323e51273 --- /dev/null +++ b/drivers/azure_blob/types.go @@ -0,0 +1,20 @@ +package azure_blob + +import "github.com/alist-org/alist/v3/internal/driver" + +// progressTracker is used to track upload progress +type progressTracker struct { + total int64 + current int64 + updateProgress driver.UpdateProgress +} + +// Write implements io.Writer to track progress +func (pt *progressTracker) Write(p []byte) (n int, err error) { + n = len(p) + pt.current += int64(n) + if pt.updateProgress != nil && pt.total > 0 { + pt.updateProgress(float64(pt.current) * 100 / float64(pt.total)) + } + return n, nil +} diff --git a/drivers/azure_blob/util.go b/drivers/azure_blob/util.go new file mode 100644 index 00000000000..2adf3a0f343 --- /dev/null +++ b/drivers/azure_blob/util.go @@ -0,0 +1,401 @@ +package azure_blob + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "path" + "sort" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + log "github.com/sirupsen/logrus" +) + +const ( + // MaxRetries defines the maximum number of retry attempts for Azure operations + MaxRetries = 3 + // RetryDelay defines the base delay between retries + RetryDelay = 3 * time.Second + // MaxBatchSize defines the maximum number of operations in a single batch request + MaxBatchSize = 128 +) + +// extractAccountName 从 Azure 存储 Endpoint 中提取账户名 +func extractAccountName(endpoint string) string { + // 移除协议前缀 + endpoint = strings.TrimPrefix(endpoint, "https://") + endpoint = strings.TrimPrefix(endpoint, "http://") + + // 获取第一个点之前的部分(即账户名) + parts := strings.Split(endpoint, ".") + if len(parts) > 0 { + // to lower case + return strings.ToLower(parts[0]) + } + return "" +} + +// isNotFoundError checks if the error is a "not found" type error +func isNotFoundError(err error) bool { + var storageErr *azcore.ResponseError + if errors.As(err, &storageErr) { + return storageErr.StatusCode == 404 + } + // Fallback to string matching for backwards compatibility + return err != nil && strings.Contains(err.Error(), "BlobNotFound") +} + +// flattenListBlobs - Optimize blob listing to handle pagination better +func (d *AzureBlob) flattenListBlobs(ctx context.Context, prefix string) ([]container.BlobItem, error) { + // Standardize prefix format + prefix = ensureTrailingSlash(prefix) + + var blobItems []container.BlobItem + pager := d.containerClient.NewListBlobsFlatPager(&container.ListBlobsFlatOptions{ + Prefix: &prefix, + Include: container.ListBlobsInclude{ + Metadata: true, + }, + }) + + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("failed to list blobs: %w", err) + } + + for _, blob := range page.Segment.BlobItems { + blobItems = append(blobItems, *blob) + } + } + + return blobItems, nil +} + +// batchDeleteBlobs - Simplify batch deletion logic +func (d *AzureBlob) batchDeleteBlobs(ctx context.Context, blobPaths []string) error { + if len(blobPaths) == 0 { + return nil + } + + // Process in batches of MaxBatchSize + for i := 0; i < len(blobPaths); i += MaxBatchSize { + end := min(i+MaxBatchSize, len(blobPaths)) + currentBatch := blobPaths[i:end] + + // Create batch builder + batchBuilder, err := d.containerClient.NewBatchBuilder() + if err != nil { + return fmt.Errorf("failed to create batch builder: %w", err) + } + + // Add delete operations + for _, blobPath := range currentBatch { + if err := batchBuilder.Delete(blobPath, nil); err != nil { + return fmt.Errorf("failed to add delete operation for %s: %w", blobPath, err) + } + } + + // Submit batch + responses, err := d.containerClient.SubmitBatch(ctx, batchBuilder, nil) + if err != nil { + return fmt.Errorf("batch delete request failed: %w", err) + } + + // Check responses + for _, resp := range responses.Responses { + if resp.Error != nil && !isNotFoundError(resp.Error) { + // 获取 blob 名称以提供更好的错误信息 + blobName := "unknown" + if resp.BlobName != nil { + blobName = *resp.BlobName + } + return fmt.Errorf("failed to delete blob %s: %v", blobName, resp.Error) + } + } + } + + return nil +} + +// deleteFolder recursively deletes a directory and all its contents +func (d *AzureBlob) deleteFolder(ctx context.Context, prefix string) error { + // Ensure directory path ends with slash + prefix = ensureTrailingSlash(prefix) + + // Get all blobs under the directory using flattenListBlobs + globs, err := d.flattenListBlobs(ctx, prefix) + if err != nil { + return fmt.Errorf("failed to list blobs for deletion: %w", err) + } + + // If there are blobs in the directory, delete them + if len(globs) > 0 { + // 分离文件和目录标记 + var filePaths []string + var dirPaths []string + + for _, blob := range globs { + blobName := *blob.Name + if isDirectory(blob) { + // remove trailing slash for directory names + dirPaths = append(dirPaths, strings.TrimSuffix(blobName, "/")) + } else { + filePaths = append(filePaths, blobName) + } + } + + // 先删除文件,再删除目录 + if len(filePaths) > 0 { + if err := d.batchDeleteBlobs(ctx, filePaths); err != nil { + return err + } + } + if len(dirPaths) > 0 { + // 按路径深度分组 + depthMap := make(map[int][]string) + for _, dir := range dirPaths { + depth := strings.Count(dir, "/") // 计算目录深度 + depthMap[depth] = append(depthMap[depth], dir) + } + + // 按深度从大到小排序 + var depths []int + for depth := range depthMap { + depths = append(depths, depth) + } + sort.Sort(sort.Reverse(sort.IntSlice(depths))) + + // 按深度逐层批量删除 + for _, depth := range depths { + batch := depthMap[depth] + if err := d.batchDeleteBlobs(ctx, batch); err != nil { + return err + } + } + } + } + + // 最后删除目录标记本身 + return d.deleteEmptyDirectory(ctx, prefix) +} + +// deleteFile deletes a single file or blob with better error handling +func (d *AzureBlob) deleteFile(ctx context.Context, path string, isDir bool) error { + blobClient := d.containerClient.NewBlobClient(path) + _, err := blobClient.Delete(ctx, nil) + if err != nil && !(isDir && isNotFoundError(err)) { + return err + } + return nil +} + +// copyFile copies a single blob from source path to destination path +func (d *AzureBlob) copyFile(ctx context.Context, srcPath, dstPath string) error { + srcBlob := d.containerClient.NewBlobClient(srcPath) + dstBlob := d.containerClient.NewBlobClient(dstPath) + + // Use configured expiration time for SAS URL + expireDuration := time.Hour * time.Duration(d.SignURLExpire) + srcURL, err := srcBlob.GetSASURL(sas.BlobPermissions{Read: true}, time.Now().Add(expireDuration), nil) + if err != nil { + return fmt.Errorf("failed to generate source SAS URL: %w", err) + } + + _, err = dstBlob.StartCopyFromURL(ctx, srcURL, nil) + return err + +} + +// createContainerIfNotExists - Create container if not exists +// Clean up commented code +func (d *AzureBlob) createContainerIfNotExists(ctx context.Context, containerName string) error { + serviceClient := d.client.ServiceClient() + containerClient := serviceClient.NewContainerClient(containerName) + + var options = service.CreateContainerOptions{} + _, err := containerClient.Create(ctx, &options) + if err != nil { + var responseErr *azcore.ResponseError + if errors.As(err, &responseErr) && responseErr.ErrorCode != "ContainerAlreadyExists" { + return fmt.Errorf("failed to create or access container [%s]: %w", containerName, err) + } + } + + d.containerClient = containerClient + return nil +} + +// mkDir creates a virtual directory marker by uploading an empty blob with metadata. +func (d *AzureBlob) mkDir(ctx context.Context, fullDirName string) error { + dirPath := ensureTrailingSlash(fullDirName) + blobClient := d.containerClient.NewBlockBlobClient(dirPath) + + // Upload an empty blob with metadata indicating it's a directory + _, err := blobClient.Upload(ctx, struct { + *bytes.Reader + io.Closer + }{ + Reader: bytes.NewReader([]byte{}), + Closer: io.NopCloser(nil), + }, &blockblob.UploadOptions{ + Metadata: map[string]*string{ + "hdi_isfolder": to.Ptr("true"), + }, + }) + return err +} + +// ensureTrailingSlash ensures the provided path ends with a trailing slash. +func ensureTrailingSlash(path string) string { + if !strings.HasSuffix(path, "/") { + return path + "/" + } + return path +} + +// moveOrRename moves or renames blobs or directories from source to destination. +func (d *AzureBlob) moveOrRename(ctx context.Context, srcPath, dstPath string, isDir bool, srcSize int64) error { + if isDir { + // Normalize paths for directory operations + srcPath = ensureTrailingSlash(srcPath) + dstPath = ensureTrailingSlash(dstPath) + + // List all blobs under the source directory + blobs, err := d.flattenListBlobs(ctx, srcPath) + if err != nil { + return fmt.Errorf("failed to list blobs: %w", err) + } + + // Iterate and copy each blob to the destination + for _, item := range blobs { + srcBlobName := *item.Name + relPath := strings.TrimPrefix(srcBlobName, srcPath) + itemDstPath := path.Join(dstPath, relPath) + + if isDirectory(item) { + // Create directory marker at destination + if err := d.mkDir(ctx, itemDstPath); err != nil { + return fmt.Errorf("failed to create directory marker [%s]: %w", itemDstPath, err) + } + } else { + // Copy file blob to destination + if err := d.copyFile(ctx, srcBlobName, itemDstPath); err != nil { + return fmt.Errorf("failed to copy blob [%s]: %w", srcBlobName, err) + } + } + } + + // Handle empty directories by creating a marker at destination + if len(blobs) == 0 { + if err := d.mkDir(ctx, dstPath); err != nil { + return fmt.Errorf("failed to create directory [%s]: %w", dstPath, err) + } + } + + // Delete source directory and its contents + if err := d.deleteFolder(ctx, srcPath); err != nil { + log.Warnf("failed to delete source directory [%s]: %v\n, and try again", srcPath, err) + // Retry deletion once more and ignore the result + if err := d.deleteFolder(ctx, srcPath); err != nil { + log.Errorf("Retry deletion of source directory [%s] failed: %v", srcPath, err) + } + } + + return nil + } + + // Single file move or rename operation + if err := d.copyFile(ctx, srcPath, dstPath); err != nil { + return fmt.Errorf("failed to copy file: %w", err) + } + + // Delete source file after successful copy + if err := d.deleteFile(ctx, srcPath, false); err != nil { + log.Errorf("Error deleting source file [%s]: %v", srcPath, err) + } + return nil +} + +// optimizedUploadOptions returns the optimal upload options based on file size +func optimizedUploadOptions(fileSize int64) *azblob.UploadStreamOptions { + options := &azblob.UploadStreamOptions{ + BlockSize: 4 * 1024 * 1024, // 4MB block size + Concurrency: 4, // Default concurrency + } + + // For large files, increase block size and concurrency + if fileSize > 256*1024*1024 { // For files larger than 256MB + options.BlockSize = 8 * 1024 * 1024 // 8MB blocks + options.Concurrency = 8 // More concurrent uploads + } + + // For very large files (>1GB) + if fileSize > 1024*1024*1024 { + options.BlockSize = 16 * 1024 * 1024 // 16MB blocks + options.Concurrency = 16 // Higher concurrency + } + + return options +} + +// isDirectory determines if a blob represents a directory +// Checks multiple indicators: path suffix, metadata, and content type +func isDirectory(blob container.BlobItem) bool { + // Check path suffix + if strings.HasSuffix(*blob.Name, "/") { + return true + } + + // Check metadata for directory marker + if blob.Metadata != nil { + if val, ok := blob.Metadata["hdi_isfolder"]; ok && val != nil && *val == "true" { + return true + } + // Azure Storage Explorer and other tools may use different metadata keys + if val, ok := blob.Metadata["is_directory"]; ok && val != nil && strings.ToLower(*val) == "true" { + return true + } + } + + // Check content type (some tools mark directories with specific content types) + if blob.Properties != nil && blob.Properties.ContentType != nil { + contentType := strings.ToLower(*blob.Properties.ContentType) + if blob.Properties.ContentLength != nil && *blob.Properties.ContentLength == 0 && (contentType == "application/directory" || contentType == "directory") { + return true + } + } + + return false +} + +// deleteEmptyDirectory deletes a directory only if it's empty +func (d *AzureBlob) deleteEmptyDirectory(ctx context.Context, dirPath string) error { + // Directory is empty, delete the directory marker + blobClient := d.containerClient.NewBlobClient(strings.TrimSuffix(dirPath, "/")) + _, err := blobClient.Delete(ctx, nil) + + // Also try deleting with trailing slash (for different directory marker formats) + if err != nil && isNotFoundError(err) { + blobClient = d.containerClient.NewBlobClient(dirPath) + _, err = blobClient.Delete(ctx, nil) + } + + // Ignore not found errors + if err != nil && isNotFoundError(err) { + log.Infof("Directory [%s] not found during deletion: %v", dirPath, err) + return nil + } + + return err +} diff --git a/drivers/baidu_netdisk/driver.go b/drivers/baidu_netdisk/driver.go index 654c8b01bc7..64539dc7d89 100644 --- a/drivers/baidu_netdisk/driver.go +++ b/drivers/baidu_netdisk/driver.go @@ -5,20 +5,26 @@ import ( "crypto/md5" "encoding/hex" "errors" + "fmt" "io" - "math" "net/url" + "os" stdpath "path" "strconv" + "strings" + "sync" "time" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/singleflight" "github.com/alist-org/alist/v3/pkg/utils" "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" ) @@ -27,9 +33,16 @@ type BaiduNetdisk struct { Addition uploadThread int + vipType int // 会员类型,0普通用户(4G/4M)、1普通会员(10G/16M)、2超级会员(20G/32M) + + upClient *resty.Client // 上传文件使用的http客户端 + uploadUrlG singleflight.Group[string] + uploadUrlMu sync.RWMutex + uploadUrl string // 上传域名 + uploadUrlUpdateTime time.Time // 上传域名上次更新时间 } -const DefaultSliceSize int64 = 4 * utils.MB +var ErrUploadIDExpired = errors.New("uploadid expired") func (d *BaiduNetdisk) Config() driver.Config { return config @@ -40,20 +53,31 @@ func (d *BaiduNetdisk) GetAddition() driver.Additional { } func (d *BaiduNetdisk) Init(ctx context.Context) error { + d.upClient = base.NewRestyClient(). + SetTimeout(UPLOAD_TIMEOUT). + SetRetryCount(UPLOAD_RETRY_COUNT). + SetRetryWaitTime(UPLOAD_RETRY_WAIT_TIME). + SetRetryMaxWaitTime(UPLOAD_RETRY_MAX_WAIT_TIME) d.uploadThread, _ = strconv.Atoi(d.UploadThread) - if d.uploadThread < 1 || d.uploadThread > 32 { - d.uploadThread, d.UploadThread = 3, "3" + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" } if _, err := url.Parse(d.UploadAPI); d.UploadAPI == "" || err != nil { - d.UploadAPI = "https://d.pcs.baidu.com" + d.UploadAPI = UPLOAD_FALLBACK_API } res, err := d.get("/xpan/nas", map[string]string{ "method": "uinfo", }, nil) - log.Debugf("[baidu] get uinfo: %s", string(res)) - return err + log.Debugf("[baidu_netdisk] get uinfo: %s", string(res)) + if err != nil { + return err + } + d.vipType = utils.Json.Get(res, "vip_type").ToInt() + return nil } func (d *BaiduNetdisk) Drop(ctx context.Context) error { @@ -73,6 +97,8 @@ func (d *BaiduNetdisk) List(ctx context.Context, dir model.Obj, args model.ListA func (d *BaiduNetdisk) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { if d.DownloadAPI == "crack" { return d.linkCrack(file, args) + } else if d.DownloadAPI == "crack_video" { + return d.linkCrackVideo(file, args) } return d.linkOfficial(file, args) } @@ -162,36 +188,68 @@ func (d *BaiduNetdisk) PutRapid(ctx context.Context, dstDir model.Obj, stream mo if err != nil { return nil, err } + // 修复时间,具体原因见 Put 方法注释的 **注意** + newFile.Ctime = stream.CreateTime().Unix() + newFile.Mtime = stream.ModTime().Unix() return fileToObj(newFile), nil } +// Put +// +// **注意**: 截至 2024/04/20 百度云盘 api 接口返回的时间永远是当前时间,而不是文件时间。 +// 而实际上云盘存储的时间是文件时间,所以此处需要覆盖时间,保证缓存与云盘的数据一致 func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // 百度网盘不允许上传空文件 + if stream.GetSize() < 1 { + return nil, ErrBaiduEmptyFilesNotAllowed + } + // rapid upload if newObj, err := d.PutRapid(ctx, dstDir, stream); err == nil { return newObj, nil } - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return nil, err + var ( + cache = stream.GetFile() + tmpF *os.File + err error + ) + if _, ok := cache.(io.ReaderAt); !ok { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF } streamSize := stream.GetSize() - count := int(math.Max(math.Ceil(float64(streamSize)/float64(DefaultSliceSize)), 1)) - lastBlockSize := streamSize % DefaultSliceSize - if streamSize > 0 && lastBlockSize == 0 { - lastBlockSize = DefaultSliceSize + sliceSize := d.getSliceSize(streamSize) + count := int(streamSize / sliceSize) + lastBlockSize := streamSize % sliceSize + if lastBlockSize > 0 { + count++ + } else { + lastBlockSize = sliceSize } //cal md5 for first 256k data - const SliceSize int64 = 256 * 1024 + const SliceSize int64 = 256 * utils.KB // cal md5 blockList := make([]string, 0, count) - byteSize := DefaultSliceSize + byteSize := sliceSize fileMd5H := md5.New() sliceMd5H := md5.New() sliceMd5H2 := md5.New() slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) + writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} + if tmpF != nil { + writers = append(writers, tmpF) + } + written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { @@ -200,13 +258,23 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if i == count { byteSize = lastBlockSize } - _, err := io.CopyN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize) + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) + written += n if err != nil && err != io.EOF { return nil, err } blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) sliceMd5H.Reset() } + if tmpF != nil { + if written != streamSize { + return nil, errs.NewErr(err, "CreateTempFile failed, size mismatch: %d != %d ", written, streamSize) + } + _, err = tmpF.Seek(0, io.SeekStart) + if err != nil { + return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") + } + } contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(blockList) @@ -214,76 +282,97 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F mtime := stream.ModTime().Unix() ctime := stream.CreateTime().Unix() - // step.1 预上传 - // 尝试获取之前的进度 + // step.1 尝试读取已保存进度 precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) if !ok { - params := map[string]string{ - "method": "precreate", - } - form := map[string]string{ - "path": path, - "size": strconv.FormatInt(streamSize, 10), - "isdir": "0", - "autoinit": "1", - "rtype": "3", - "block_list": blockListStr, - "content-md5": contentMd5, - "slice-md5": sliceMd5, - } - joinTime(form, ctime, mtime) - - log.Debugf("[baidu_netdisk] precreate data: %s", form) - _, err = d.postForm("/xpan/file", params, form, &precreateResp) + // 没有进度,走预上传 + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) if err != nil { return nil, err } - log.Debugf("%+v", precreateResp) if precreateResp.ReturnType == 2 { //rapid upload, since got md5 match from baidu server - if err != nil { - return nil, err - } + // 修复时间,具体原因见 Put 方法注释的 **注意** return fileToObj(precreateResp.File), nil } } + // step.2 上传分片 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, - retry.Attempts(3), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - for i, partseq := range precreateResp.BlockList { - if utils.IsCanceled(upCtx) { - break +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + // 获取上传域名 + uploadUrl := d.getUploadUrl(path, precreateResp.Uploadid) + // 并发上传 + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + cacheReaderAt, okReaderAt := cache.(io.ReaderAt) + if !okReaderAt { + return nil, fmt.Errorf("cache object must implement io.ReaderAt interface for upload operations") } - i, partseq, offset, byteSize := i, partseq, int64(partseq)*DefaultSliceSize, DefaultSliceSize - if partseq+1 == count { - byteSize = lastBlockSize - } - threadG.Go(func(ctx context.Context) error { - params := map[string]string{ - "method": "upload", - "access_token": d.AccessToken, - "type": "tmpfile", - "path": path, - "uploadid": precreateResp.Uploadid, - "partseq": strconv.Itoa(partseq), + totalParts := len(precreateResp.BlockList) + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue } - err := d.uploadSlice(ctx, params, stream.GetName(), io.NewSectionReader(tempFile, offset, byteSize)) - if err != nil { - return err + + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize } - up(int(threadG.Success()) * 100 / len(precreateResp.BlockList)) - precreateResp.BlockList[i] = -1 - return nil - }) - } - if err = threadG.Wait(); err != nil { - // 如果属于用户主动取消,则保存上传进度 + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "access_token": d.AccessToken, + "type": "tmpfile", + "path": path, + "uploadid": precreateResp.Uploadid, + "partseq": strconv.Itoa(partseq), + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + err := d.uploadSlice(ctx, uploadUrl, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)) + if err != nil { + return err + } + precreateResp.BlockList[i] = -1 + // 当前goroutine还没退出,+1才是真正成功的数量 + success := threadG.Success() + 1 + progress := float64(success) * 100 / float64(totalParts) + up(progress) + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + // 保存进度(所有错误都会保存) + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + if errors.Is(err, context.Canceled) { - precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + log.Warn("[baidu_netdisk] uploadid expired, will restart from scratch") + // 重新 precreate(所有分片都要重传) + newPre, err2 := d.precreate(ctx, path, streamSize, blockListStr, "", "", ctime, mtime) + if err2 != nil { + return nil, err2 + } + if newPre.ReturnType == 2 { + return fileToObj(newPre.File), nil + } + precreateResp = newPre + // 覆盖掉旧的进度 base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + continue uploadLoop } return nil, err } @@ -294,23 +383,70 @@ func (d *BaiduNetdisk) Put(ctx context.Context, dstDir model.Obj, stream model.F if err != nil { return nil, err } + // 修复时间,具体原因见 Put 方法注释的 **注意** + newFile.Ctime = ctime + newFile.Mtime = mtime + // 上传成功清理进度 + base.SaveUploadProgress(d, nil, d.AccessToken, contentMd5) return fileToObj(newFile), nil } -func (d *BaiduNetdisk) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { - res, err := base.RestyClient.R(). +// precreate 执行预上传操作,支持首次上传和 uploadid 过期重试 +func (d *BaiduNetdisk) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + params := map[string]string{"method": "precreate"} + form := map[string]string{ + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "isdir": "0", + "autoinit": "1", + "rtype": "3", + "block_list": blockListStr, + } + + // 只有在首次上传时才包含 content-md5 和 slice-md5 + if contentMd5 != "" && sliceMd5 != "" { + form["content-md5"] = contentMd5 + form["slice-md5"] = sliceMd5 + } + + joinTime(form, ctime, mtime) + + var precreateResp PrecreateResp + _, err := d.postForm("/xpan/file", params, form, &precreateResp) + if err != nil { + return nil, err + } + + // 修复时间,具体原因见 Put 方法注释的 **注意** + if precreateResp.ReturnType == 2 { + precreateResp.File.Ctime = ctime + precreateResp.File.Mtime = mtime + } + + return &precreateResp, nil +} + +func (d *BaiduNetdisk) uploadSlice(ctx context.Context, uploadUrl string, params map[string]string, fileName string, file io.Reader) error { + res, err := d.upClient.R(). SetContext(ctx). SetQueryParams(params). SetFileReader("file", fileName, file). - Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") + Post(uploadUrl + "/rest/2.0/pcs/superfile2") if err != nil { return err } log.Debugln(res.RawResponse.Status + res.String()) errCode := utils.Json.Get(res.Body(), "error_code").ToInt() errNo := utils.Json.Get(res.Body(), "errno").ToInt() + respStr := res.String() + lower := strings.ToLower(respStr) + if strings.Contains(lower, "uploadid") && + (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + if errCode != 0 || errNo != 0 { - return errs.NewErr(errs.StreamIncomplete, "error in uploading to baidu, will retry. response=%s", res.String()) + return errs.NewErr(errs.StreamIncomplete, "error uploading to baidu, response=%s", res.String()) } return nil } diff --git a/drivers/baidu_netdisk/meta.go b/drivers/baidu_netdisk/meta.go index b257986b9c1..b75650ef063 100644 --- a/drivers/baidu_netdisk/meta.go +++ b/drivers/baidu_netdisk/meta.go @@ -1,6 +1,8 @@ package baidu_netdisk import ( + "time" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/op" ) @@ -8,17 +10,30 @@ import ( type Addition struct { RefreshToken string `json:"refresh_token" required:"true"` driver.RootPath - OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` - DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` - ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` - ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` - CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` - AccessToken string - UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` - UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + DownloadAPI string `json:"download_api" type:"select" options:"official,crack,crack_video" default:"official"` + ClientID string `json:"client_id" required:"true" default:"hq9yQ9w9kR4YHj1kyYafLygVocobh7Sf"` + ClientSecret string `json:"client_secret" required:"true" default:"YH2VpZcFJHYNnV6vLfHQXDBhcE7ZChyE"` + CustomCrackUA string `json:"custom_crack_ua" required:"true" default:"netdisk"` + AccessToken string + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` + UseDynamicUploadAPI bool `json:"use_dynamic_upload_api" default:"true" help:"dynamically get upload api domain, when enabled, the 'Upload API' setting will be used as a fallback if failed to get"` + CustomUploadPartSize int64 `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"` + LowBandwithUploadMode bool `json:"low_bandwith_upload_mode" default:"false"` + OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` } +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" // 备用上传地址 + UPLOAD_URL_EXPIRE_TIME = time.Minute * 60 // 上传地址有效期(分钟) + UPLOAD_TIMEOUT = time.Minute * 30 // 上传请求超时时间 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second * 1 + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 +) + var config = driver.Config{ Name: "BaiduNetdisk", DefaultRoot: "/", diff --git a/drivers/baidu_netdisk/types.go b/drivers/baidu_netdisk/types.go index cbec0bcfcd6..a158956d09b 100644 --- a/drivers/baidu_netdisk/types.go +++ b/drivers/baidu_netdisk/types.go @@ -1,11 +1,17 @@ package baidu_netdisk import ( + "errors" "path" "strconv" "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var ( + ErrBaiduEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu netdisk") ) type TokenErrResp struct { @@ -16,7 +22,7 @@ type TokenErrResp struct { type File struct { //TkbindId int `json:"tkbind_id"` //OwnerType int `json:"owner_type"` - //Category int `json:"category"` + Category int `json:"category"` //RealCategory string `json:"real_category"` FsId int64 `json:"fs_id"` //OperId int `json:"oper_id"` @@ -55,11 +61,11 @@ func fileToObj(f File) *model.ObjThumb { if f.ServerFilename == "" { f.ServerFilename = path.Base(f.Path) } - if f.LocalCtime == 0 { - f.LocalCtime = f.Ctime + if f.ServerCtime == 0 { + f.ServerCtime = f.Ctime } - if f.LocalMtime == 0 { - f.LocalMtime = f.Mtime + if f.ServerMtime == 0 { + f.ServerMtime = f.Mtime } return &model.ObjThumb{ Object: model.Object{ @@ -67,12 +73,12 @@ func fileToObj(f File) *model.ObjThumb { Path: f.Path, Name: f.ServerFilename, Size: f.Size, - Modified: time.Unix(f.LocalMtime, 0), - Ctime: time.Unix(f.LocalCtime, 0), + Modified: time.Unix(f.ServerMtime, 0), + Ctime: time.Unix(f.ServerCtime, 0), IsFolder: f.Isdir == 1, // 直接获取的MD5是错误的 - // HashInfo: utils.NewHashInfo(utils.MD5, f.Md5), + HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), }, Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, } @@ -188,3 +194,27 @@ type PrecreateResp struct { // return_type=2 File File `json:"info"` } + +type UploadServerResp struct { + BakServer []any `json:"bak_server"` + BakServers []struct { + Server string `json:"server"` + } `json:"bak_servers"` + ClientIP string `json:"client_ip"` + ErrorCode int `json:"error_code"` + ErrorMsg string `json:"error_msg"` + Expire int `json:"expire"` + Host string `json:"host"` + Newno string `json:"newno"` + QuicServer []any `json:"quic_server"` + QuicServers []struct { + Server string `json:"server"` + } `json:"quic_servers"` + RequestID int64 `json:"request_id"` + Server []any `json:"server"` + ServerTime int `json:"server_time"` + Servers []struct { + Server string `json:"server"` + } `json:"servers"` + Sl int `json:"sl"` +} diff --git a/drivers/baidu_netdisk/util.go b/drivers/baidu_netdisk/util.go index d972eb83fa7..c5a7334315d 100644 --- a/drivers/baidu_netdisk/util.go +++ b/drivers/baidu_netdisk/util.go @@ -1,11 +1,14 @@ package baidu_netdisk import ( + "encoding/hex" "errors" "fmt" "net/http" "strconv" + "strings" "time" + "unicode" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/errs" @@ -70,12 +73,18 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall errno := utils.Json.Get(res.Body(), "errno").ToInt() if errno != 0 { if utils.SliceContains([]int{111, -6}, errno) { - log.Info("refreshing baidu_netdisk token.") + log.Info("[baidu_netdisk] refreshing baidu_netdisk token.") err2 := d.refreshToken() if err2 != nil { return retry.Unrecoverable(err2) } } + + if 31023 == errno && d.DownloadAPI == "crack_video" { + result = res.Body() + return nil + } + return fmt.Errorf("req: [%s] ,errno: %d, refer to https://pan.baidu.com/union/doc/", furl, errno) } result = res.Body() @@ -128,12 +137,21 @@ func (d *BaiduNetdisk) getFiles(dir string) ([]File, error) { if len(resp.List) == 0 { break } - res = append(res, resp.List...) + + if d.OnlyListVideoFile { + for _, file := range resp.List { + if file.Isdir == 1 || file.Category == 1 { + res = append(res, file) + } + } + } else { + res = append(res, resp.List...) + } } return res, nil } -func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model.Link, error) { +func (d *BaiduNetdisk) linkOfficial(file model.Obj, _ model.LinkArgs) (*model.Link, error) { var resp DownloadResp params := map[string]string{ "method": "filemetas", @@ -153,8 +171,6 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model u = res.Header().Get("location") //} - updateObjMd5(file, "pan.baidu.com", u) - return &model.Link{ URL: u, Header: http.Header{ @@ -163,7 +179,7 @@ func (d *BaiduNetdisk) linkOfficial(file model.Obj, args model.LinkArgs) (*model }, nil } -func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Link, error) { +func (d *BaiduNetdisk) linkCrack(file model.Obj, _ model.LinkArgs) (*model.Link, error) { var resp DownloadResp2 param := map[string]string{ "target": fmt.Sprintf("[\"%s\"]", file.GetPath()), @@ -178,8 +194,6 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li return nil, err } - updateObjMd5(file, d.CustomCrackUA, resp.Info[0].Dlink) - return &model.Link{ URL: resp.Info[0].Dlink, Header: http.Header{ @@ -188,6 +202,34 @@ func (d *BaiduNetdisk) linkCrack(file model.Obj, args model.LinkArgs) (*model.Li }, nil } +func (d *BaiduNetdisk) linkCrackVideo(file model.Obj, _ model.LinkArgs) (*model.Link, error) { + param := map[string]string{ + "type": "VideoURL", + "path": fmt.Sprintf("%s", file.GetPath()), + "fs_id": file.GetID(), + "devuid": "0%1", + "clienttype": "1", + "channel": "android_15_25010PN30C_bd-netdisk_1523a", + "nom3u8": "1", + "dlink": "1", + "media": "1", + "origin": "dlna", + } + resp, err := d.request("https://pan.baidu.com/api/mediainfo", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(param) + }, nil) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: utils.Json.Get(resp, "info", "dlink").ToString(), + Header: http.Header{ + "User-Agent": []string{d.CustomCrackUA}, + }, + }, nil +} + func (d *BaiduNetdisk) manage(opera string, filelist any) ([]byte, error) { params := map[string]string{ "method": "filemanager", @@ -229,17 +271,151 @@ func joinTime(form map[string]string, ctime, mtime int64) { form["local_ctime"] = strconv.FormatInt(ctime, 10) } -func updateObjMd5(obj model.Obj, userAgent, u string) { - object := model.GetRawObject(obj) - if object != nil { - req, _ := http.NewRequest(http.MethodHead, u, nil) - req.Header.Add("User-Agent", userAgent) - resp, _ := base.HttpClient.Do(req) - if resp != nil { - contentMd5 := resp.Header.Get("Content-Md5") - object.HashInfo = utils.NewHashInfo(utils.MD5, contentMd5) +const ( + DefaultSliceSize int64 = 4 * utils.MB + VipSliceSize int64 = 16 * utils.MB + SVipSliceSize int64 = 32 * utils.MB + + MaxSliceNum = 2048 // 文档写的是 1024/没写 ,但实际测试是 2048 + SliceStep int64 = 1 * utils.MB +) + +func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 { + // 非会员固定为 4MB + if d.vipType == 0 { + if d.CustomUploadPartSize != 0 { + log.Warnf("[baidu_netdisk] CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize") + } + if filesize > MaxSliceNum*DefaultSliceSize { + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) + } + + return DefaultSliceSize + } + + if d.CustomUploadPartSize != 0 { + if d.CustomUploadPartSize < DefaultSliceSize { + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize) + return DefaultSliceSize + } + + if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize { + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize) + return VipSliceSize + } + + if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize { + log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize) + return SVipSliceSize + } + + return d.CustomUploadPartSize + } + + maxSliceSize := DefaultSliceSize + + switch d.vipType { + case 1: + maxSliceSize = VipSliceSize + case 2: + maxSliceSize = SVipSliceSize + } + + // upload on low bandwidth + if d.LowBandwithUploadMode { + size := DefaultSliceSize + + for size <= maxSliceSize { + if filesize <= MaxSliceNum*size { + return size + } + + size += SliceStep + } + } + + if filesize > MaxSliceNum*maxSliceSize { + log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize) + } + + return maxSliceSize +} + +// getUploadUrl 从开放平台获取上传域名/地址,并发请求会被合并,结果会被缓存1h。 +// 如果获取失败,则返回 Upload API设置项。 +func (d *BaiduNetdisk) getUploadUrl(path, uploadId string) string { + if !d.UseDynamicUploadAPI { + return d.UploadAPI + } + getCachedUrlFunc := func() string { + d.uploadUrlMu.RLock() + defer d.uploadUrlMu.RUnlock() + if d.uploadUrl != "" && time.Since(d.uploadUrlUpdateTime) < UPLOAD_URL_EXPIRE_TIME { + return d.uploadUrl } + return "" + } + // 检查地址缓存 + if uploadUrl := getCachedUrlFunc(); uploadUrl != "" { + return uploadUrl + } + + uploadUrlGetFunc := func() (string, error) { + // 双重检查缓存 + if uploadUrl := getCachedUrlFunc(); uploadUrl != "" { + return uploadUrl, nil + } + + uploadUrl, err := d.requestForUploadUrl(path, uploadId) + if err != nil { + return "", err + } + + d.uploadUrlMu.Lock() + defer d.uploadUrlMu.Unlock() + d.uploadUrl = uploadUrl + d.uploadUrlUpdateTime = time.Now() + return uploadUrl, nil + } + + uploadUrl, err, _ := d.uploadUrlG.Do("", uploadUrlGetFunc) + if err != nil { + fallback := d.UploadAPI + log.Warnf("[baidu_netdisk] get upload URL failed (%v), will use fallback URL: %s", err, fallback) + return fallback } + return uploadUrl +} + +// requestForUploadUrl 请求获取上传地址。 +// 实测此接口不需要认证,传method和upload_version就行,不过还是按文档规范调用。 +// https://pan.baidu.com/union/doc/Mlvw5hfnr +func (d *BaiduNetdisk) requestForUploadUrl(path, uploadId string) (string, error) { + params := map[string]string{ + "method": "locateupload", + "appid": "250528", + "path": path, + "uploadid": uploadId, + "upload_version": "2.0", + } + apiUrl := "https://d.pcs.baidu.com/rest/2.0/pcs/file" + var resp UploadServerResp + _, err := d.request(apiUrl, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(params) + }, &resp) + if err != nil { + return "", err + } + var uploadUrl string + if len(resp.Servers) > 0 { + uploadUrl = resp.Servers[0].Server + } else if len(resp.BakServers) > 0 { + uploadUrl = resp.BakServers[0].Server + } + if uploadUrl == "" { + return "", errors.New("upload URL is empty") + } + return uploadUrl, nil } // func encodeURIComponent(str string) string { @@ -247,3 +423,40 @@ func updateObjMd5(obj model.Obj, userAgent, u string) { // r = strings.ReplaceAll(r, "+", "%20") // return r // } + +func DecryptMd5(encryptMd5 string) string { + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} + +func EncryptMd5(originalMd5 string) string { + reversed := originalMd5[8:16] + originalMd5[:8] + originalMd5[24:32] + originalMd5[16:24] + + var out strings.Builder + out.Grow(len(reversed)) + for i, n := 0, int64(0); i < len(reversed); i++ { + n, _ = strconv.ParseInt(reversed[i:i+1], 16, 64) + n ^= int64(15 & i) + if i == 9 { + out.WriteRune(rune(n) + 'g') + } else { + out.WriteString(strconv.FormatInt(n, 16)) + } + } + return out.String() +} diff --git a/drivers/baidu_photo/driver.go b/drivers/baidu_photo/driver.go index 9105260d94d..5a34fcb4639 100644 --- a/drivers/baidu_photo/driver.go +++ b/drivers/baidu_photo/driver.go @@ -7,13 +7,16 @@ import ( "errors" "fmt" "io" - "math" + "os" "regexp" "strconv" "strings" "time" + "golang.org/x/sync/semaphore" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -27,9 +30,10 @@ type BaiduPhoto struct { model.Storage Addition - AccessToken string - Uk int64 - root model.Obj + // AccessToken string + Uk int64 + bdstoken string + root model.Obj uploadThread int } @@ -48,9 +52,9 @@ func (d *BaiduPhoto) Init(ctx context.Context) error { d.uploadThread, d.UploadThread = 3, "3" } - if err := d.refreshToken(); err != nil { - return err - } + // if err := d.refreshToken(); err != nil { + // return err + // } // root if d.AlbumID != "" { @@ -73,6 +77,10 @@ func (d *BaiduPhoto) Init(ctx context.Context) error { if err != nil { return err } + d.bdstoken, err = d.getBDStoken() + if err != nil { + return err + } d.Uk, err = strconv.ParseInt(info.YouaID, 10, 64) return err } @@ -82,7 +90,7 @@ func (d *BaiduPhoto) GetRoot(ctx context.Context) (model.Obj, error) { } func (d *BaiduPhoto) Drop(ctx context.Context) error { - d.AccessToken = "" + // d.AccessToken = "" d.Uk = 0 d.root = nil return nil @@ -137,13 +145,18 @@ func (d *BaiduPhoto) Link(ctx context.Context, file model.Obj, args model.LinkAr case *File: return d.linkFile(ctx, file, args) case *AlbumFile: - f, err := d.CopyAlbumFile(ctx, file) - if err != nil { - return nil, err + // 处理共享相册 + if d.Uk != file.Uk { + // 有概率无法获取到链接 + // return d.linkAlbum(ctx, file, args) + + f, err := d.CopyAlbumFile(ctx, file) + if err != nil { + return nil, err + } + return d.linkFile(ctx, f, args) } - return d.linkFile(ctx, f, args) - // 有概率无法获取到链接 - //return d.linkAlbum(ctx, file, args) + return d.linkFile(ctx, &file.File, args) } return nil, errs.NotFile } @@ -229,11 +242,21 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil // TODO: // 暂时没有找到妙传方式 - - // 需要获取完整文件md5,必须支持 io.Seek - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return nil, err + var ( + cache = stream.GetFile() + tmpF *os.File + err error + ) + if _, ok := cache.(io.ReaderAt); !ok { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF } const DEFAULT int64 = 1 << 22 @@ -241,9 +264,11 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil // 计算需要的数据 streamSize := stream.GetSize() - count := int(math.Ceil(float64(streamSize) / float64(DEFAULT))) + count := int(streamSize / DEFAULT) lastBlockSize := streamSize % DEFAULT - if lastBlockSize == 0 { + if lastBlockSize > 0 { + count++ + } else { lastBlockSize = DEFAULT } @@ -254,6 +279,11 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil sliceMd5H := md5.New() sliceMd5H2 := md5.New() slicemd5H2Write := utils.LimitWriter(sliceMd5H2, SliceSize) + writers := []io.Writer{fileMd5H, sliceMd5H, slicemd5H2Write} + if tmpF != nil { + writers = append(writers, tmpF) + } + written := int64(0) for i := 1; i <= count; i++ { if utils.IsCanceled(ctx) { return nil, ctx.Err() @@ -261,13 +291,23 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if i == count { byteSize = lastBlockSize } - _, err := io.CopyN(io.MultiWriter(fileMd5H, sliceMd5H, slicemd5H2Write), tempFile, byteSize) + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) + written += n if err != nil && err != io.EOF { return nil, err } sliceMD5List = append(sliceMD5List, hex.EncodeToString(sliceMd5H.Sum(nil))) sliceMd5H.Reset() } + if tmpF != nil { + if written != streamSize { + return nil, errs.NewErr(err, "CreateTempFile failed, incoming stream actual size= %d, expect = %d ", written, streamSize) + } + _, err = tmpF.Seek(0, io.SeekStart) + if err != nil { + return nil, errs.NewErr(err, "CreateTempFile failed, can't seek to 0 ") + } + } contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) blockListStr, _ := utils.Json.MarshalToString(sliceMD5List) @@ -279,18 +319,19 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil "rtype": "1", "ctype": "11", "path": fmt.Sprintf("/%s", stream.GetName()), - "size": fmt.Sprint(stream.GetSize()), + "size": fmt.Sprint(streamSize), "slice-md5": sliceMd5, "content-md5": contentMd5, "block_list": blockListStr, } // 尝试获取之前的进度 - precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, d.AccessToken, contentMd5) + precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, strconv.FormatInt(d.Uk, 10), contentMd5) if !ok { _, err = d.Post(FILE_API_URL_V1+"/precreate", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(params) + r.SetQueryParam("bdstoken", d.bdstoken) }, &precreateResp) if err != nil { return nil, err @@ -303,6 +344,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) + sem := semaphore.NewWeighted(3) for i, partseq := range precreateResp.BlockList { if utils.IsCanceled(upCtx) { break @@ -314,22 +356,27 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil } threadG.Go(func(ctx context.Context) error { + if err = sem.Acquire(ctx, 1); err != nil { + return err + } + defer sem.Release(1) uploadParams := map[string]string{ "method": "upload", "path": params["path"], "partseq": fmt.Sprint(partseq), "uploadid": precreateResp.UploadID, + "app_id": "16051585", } - _, err = d.Post("https://c3.pcs.baidu.com/rest/2.0/pcs/superfile2", func(r *resty.Request) { r.SetContext(ctx) r.SetQueryParams(uploadParams) - r.SetFileReader("file", stream.GetName(), io.NewSectionReader(tempFile, offset, byteSize)) + r.SetFileReader("file", stream.GetName(), + driver.NewLimitedUploadStream(ctx, io.NewSectionReader(cache, offset, byteSize))) }, nil) if err != nil { return err } - up(int(threadG.Success()) * 100 / len(precreateResp.BlockList)) + up(float64(threadG.Success()) * 100 / float64(len(precreateResp.BlockList))) precreateResp.BlockList[i] = -1 return nil }) @@ -337,7 +384,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if err = threadG.Wait(); err != nil { if errors.Is(err, context.Canceled) { precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(s int) bool { return s >= 0 }) - base.SaveUploadProgress(d, precreateResp, d.AccessToken, contentMd5) + base.SaveUploadProgress(d, strconv.FormatInt(d.Uk, 10), contentMd5) } return nil, err } @@ -347,6 +394,7 @@ func (d *BaiduPhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fil _, err = d.Post(FILE_API_URL_V1+"/create", func(r *resty.Request) { r.SetContext(ctx) r.SetFormData(params) + r.SetQueryParam("bdstoken", d.bdstoken) }, &precreateResp) if err != nil { return nil, err diff --git a/drivers/baidu_photo/meta.go b/drivers/baidu_photo/meta.go index da2229f5424..3bc2f6227c5 100644 --- a/drivers/baidu_photo/meta.go +++ b/drivers/baidu_photo/meta.go @@ -6,13 +6,14 @@ import ( ) type Addition struct { - RefreshToken string `json:"refresh_token" required:"true"` - ShowType string `json:"show_type" type:"select" options:"root,root_only_album,root_only_file" default:"root"` - AlbumID string `json:"album_id"` + // RefreshToken string `json:"refresh_token" required:"true"` + Cookie string `json:"cookie" required:"true"` + ShowType string `json:"show_type" type:"select" options:"root,root_only_album,root_only_file" default:"root"` + AlbumID string `json:"album_id"` //AlbumPassword string `json:"album_password"` - DeleteOrigin bool `json:"delete_origin"` - ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` - ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` + DeleteOrigin bool `json:"delete_origin"` + // ClientID string `json:"client_id" required:"true" default:"iYCeC9g08h5vuP9UqvPHKKSVrKFXGa1v"` + // ClientSecret string `json:"client_secret" required:"true" default:"jXiFMOPVPCWlO2M5CwWQzffpNPaGTRBG"` UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` } diff --git a/drivers/baidu_photo/types.go b/drivers/baidu_photo/types.go index 2bbacd303f7..0e5cbb2cdd5 100644 --- a/drivers/baidu_photo/types.go +++ b/drivers/baidu_photo/types.go @@ -72,7 +72,7 @@ func (c *File) Thumb() string { } func (c *File) GetHash() utils.HashInfo { - return utils.NewHashInfo(utils.MD5, c.Md5) + return utils.NewHashInfo(utils.MD5, DecryptMd5(c.Md5)) } /*相册部分*/ diff --git a/drivers/baidu_photo/utils.go b/drivers/baidu_photo/utils.go index c93f6f1265a..6061600ea09 100644 --- a/drivers/baidu_photo/utils.go +++ b/drivers/baidu_photo/utils.go @@ -2,13 +2,15 @@ package baiduphoto import ( "context" + "encoding/hex" "fmt" "net/http" + "strconv" + "strings" + "unicode" "github.com/alist-org/alist/v3/drivers/base" - "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -21,9 +23,10 @@ const ( FILE_API_URL_V2 = API_URL + "/file/v2" ) -func (d *BaiduPhoto) Request(furl string, method string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { - req := base.RestyClient.R(). - SetQueryParam("access_token", d.AccessToken) +func (d *BaiduPhoto) Request(client *resty.Client, furl string, method string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { + req := client.R(). + // SetQueryParam("access_token", d.AccessToken) + SetHeader("Cookie", d.Cookie) if callback != nil { callback(req) } @@ -45,10 +48,10 @@ func (d *BaiduPhoto) Request(furl string, method string, callback base.ReqCallba return nil, fmt.Errorf("no shared albums found") case 50100: return nil, fmt.Errorf("illegal title, only supports 50 characters") - case -6: - if err = d.refreshToken(); err != nil { - return nil, err - } + // case -6: + // if err = d.refreshToken(); err != nil { + // return nil, err + // } default: return nil, fmt.Errorf("errno: %d, refer to https://photo.baidu.com/union/doc", erron) } @@ -63,36 +66,36 @@ func (d *BaiduPhoto) Request(furl string, method string, callback base.ReqCallba // return res.Body(), nil //} -func (d *BaiduPhoto) refreshToken() error { - u := "https://openapi.baidu.com/oauth/2.0/token" - var resp base.TokenResp - var e TokenErrResp - _, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetQueryParams(map[string]string{ - "grant_type": "refresh_token", - "refresh_token": d.RefreshToken, - "client_id": d.ClientID, - "client_secret": d.ClientSecret, - }).Get(u) - if err != nil { - return err - } - if e.ErrorMsg != "" { - return &e - } - if resp.RefreshToken == "" { - return errs.EmptyToken - } - d.AccessToken, d.RefreshToken = resp.AccessToken, resp.RefreshToken - op.MustSaveDriverStorage(d) - return nil -} +// func (d *BaiduPhoto) refreshToken() error { +// u := "https://openapi.baidu.com/oauth/2.0/token" +// var resp base.TokenResp +// var e TokenErrResp +// _, err := base.RestyClient.R().SetResult(&resp).SetError(&e).SetQueryParams(map[string]string{ +// "grant_type": "refresh_token", +// "refresh_token": d.RefreshToken, +// "client_id": d.ClientID, +// "client_secret": d.ClientSecret, +// }).Get(u) +// if err != nil { +// return err +// } +// if e.ErrorMsg != "" { +// return &e +// } +// if resp.RefreshToken == "" { +// return errs.EmptyToken +// } +// d.AccessToken, d.RefreshToken = resp.AccessToken, resp.RefreshToken +// op.MustSaveDriverStorage(d) +// return nil +// } func (d *BaiduPhoto) Get(furl string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { - return d.Request(furl, http.MethodGet, callback, resp) + return d.Request(base.RestyClient, furl, http.MethodGet, callback, resp) } func (d *BaiduPhoto) Post(furl string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { - return d.Request(furl, http.MethodPost, callback, resp) + return d.Request(base.RestyClient, furl, http.MethodPost, callback, resp) } // 获取所有文件 @@ -338,24 +341,29 @@ func (d *BaiduPhoto) linkAlbum(ctx context.Context, file *AlbumFile, args model. headers["X-Forwarded-For"] = args.IP } - res, err := base.NoRedirectClient.R(). - SetContext(ctx). - SetHeaders(headers). - SetQueryParams(map[string]string{ - "access_token": d.AccessToken, - "fsid": fmt.Sprint(file.Fsid), - "album_id": file.AlbumID, - "tid": fmt.Sprint(file.Tid), - "uk": fmt.Sprint(file.Uk), - }). - Head(ALBUM_API_URL + "/download") + resp, err := d.Request(base.NoRedirectClient, ALBUM_API_URL+"/download", http.MethodHead, func(r *resty.Request) { + r.SetContext(ctx) + r.SetHeaders(headers) + r.SetQueryParams(map[string]string{ + "fsid": fmt.Sprint(file.Fsid), + "album_id": file.AlbumID, + "tid": fmt.Sprint(file.Tid), + "uk": fmt.Sprint(file.Uk), + }) + }, nil) if err != nil { return nil, err } + if resp.StatusCode() != 302 { + return nil, fmt.Errorf("not found 302 redirect") + } + + location := resp.Header().Get("Location") + link := &model.Link{ - URL: res.Header().Get("location"), + URL: location, Header: http.Header{ "User-Agent": []string{headers["User-Agent"]}, "Referer": []string{"https://photo.baidu.com/"}, @@ -385,10 +393,24 @@ func (d *BaiduPhoto) linkFile(ctx context.Context, file *File, args model.LinkAr "fsid": fmt.Sprint(file.Fsid), }) }, &downloadUrl) + + // resp, err := d.Request(base.NoRedirectClient, FILE_API_URL_V1+"/download", http.MethodHead, func(r *resty.Request) { + // r.SetContext(ctx) + // r.SetHeaders(headers) + // r.SetQueryParams(map[string]string{ + // "fsid": fmt.Sprint(file.Fsid), + // }) + // }, nil) + if err != nil { return nil, err } + // if resp.StatusCode() != 302 { + // return nil, fmt.Errorf("not found 302 redirect") + // } + + // location := resp.Header().Get("Location") link := &model.Link{ URL: downloadUrl.Dlink, Header: http.Header{ @@ -453,3 +475,55 @@ func (d *BaiduPhoto) uInfo() (*UInfo, error) { } return &info, nil } + +func (d *BaiduPhoto) getBDStoken() (string, error) { + var info struct { + Result struct { + Bdstoken string `json:"bdstoken"` + Token string `json:"token"` + Uk int64 `json:"uk"` + } `json:"result"` + } + _, err := d.Get("https://pan.baidu.com/api/gettemplatevariable?fields=[%22bdstoken%22,%22token%22,%22uk%22]", nil, &info) + if err != nil { + return "", err + } + return info.Result.Bdstoken, nil +} + +func DecryptMd5(encryptMd5 string) string { + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} + +func EncryptMd5(originalMd5 string) string { + reversed := originalMd5[8:16] + originalMd5[:8] + originalMd5[24:32] + originalMd5[16:24] + + var out strings.Builder + out.Grow(len(reversed)) + for i, n := 0, int64(0); i < len(reversed); i++ { + n, _ = strconv.ParseInt(reversed[i:i+1], 16, 64) + n ^= int64(15 & i) + if i == 9 { + out.WriteRune(rune(n) + 'g') + } else { + out.WriteString(strconv.FormatInt(n, 16)) + } + } + return out.String() +} diff --git a/drivers/baidu_youth/driver.go b/drivers/baidu_youth/driver.go new file mode 100644 index 00000000000..a0b6a4853ec --- /dev/null +++ b/drivers/baidu_youth/driver.go @@ -0,0 +1,425 @@ +package baidu_youth + +import ( + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "os" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" +) + +type BaiduYouth struct { + model.Storage + Addition + + uk int64 + bdstoken string + sk string + uploadThread int + upClient *resty.Client +} + +var ErrUploadIDExpired = errors.New("uploadid expired") + +func (d *BaiduYouth) Config() driver.Config { + return config +} + +func (d *BaiduYouth) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BaiduYouth) Init(ctx context.Context) error { + d.Cookie = strings.TrimSpace(d.Cookie) + if d.Storage.Addition != "" { + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(d.Storage.Addition), &raw); err == nil { + if _, ok := raw["force_proxy"]; !ok { + d.ForceProxy = true + } + } + } + d.upClient = base.NewRestyClient(). + SetTimeout(UPLOAD_TIMEOUT). + SetRetryCount(UPLOAD_RETRY_COUNT). + SetRetryWaitTime(UPLOAD_RETRY_WAIT_TIME). + SetRetryMaxWaitTime(UPLOAD_RETRY_MAX_WAIT_TIME) + + d.uploadThread, _ = strconv.Atoi(d.UploadThread) + if d.uploadThread < 1 { + d.uploadThread, d.UploadThread = 1, "1" + } else if d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 32, "32" + } + + u, err := url.Parse(d.UploadAPI) + if d.UploadAPI == "" || err != nil || u.Scheme == "" || u.Host == "" { + d.UploadAPI = UPLOAD_FALLBACK_API + } else { + d.UploadAPI = strings.TrimRight(d.UploadAPI, "/") + } + + uk, bdstoken, sk, err := d.getUserSession(ctx) + if err != nil { + return err + } + d.uk = uk + d.bdstoken = bdstoken + d.sk = sk + return nil +} + +func (d *BaiduYouth) ShouldProxyDownloads() bool { + return d.ForceProxy +} + +func (d *BaiduYouth) Drop(ctx context.Context) error { + d.uk = 0 + d.bdstoken = "" + d.sk = "" + return nil +} + +func (d *BaiduYouth) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetPath()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *BaiduYouth) Get(ctx context.Context, path string) (model.Obj, error) { + return d.getByPath(ctx, path) +} + +func (d *BaiduYouth) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if d.DownloadAPI == "crack" { + return d.linkCrack(ctx, file) + } + return d.linkOfficial(ctx, file) +} + +func (d *BaiduYouth) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + path := stdpath.Join(parentDir.GetPath(), dirName) + var resp CreateResp + _, err := d.postForm(ctx, "/youth/api/create", map[string]string{ + "a": "commit", + "bdstoken": d.bdstoken, + }, map[string]string{ + "block_list": "[]", + "isdir": "1", + "path": path, + }, &resp) + if err != nil { + return nil, err + } + if result := resp.ResultFile(); result.Path != "" || result.FsId != 0 { + return fileToObj(result), nil + } + return d.getByPath(ctx, path) +} + +func (d *BaiduYouth) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + _, err := d.manage(ctx, "move", []base.Json{ + { + "dest": dstDir.GetPath(), + "newname": srcObj.GetName(), + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + newPath := stdpath.Join(dstDir.GetPath(), srcObj.GetName()) + if obj, ok := srcObj.(*model.ObjThumb); ok { + obj.SetPath(newPath) + obj.Modified = time.Now() + return obj, nil + } + return d.getByPath(ctx, newPath) +} + +func (d *BaiduYouth) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + _, err := d.manage(ctx, "rename", []base.Json{ + { + "id": srcObj.GetID(), + "newname": newName, + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + newPath := stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName) + if obj, ok := srcObj.(*model.ObjThumb); ok { + obj.SetPath(newPath) + obj.Name = newName + obj.Modified = time.Now() + return obj, nil + } + return d.getByPath(ctx, newPath) +} + +func (d *BaiduYouth) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + newPath := stdpath.Join(dstDir.GetPath(), srcObj.GetName()) + _, err := d.manage(ctx, "copy", []base.Json{ + { + "dest": dstDir.GetPath(), + "newname": srcObj.GetName(), + "path": srcObj.GetPath(), + }, + }) + if err != nil { + return nil, err + } + if obj, ok := srcObj.(*model.ObjThumb); ok { + copied := *obj + copied.SetPath(newPath) + copied.Modified = time.Now() + return &copied, nil + } + // Youth copy returns success before /api/filemetas can resolve the new path. + // Avoid turning a successful copy into a false failure because the immediate + // post-copy lookup is temporarily unavailable. + return &model.Object{ + ID: newPath, + Path: newPath, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + HashInfo: srcObj.GetHash(), + }, nil +} + +func (d *BaiduYouth) Remove(ctx context.Context, obj model.Obj) error { + _, err := d.manage(ctx, "delete", []string{obj.GetPath()}) + return err +} + +func (d *BaiduYouth) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if stream.GetSize() < 1 { + return nil, ErrBaiduYouthEmptyFilesNotAllowed + } + + var ( + cache = stream.GetFile() + tmpF *os.File + err error + ) + if _, ok := cache.(io.ReaderAt); !ok { + tmpF, err = os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + defer func() { + _ = tmpF.Close() + _ = os.Remove(tmpF.Name()) + }() + cache = tmpF + } + + streamSize := stream.GetSize() + sliceSize := DefaultSliceSize + count := int(streamSize / sliceSize) + lastBlockSize := streamSize % sliceSize + if lastBlockSize > 0 { + count++ + } else { + lastBlockSize = sliceSize + } + + const sliceMD5Size int64 = 256 * utils.KB + blockList := make([]string, 0, count) + byteSize := sliceSize + fileMd5H := md5.New() + sliceMd5H := md5.New() + sliceMd5H2 := md5.New() + sliceMd5H2Writer := utils.LimitWriter(sliceMd5H2, sliceMD5Size) + writers := []io.Writer{fileMd5H, sliceMd5H, sliceMd5H2Writer} + if tmpF != nil { + writers = append(writers, tmpF) + } + + written := int64(0) + for i := 1; i <= count; i++ { + if utils.IsCanceled(ctx) { + return nil, ctx.Err() + } + if i == count { + byteSize = lastBlockSize + } + n, err := utils.CopyWithBufferN(io.MultiWriter(writers...), stream, byteSize) + written += n + if err != nil && err != io.EOF { + return nil, err + } + blockList = append(blockList, hex.EncodeToString(sliceMd5H.Sum(nil))) + sliceMd5H.Reset() + } + + if tmpF != nil { + if written != streamSize { + return nil, errs.NewErr(errs.StreamIncomplete, "temp file size mismatch: %d != %d", written, streamSize) + } + if _, err = tmpF.Seek(0, io.SeekStart); err != nil { + return nil, err + } + } + + contentMd5 := hex.EncodeToString(fileMd5H.Sum(nil)) + sliceMd5 := hex.EncodeToString(sliceMd5H2.Sum(nil)) + blockListStr, err := utils.Json.MarshalToString(blockList) + if err != nil { + return nil, err + } + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) + mtime := stream.ModTime().Unix() + ctime := stream.CreateTime().Unix() + + progressKey := d.uploadProgressKey() + precreateResp, ok := base.GetUploadProgress[*PrecreateResp](d, progressKey, contentMd5) + if !ok { + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) + if err != nil { + return nil, err + } + } + + if precreateResp.ReturnType >= 2 { + result := precreateResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil + } + + cacheReaderAt, ok := cache.(io.ReaderAt) + if !ok { + return nil, fmt.Errorf("cache object must implement io.ReaderAt") + } + +uploadLoop: + for attempt := 0; attempt < 2; attempt++ { + completed := count - len(precreateResp.BlockList) + threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + for i, partseq := range precreateResp.BlockList { + if utils.IsCanceled(upCtx) || partseq < 0 { + continue + } + + i, partseq := i, partseq + offset, size := int64(partseq)*sliceSize, sliceSize + if partseq+1 == count { + size = lastBlockSize + } + + threadG.Go(func(ctx context.Context) error { + params := map[string]string{ + "method": "upload", + "partseq": strconv.Itoa(partseq), + "path": path, + "type": "tmpfile", + "uploadid": precreateResp.Uploadid, + } + if precreateResp.Uploadsign != "" { + params["uploadsign"] = precreateResp.Uploadsign + } + section := io.NewSectionReader(cacheReaderAt, offset, size) + if err := d.uploadSlice(ctx, params, stream.GetName(), driver.NewLimitedUploadStream(ctx, section)); err != nil { + return err + } + precreateResp.BlockList[i] = -1 + success := completed + int(threadG.Success()) + 1 + up(float64(success) * 100 / float64(count)) + return nil + }) + } + + err = threadG.Wait() + if err == nil { + break uploadLoop + } + + precreateResp.BlockList = utils.SliceFilter(precreateResp.BlockList, func(part int) bool { + return part >= 0 + }) + base.SaveUploadProgress(d, precreateResp, progressKey, contentMd5) + + if errors.Is(err, context.Canceled) { + return nil, err + } + if errors.Is(err, ErrUploadIDExpired) { + precreateResp, err = d.precreate(ctx, path, streamSize, blockListStr, contentMd5, sliceMd5, ctime, mtime) + if err != nil { + return nil, err + } + if precreateResp.ReturnType >= 2 { + result := precreateResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil + } + base.SaveUploadProgress(d, precreateResp, progressKey, contentMd5) + continue uploadLoop + } + return nil, err + } + + var createResp CreateResp + _, err = d.createFile(ctx, path, stdpath.Dir(path), streamSize, precreateResp.Uploadid, precreateResp.Uploadsign, blockListStr, &createResp, mtime, ctime) + if err != nil { + return nil, err + } + + base.SaveUploadProgress(d, nil, progressKey, contentMd5) + result := createResp.ResultFile() + if result.Path == "" && result.FsId == 0 { + return d.getByPath(ctx, path) + } + result.Ctime = ctime + result.Mtime = mtime + return fileToObj(result), nil +} + +var _ driver.Driver = (*BaiduYouth)(nil) +var _ driver.Getter = (*BaiduYouth)(nil) +var _ driver.MkdirResult = (*BaiduYouth)(nil) +var _ driver.MoveResult = (*BaiduYouth)(nil) +var _ driver.RenameResult = (*BaiduYouth)(nil) +var _ driver.CopyResult = (*BaiduYouth)(nil) +var _ driver.Remove = (*BaiduYouth)(nil) +var _ driver.PutResult = (*BaiduYouth)(nil) diff --git a/drivers/baidu_youth/meta.go b/drivers/baidu_youth/meta.go new file mode 100644 index 00000000000..7d5cb6a8698 --- /dev/null +++ b/drivers/baidu_youth/meta.go @@ -0,0 +1,40 @@ +package baidu_youth + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Cookie string `json:"cookie" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + ForceProxy bool `json:"force_proxy" type:"bool" default:"true" help:"Proxy downloads through AList. Disable to redirect the browser to a fresh Baidu direct link."` + DownloadAPI string `json:"download_api" type:"select" options:"official,crack" default:"official"` + UploadThread string `json:"upload_thread" default:"3" help:"1<=thread<=32"` + UploadAPI string `json:"upload_api" default:"https://d.pcs.baidu.com"` +} + +const ( + UPLOAD_FALLBACK_API = "https://d.pcs.baidu.com" + UPLOAD_TIMEOUT = time.Minute * 30 + UPLOAD_RETRY_COUNT = 3 + UPLOAD_RETRY_WAIT_TIME = time.Second + UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5 + DefaultSliceSize int64 = 4 * 1024 * 1024 +) + +var config = driver.Config{ + Name: "BaiduYouth", + DefaultRoot: "/", + OnlyProxy: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BaiduYouth{} + }) +} diff --git a/drivers/baidu_youth/types.go b/drivers/baidu_youth/types.go new file mode 100644 index 00000000000..82024009403 --- /dev/null +++ b/drivers/baidu_youth/types.go @@ -0,0 +1,130 @@ +package baidu_youth + +import ( + "errors" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var ( + ErrBaiduYouthEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu youth") +) + +type File struct { + Category int `json:"category"` + FsId int64 `json:"fs_id"` + Thumbs struct { + Url3 string `json:"url3"` + } `json:"thumbs"` + Size int64 `json:"size"` + Path string `json:"path"` + ServerFilename string `json:"server_filename"` + Md5 string `json:"md5"` + Isdir int `json:"isdir"` + ServerCtime int64 `json:"server_ctime"` + ServerMtime int64 `json:"server_mtime"` + LocalMtime int64 `json:"local_mtime"` + LocalCtime int64 `json:"local_ctime"` + Ctime int64 `json:"ctime"` + Mtime int64 `json:"mtime"` + Dlink string `json:"dlink"` +} + +func fileToObj(f File) *model.ObjThumb { + if f.ServerFilename == "" { + f.ServerFilename = path.Base(f.Path) + } + if f.ServerCtime == 0 { + if f.LocalCtime != 0 { + f.ServerCtime = f.LocalCtime + } else { + f.ServerCtime = f.Ctime + } + } + if f.ServerMtime == 0 { + if f.LocalMtime != 0 { + f.ServerMtime = f.LocalMtime + } else { + f.ServerMtime = f.Mtime + } + } + return &model.ObjThumb{ + Object: model.Object{ + ID: strconv.FormatInt(f.FsId, 10), + Path: f.Path, + Name: f.ServerFilename, + Size: f.Size, + Modified: time.Unix(f.ServerMtime, 0), + Ctime: time.Unix(f.ServerCtime, 0), + IsFolder: f.Isdir == 1, + HashInfo: utils.NewHashInfo(utils.MD5, DecryptMd5(f.Md5)), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.Thumbs.Url3}, + } +} + +type ListResp struct { + Errno int `json:"errno"` + List []File `json:"list"` +} + +type FileMetaResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + Info []File `json:"info"` + List []File `json:"list"` +} + +type LocateDownloadResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Path string `json:"path"` + URL string `json:"url"` +} + +type MediaInfoResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Info File `json:"info"` +} + +type CreateResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + Info File `json:"info"` + File +} + +func (r CreateResp) ResultFile() File { + if r.Info.Path != "" || r.Info.FsId != 0 { + return r.Info + } + return r.File +} + +type PrecreateResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` + ShowMsg string `json:"show_msg"` + ReturnType int `json:"return_type"` + Path string `json:"path"` + Uploadid string `json:"uploadid"` + Uploadsign string `json:"uploadsign"` + BlockList []int `json:"block_list"` + Info File `json:"info"` + File +} + +func (r PrecreateResp) ResultFile() File { + if r.Info.Path != "" || r.Info.FsId != 0 { + return r.Info + } + return r.File +} diff --git a/drivers/baidu_youth/util.go b/drivers/baidu_youth/util.go new file mode 100644 index 00000000000..b83518fe350 --- /dev/null +++ b/drivers/baidu_youth/util.go @@ -0,0 +1,606 @@ +package baidu_youth + +import ( + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + stdpath "path" + "strconv" + "strings" + "time" + "unicode" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + panBaseURL = "https://pan.baidu.com" + panReferer = panBaseURL + "/" + panUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36" + youthReferer = panBaseURL + "/youth/pan/main#/index?category=all" + youthAppID = "250528" + uploadAppID = "25179614" + videoAPIChannel = "android_15_25010PN30C_bd-netdisk_1523a" + videoAPIDevUID = "0%1" +) + +func (d *BaiduYouth) normalizeURL(furl string) string { + if strings.HasPrefix(furl, "http://") || strings.HasPrefix(furl, "https://") { + return furl + } + return panBaseURL + furl +} + +func (d *BaiduYouth) commonHeaders() map[string]string { + return map[string]string{ + "Accept": "application/json, text/plain, */*", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Cache-Control": "no-cache", + "Cookie": d.Cookie, + "Origin": panBaseURL, + "Pragma": "no-cache", + "Referer": youthReferer, + "User-Agent": panUserAgent, + "X-Requested-With": "XMLHttpRequest", + } +} + +func youthQueryParams() map[string]string { + return map[string]string{ + "app_id": youthAppID, + "clienttype": "0", + "web": "1", + } +} + +func youthUploadQueryParams() map[string]string { + return map[string]string{ + "app_id": uploadAppID, + "channel": "chunlei", + "clienttype": "0", + "web": "1", + } +} + +func extractBaiduMessage(body []byte) string { + for _, path := range [][]interface{}{ + {"show_msg"}, + {"errmsg"}, + {"error_msg"}, + {"error_description"}, + {"message"}, + } { + msg := utils.Json.Get(body, path...).ToString() + if msg != "" { + return msg + } + } + return "" +} + +func (d *BaiduYouth) doRequest(furl string, method string, defaultQuery map[string]string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R().SetHeaders(d.commonHeaders()) + if defaultQuery != nil { + req.SetQueryParams(defaultQuery) + } + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, d.normalizeURL(furl)) + if err != nil { + return nil, err + } + body := res.Body() + if errno := utils.Json.Get(body, "errno").ToInt(); errno != 0 { + msg := extractBaiduMessage(body) + if errno == -6 && msg == "" { + msg = "cookie expired or invalid" + } + if msg == "" { + msg = "request failed" + } + return nil, fmt.Errorf("[baidu_youth] %s (errno=%d)", msg, errno) + } + return body, nil +} + +func (d *BaiduYouth) get(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, youthQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) postForm(ctx context.Context, pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodPost, youthQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(form) + }, resp) +} + +func (d *BaiduYouth) getUpload(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, youthUploadQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) postUploadForm(ctx context.Context, pathname string, params map[string]string, form map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodPost, youthUploadQueryParams(), func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(form) + }, resp) +} + +func (d *BaiduYouth) getBare(ctx context.Context, pathname string, params map[string]string, resp interface{}) ([]byte, error) { + return d.doRequest(pathname, http.MethodGet, nil, func(req *resty.Request) { + req.SetContext(ctx) + if params != nil { + req.SetQueryParams(params) + } + }, resp) +} + +func (d *BaiduYouth) getUserSK(ctx context.Context) (string, error) { + body, err := d.get(ctx, "/youth/api/report/user", map[string]string{ + "action": "sapi_auth", + "timestamp": strconv.FormatInt(time.Now().UnixMilli(), 10), + }, nil) + if err != nil { + return "", err + } + return utils.Json.Get(body, "uinfo").ToString(), nil +} + +func (d *BaiduYouth) getUserSession(ctx context.Context) (int64, string, string, error) { + body, err := d.get(ctx, "/youth/api/user/getinfo", map[string]string{ + "need_selfinfo": "1", + }, nil) + if err != nil { + return 0, "", "", err + } + uk := int64(utils.Json.Get(body, "records", 0, "uk").ToInt()) + bdstoken := utils.Json.Get(body, "records", 0, "bdstoken").ToString() + sk := utils.Json.Get(body, "records", 0, "sk").ToString() + if bdstoken == "" || uk == 0 || sk == "" { + body, err = d.getBare(ctx, "/api/gettemplatevariable", map[string]string{ + "fields": `["bdstoken","uk","sk"]`, + }, nil) + if err != nil { + return 0, "", "", err + } + if uk == 0 { + uk = int64(utils.Json.Get(body, "result", "uk").ToInt()) + } + if bdstoken == "" { + bdstoken = utils.Json.Get(body, "result", "bdstoken").ToString() + } + if sk == "" { + sk = utils.Json.Get(body, "result", "sk").ToString() + } + } + if sk == "" { + sk, _ = d.getUserSK(ctx) + } + if bdstoken == "" { + return 0, "", "", fmt.Errorf("failed to get bdstoken from baidu youth cookie") + } + if uk == 0 { + return 0, "", "", fmt.Errorf("failed to get uk from baidu youth cookie") + } + return uk, bdstoken, sk, nil +} + +func (d *BaiduYouth) getFiles(ctx context.Context, dir string) ([]File, error) { + page := 1 + num := 1000 + params := map[string]string{ + "dir": dir, + } + if d.OrderBy != "" { + params["order"] = d.OrderBy + if d.OrderDirection == "desc" { + params["desc"] = "1" + } else { + params["desc"] = "0" + } + } + files := make([]File, 0) + for { + params["page"] = strconv.Itoa(page) + params["num"] = strconv.Itoa(num) + var resp ListResp + _, err := d.get(ctx, "/youth/api/list", params, &resp) + if err != nil { + return nil, err + } + if len(resp.List) == 0 { + return files, nil + } + files = append(files, resp.List...) + if len(resp.List) < num { + return files, nil + } + page++ + } +} + +func (d *BaiduYouth) getFileByPath(ctx context.Context, path string) (File, error) { + if path == "/" { + return File{ + Path: "/", + ServerFilename: "/", + Isdir: 1, + }, nil + } + target, err := utils.Json.MarshalToString([]string{path}) + if err != nil { + return File{}, err + } + var resp FileMetaResp + _, err = d.get(ctx, "/api/filemetas", map[string]string{ + "target": target, + }, &resp) + if err != nil { + return File{}, err + } + if len(resp.Info) > 0 { + return resp.Info[0], nil + } + if len(resp.List) > 0 { + return resp.List[0], nil + } + return File{}, errs.NewErr(errs.ObjectNotFound, "baidu youth path not found: %s", path) +} + +func (d *BaiduYouth) getByPath(ctx context.Context, path string) (model.Obj, error) { + if path == "/" { + return &model.Object{ + Path: "/", + Name: "/", + IsFolder: true, + }, nil + } + file, err := d.getFileByPath(ctx, path) + if err != nil { + return nil, err + } + return fileToObj(file), nil +} + +func (d *BaiduYouth) linkOfficial(ctx context.Context, file model.Obj) (*model.Link, error) { + return d.buildDownloadLink(ctx, file) +} + +func (d *BaiduYouth) linkCrack(ctx context.Context, file model.Obj) (*model.Link, error) { + return d.buildDownloadLink(ctx, file) +} + +func (d *BaiduYouth) downloadHeaders() http.Header { + return http.Header{ + "Accept": []string{"*/*"}, + "Accept-Language": []string{"zh-CN,zh;q=0.9,en;q=0.8"}, + "Cache-Control": []string{"no-cache"}, + "Pragma": []string{"no-cache"}, + "Referer": []string{panReferer}, + "User-Agent": []string{panUserAgent}, + } +} + +func nextDPLogID() string { + return strconv.FormatInt(time.Now().UnixNano(), 10) +} + +func (d *BaiduYouth) getCurrentUserSK(ctx context.Context) (string, error) { + sk, err := d.getUserSK(ctx) + if err == nil && sk != "" { + return sk, nil + } + if d.sk != "" { + return d.sk, nil + } + if err != nil { + return "", err + } + return "", fmt.Errorf("baidu youth sk is empty") +} + +func (d *BaiduYouth) locatedownloadRand(sk string, nowMilli int64) string { + sum := sha1.Sum([]byte(strconv.FormatInt(d.uk, 10) + sk + strconv.FormatInt(nowMilli, 10) + "0")) + return hex.EncodeToString(sum[:]) +} + +func (d *BaiduYouth) locatedownloadSign(fileMD5 string, fileID string, nowMilli int64) string { + sum := md5.Sum([]byte(fileMD5 + "_" + strconv.FormatInt(d.uk, 10) + "_" + fileID + "_" + strconv.FormatInt(nowMilli, 10))) + return hex.EncodeToString(sum[:]) +} + +func (d *BaiduYouth) resolveDownloadMeta(ctx context.Context, file model.Obj) (string, string, string, error) { + parentDir := stdpath.Dir(file.GetPath()) + if parentDir == "." { + parentDir = "/" + } + + files, err := d.getFiles(ctx, parentDir) + if err != nil { + return "", "", "", err + } + for _, listedFile := range files { + if listedFile.Path != file.GetPath() { + continue + } + fileID := strconv.FormatInt(listedFile.FsId, 10) + if listedFile.Path != "" && fileID != "" && listedFile.Md5 != "" { + return listedFile.Path, fileID, listedFile.Md5, nil + } + return "", "", "", fmt.Errorf("baidu youth list metadata incomplete for %s", file.GetPath()) + } + return "", "", "", errs.NewErr(errs.ObjectNotFound, "baidu youth list metadata not found: %s", file.GetPath()) +} + +func normalizeLocatedownloadURL(rawURL string) (string, error) { + if rawURL == "" { + return "", fmt.Errorf("baidu youth locatedownload url is empty") + } + if !strings.Contains(rawURL, "response-cache-control=") { + sep := "&" + if !strings.Contains(rawURL, "?") { + sep = "?" + } + rawURL += sep + "response-cache-control=private" + } + return rawURL, nil +} + +func (d *BaiduYouth) getMediaInfoDLink(ctx context.Context, file model.Obj) (string, error) { + path, fileID, _, err := d.resolveDownloadMeta(ctx, file) + if err != nil { + return "", err + } + + var resp MediaInfoResp + _, err = d.doRequest("/youth/api/mediainfo", http.MethodGet, nil, func(req *resty.Request) { + req.SetContext(ctx) + req.SetQueryParams(map[string]string{ + "channel": videoAPIChannel, + "clienttype": "1", + "devuid": videoAPIDevUID, + "dlink": "1", + "fs_id": fileID, + "media": "1", + "nom3u8": "1", + "origin": "dlna", + "path": path, + "type": "VideoURL", + }) + }, &resp) + if err != nil { + return "", err + } + if resp.Info.Dlink == "" { + return "", fmt.Errorf("baidu youth video dlink not found for %s", path) + } + return resp.Info.Dlink, nil +} + +func (d *BaiduYouth) buildVideoLink(ctx context.Context, file model.Obj) (*model.Link, error) { + dlink, err := d.getMediaInfoDLink(ctx, file) + if err != nil { + return nil, err + } + return &model.Link{ + URL: dlink, + Header: http.Header{ + "Referer": []string{panReferer}, + }, + }, nil +} + +func (d *BaiduYouth) requestLocateDownloadURL(ctx context.Context, path string, fileID string, fileMD5 string, sk string) (string, error) { + nowMilli := time.Now().UnixMilli() + + var resp LocateDownloadResp + _, err := d.get(ctx, "/youth/api/locatedownload", map[string]string{ + "devuid": "0", + "dp-logid": nextDPLogID(), + "path": path, + "rand": d.locatedownloadRand(sk, nowMilli), + "sign": d.locatedownloadSign(fileMD5, fileID, nowMilli), + "time": strconv.FormatInt(nowMilli, 10), + }, &resp) + if err != nil { + return "", err + } + if resp.URL == "" { + return "", fmt.Errorf("baidu youth locatedownload url not found for %s", path) + } + return normalizeLocatedownloadURL(resp.URL) +} + +func (d *BaiduYouth) getLocateDownloadURL(ctx context.Context, file model.Obj) (string, error) { + path, fileID, fileMD5, err := d.resolveDownloadMeta(ctx, file) + if err != nil { + return "", err + } + + sk, err := d.getCurrentUserSK(ctx) + if err != nil { + return "", err + } + downloadURL, err := d.requestLocateDownloadURL(ctx, path, fileID, fileMD5, sk) + if err == nil { + return downloadURL, nil + } + + if !strings.Contains(err.Error(), "errno=-30006") { + return "", err + } + sk, refreshErr := d.getUserSK(ctx) + if refreshErr != nil { + return "", err + } + if sk == "" { + return "", err + } + return d.requestLocateDownloadURL(ctx, path, fileID, fileMD5, sk) +} + +func (d *BaiduYouth) buildDownloadLink(ctx context.Context, file model.Obj) (*model.Link, error) { + downloadURL, err := d.getLocateDownloadURL(ctx, file) + if err != nil { + return nil, err + } + return &model.Link{ + URL: downloadURL, + Header: d.downloadHeaders(), + }, nil +} + +func (d *BaiduYouth) manage(ctx context.Context, opera string, filelist any) ([]byte, error) { + marshal, err := utils.Json.MarshalToString(filelist) + if err != nil { + return nil, err + } + return d.postForm(ctx, "/youth/api/filemanager", map[string]string{ + "async": "0", + "bdstoken": d.bdstoken, + "onnest": "fail", + "opera": opera, + }, map[string]string{ + "filelist": marshal, + "ondup": "fail", + }, nil) +} + +func (d *BaiduYouth) precreate(ctx context.Context, path string, streamSize int64, blockListStr, contentMd5, sliceMd5 string, ctime, mtime int64) (*PrecreateResp, error) { + form := map[string]string{ + "autoinit": "1", + "block_list": blockListStr, + "isdir": "0", + "path": path, + "size": strconv.FormatInt(streamSize, 10), + "target_path": stdpath.Dir(path), + } + if contentMd5 != "" { + form["content-md5"] = contentMd5 + } + if sliceMd5 != "" { + form["slice-md5"] = sliceMd5 + } + joinTime(form, ctime, mtime) + var resp PrecreateResp + _, err := d.postUploadForm(ctx, "/youth/api/precreate", map[string]string{ + "bdstoken": d.bdstoken, + }, form, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *BaiduYouth) createFile(ctx context.Context, path, targetPath string, size int64, uploadID, uploadSign, blockList string, resp interface{}, mtime, ctime int64) ([]byte, error) { + form := map[string]string{ + "block_list": blockList, + "path": path, + "size": strconv.FormatInt(size, 10), + "target_path": targetPath, + "uploadid": uploadID, + } + if uploadSign != "" { + form["uploadsign"] = uploadSign + } + joinTime(form, ctime, mtime) + return d.postUploadForm(ctx, "/youth/api/create", map[string]string{ + "bdstoken": d.bdstoken, + "isdir": "0", + }, form, resp) +} + +func joinTime(form map[string]string, ctime, mtime int64) { + if ctime != 0 { + form["local_ctime"] = strconv.FormatInt(ctime, 10) + } + if mtime != 0 { + form["local_mtime"] = strconv.FormatInt(mtime, 10) + } +} + +func (d *BaiduYouth) uploadSlice(ctx context.Context, params map[string]string, fileName string, file io.Reader) error { + res, err := d.upClient.R(). + SetContext(ctx). + SetHeaders(d.commonHeaders()). + SetQueryParams(youthUploadQueryParams()). + SetQueryParams(params). + SetFileReader("file", fileName, file). + Post(d.UploadAPI + "/rest/2.0/pcs/superfile2") + if err != nil { + return err + } + body := res.Body() + errCode := utils.Json.Get(body, "error_code").ToInt() + errNo := utils.Json.Get(body, "errno").ToInt() + lower := strings.ToLower(string(body)) + if strings.Contains(lower, "uploadid") && (strings.Contains(lower, "invalid") || strings.Contains(lower, "expired") || strings.Contains(lower, "not found")) { + return ErrUploadIDExpired + } + if errCode != 0 || errNo != 0 { + msg := extractBaiduMessage(body) + if msg == "" { + msg = "error uploading to baidu youth" + } + return errs.NewErr(errs.StreamIncomplete, "%s: %s", msg, string(body)) + } + return nil +} + +func (d *BaiduYouth) uploadProgressKey() string { + if d.uk != 0 { + return strconv.FormatInt(d.uk, 10) + } + sum := md5.Sum([]byte(d.Cookie)) + return hex.EncodeToString(sum[:]) +} + +func DecryptMd5(encryptMd5 string) string { + if encryptMd5 == "" { + return "" + } + if _, err := hex.DecodeString(encryptMd5); err == nil { + return encryptMd5 + } + + var out strings.Builder + out.Grow(len(encryptMd5)) + for i, n := 0, int64(0); i < len(encryptMd5); i++ { + if i == 9 { + n = int64(unicode.ToLower(rune(encryptMd5[i])) - 'g') + } else { + n, _ = strconv.ParseInt(encryptMd5[i:i+1], 16, 64) + } + out.WriteString(strconv.FormatInt(n^int64(15&i), 16)) + } + + encryptMd5 = out.String() + return encryptMd5[8:16] + encryptMd5[:8] + encryptMd5[24:32] + encryptMd5[16:24] +} diff --git a/drivers/base/client.go b/drivers/base/client.go index bc08d6fb717..538c43a66c5 100644 --- a/drivers/base/client.go +++ b/drivers/base/client.go @@ -6,6 +6,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/net" "github.com/go-resty/resty/v2" ) @@ -26,24 +27,15 @@ func InitClient() { NoRedirectClient.SetHeader("user-agent", UserAgent) RestyClient = NewRestyClient() - HttpClient = NewHttpClient() + HttpClient = net.NewHttpClient() } func NewRestyClient() *resty.Client { client := resty.New(). SetHeader("user-agent", UserAgent). SetRetryCount(3). + SetRetryResetReaders(true). SetTimeout(DefaultTimeout). SetTLSClientConfig(&tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) return client } - -func NewHttpClient() *http.Client { - return &http.Client{ - Timeout: time.Hour * 48, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, - }, - } -} diff --git a/drivers/bitqiu/driver.go b/drivers/bitqiu/driver.go new file mode 100644 index 00000000000..048377fee93 --- /dev/null +++ b/drivers/bitqiu/driver.go @@ -0,0 +1,767 @@ +package bitqiu + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http/cookiejar" + "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + streamPkg "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" +) + +const ( + baseURL = "https://pan.bitqiu.com" + loginURL = baseURL + "/loginServer/login" + userInfoURL = baseURL + "/user/getInfo" + listURL = baseURL + "/apiToken/cfi/fs/resources/pages" + uploadInitializeURL = baseURL + "/apiToken/cfi/fs/upload/v2/initialize" + uploadCompleteURL = baseURL + "/apiToken/cfi/fs/upload/v2/complete" + downloadURL = baseURL + "/download/getUrl" + createDirURL = baseURL + "/resource/create" + moveResourceURL = baseURL + "/resource/remove" + renameResourceURL = baseURL + "/resource/rename" + copyResourceURL = baseURL + "/apiToken/cfi/fs/async/copy" + copyManagerURL = baseURL + "/apiToken/cfi/fs/async/manager" + deleteResourceURL = baseURL + "/resource/delete" + + successCode = "10200" + uploadSuccessCode = "30010" + copySubmittedCode = "10300" + orgChannel = "default|default|default" +) + +const ( + copyPollInterval = time.Second + copyPollMaxAttempts = 60 + chunkSize = int64(1 << 20) +) + +const defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36" + +type BitQiu struct { + model.Storage + Addition + + client *resty.Client + userID string +} + +func (d *BitQiu) Config() driver.Config { + return config +} + +func (d *BitQiu) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *BitQiu) Init(ctx context.Context) error { + if d.Addition.UserPlatform == "" { + d.Addition.UserPlatform = uuid.NewString() + op.MustSaveDriverStorage(d) + } + + if d.client == nil { + jar, err := cookiejar.New(nil) + if err != nil { + return err + } + d.client = base.NewRestyClient() + d.client.SetBaseURL(baseURL) + d.client.SetCookieJar(jar) + } + d.client.SetHeader("user-agent", d.userAgent()) + + return d.login(ctx) +} + +func (d *BitQiu) Drop(ctx context.Context) error { + d.client = nil + d.userID = "" + return nil +} + +func (d *BitQiu) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + parentID := d.resolveParentID(dir) + dirPath := "" + if dir != nil { + dirPath = dir.GetPath() + } + pageSize := d.pageSize() + orderType := d.orderType() + desc := d.orderDesc() + + var results []model.Obj + page := 1 + for { + form := map[string]string{ + "parentId": parentID, + "limit": strconv.Itoa(pageSize), + "orderType": orderType, + "desc": desc, + "model": "1", + "userId": d.userID, + "currentPage": strconv.Itoa(page), + "page": strconv.Itoa(page), + "org_channel": orgChannel, + } + var resp Response[ResourcePage] + if err := d.postForm(ctx, listURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != successCode { + if resp.Code == "10401" || resp.Code == "10404" { + if err := d.login(ctx); err != nil { + return nil, err + } + continue + } + return nil, fmt.Errorf("list failed: %s", resp.Message) + } + + objs, err := utils.SliceConvert(resp.Data.Data, func(item Resource) (model.Obj, error) { + return item.toObject(parentID, dirPath) + }) + if err != nil { + return nil, err + } + results = append(results, objs...) + + if !resp.Data.HasNext || len(resp.Data.Data) == 0 { + break + } + page++ + } + + return results, nil +} + +func (d *BitQiu) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + form := map[string]string{ + "fileIds": file.GetID(), + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[DownloadData] + if err := d.postForm(ctx, downloadURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + if resp.Data.URL == "" { + return nil, fmt.Errorf("empty download url returned") + } + return &model.Link{URL: resp.Data.URL}, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("get link failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("get link failed: retry limit reached") +} + +func (d *BitQiu) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + parentID := d.resolveParentID(parentDir) + parentPath := "" + if parentDir != nil { + parentPath = parentDir.GetPath() + } + form := map[string]string{ + "parentId": parentID, + "name": dirName, + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[CreateDirData] + if err := d.postForm(ctx, createDirURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + newParentID := parentID + if resp.Data.ParentID != "" { + newParentID = resp.Data.ParentID + } + name := resp.Data.Name + if name == "" { + name = dirName + } + resource := Resource{ + ResourceID: resp.Data.DirID, + ResourceType: 1, + Name: name, + ParentID: newParentID, + } + obj, err := resource.toObject(newParentID, parentPath) + if err != nil { + return nil, err + } + if o, ok := obj.(*Object); ok { + o.ParentID = newParentID + } + return obj, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("create folder failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("create folder failed: retry limit reached") +} + +func (d *BitQiu) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + targetParentID := d.resolveParentID(dstDir) + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "parentId": targetParentID, + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["dirIds"] = srcObj.GetID() + } else { + form["fileIds"] = srcObj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, moveResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + dstPath := "" + if dstDir != nil { + dstPath = dstDir.GetPath() + } + if setter, ok := srcObj.(model.SetPath); ok { + setter.SetPath(path.Join(dstPath, srcObj.GetName())) + } + if o, ok := srcObj.(*Object); ok { + o.ParentID = targetParentID + } + return srcObj, nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("move failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("move failed: retry limit reached") +} + +func (d *BitQiu) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + form := map[string]string{ + "resourceId": srcObj.GetID(), + "name": newName, + "type": "0", + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["type"] = "1" + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, renameResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode: + return updateObjectName(srcObj, newName), nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("rename failed: %s", resp.Message) + } + } + return nil, fmt.Errorf("rename failed: retry limit reached") +} + +func (d *BitQiu) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + targetParentID := d.resolveParentID(dstDir) + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "parentId": targetParentID, + "org_channel": orgChannel, + } + if srcObj.IsDir() { + form["dirIds"] = srcObj.GetID() + } else { + form["fileIds"] = srcObj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, copyResourceURL, form, &resp); err != nil { + return nil, err + } + switch resp.Code { + case successCode, copySubmittedCode: + return d.waitForCopiedObject(ctx, srcObj, dstDir) + case "10401", "10404": + if err := d.login(ctx); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("copy failed: %s", resp.Message) + } + } + + return nil, fmt.Errorf("copy failed: retry limit reached") +} + +func (d *BitQiu) Remove(ctx context.Context, obj model.Obj) error { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return err + } + } + + form := map[string]string{ + "dirIds": "", + "fileIds": "", + "org_channel": orgChannel, + } + if obj.IsDir() { + form["dirIds"] = obj.GetID() + } else { + form["fileIds"] = obj.GetID() + } + + for attempt := 0; attempt < 2; attempt++ { + var resp Response[any] + if err := d.postForm(ctx, deleteResourceURL, form, &resp); err != nil { + return err + } + switch resp.Code { + case successCode: + return nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return err + } + default: + return fmt.Errorf("remove failed: %s", resp.Message) + } + } + return fmt.Errorf("remove failed: retry limit reached") +} + +func (d *BitQiu) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.userID == "" { + if err := d.login(ctx); err != nil { + return nil, err + } + } + + up(0) + tmpFile, md5sum, err := streamPkg.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return nil, err + } + defer tmpFile.Close() + + parentID := d.resolveParentID(dstDir) + parentPath := "" + if dstDir != nil { + parentPath = dstDir.GetPath() + } + form := map[string]string{ + "parentId": parentID, + "name": file.GetName(), + "size": strconv.FormatInt(file.GetSize(), 10), + "hash": md5sum, + "sampleMd5": md5sum, + "org_channel": orgChannel, + } + var resp Response[json.RawMessage] + if err = d.postForm(ctx, uploadInitializeURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != uploadSuccessCode { + switch resp.Code { + case successCode: + var initData UploadInitData + if err := json.Unmarshal(resp.Data, &initData); err != nil { + return nil, fmt.Errorf("parse upload init response failed: %w", err) + } + serverCode, err := d.uploadFileInChunks(ctx, tmpFile, file.GetSize(), md5sum, initData, up) + if err != nil { + return nil, err + } + obj, err := d.completeChunkUpload(ctx, initData, parentID, parentPath, file.GetName(), file.GetSize(), md5sum, serverCode) + if err != nil { + return nil, err + } + up(100) + return obj, nil + default: + return nil, fmt.Errorf("upload failed: %s", resp.Message) + } + } + + var resource Resource + if err := json.Unmarshal(resp.Data, &resource); err != nil { + return nil, fmt.Errorf("parse upload response failed: %w", err) + } + obj, err := resource.toObject(parentID, parentPath) + if err != nil { + return nil, err + } + up(100) + return obj, nil +} + +func (d *BitQiu) uploadFileInChunks(ctx context.Context, tmpFile model.File, size int64, md5sum string, initData UploadInitData, up driver.UpdateProgress) (string, error) { + if d.client == nil { + return "", fmt.Errorf("client not initialized") + } + if size <= 0 { + return "", fmt.Errorf("invalid file size") + } + buf := make([]byte, chunkSize) + offset := int64(0) + var finishedFlag string + + for offset < size { + chunkLen := chunkSize + remaining := size - offset + if remaining < chunkLen { + chunkLen = remaining + } + + reader := io.NewSectionReader(tmpFile, offset, chunkLen) + chunkBuf := buf[:chunkLen] + if _, err := io.ReadFull(reader, chunkBuf); err != nil { + return "", fmt.Errorf("read chunk failed: %w", err) + } + + headers := map[string]string{ + "accept": "*/*", + "content-type": "application/octet-stream", + "appid": initData.AppID, + "token": initData.Token, + "userid": strconv.FormatInt(initData.UserID, 10), + "serialnumber": initData.SerialNumber, + "hash": md5sum, + "len": strconv.FormatInt(chunkLen, 10), + "offset": strconv.FormatInt(offset, 10), + "user-agent": d.userAgent(), + } + + var chunkResp ChunkUploadResponse + req := d.client.R(). + SetContext(ctx). + SetHeaders(headers). + SetBody(chunkBuf). + SetResult(&chunkResp) + + if _, err := req.Post(initData.UploadURL); err != nil { + return "", err + } + if chunkResp.ErrCode != 0 { + return "", fmt.Errorf("chunk upload failed with code %d", chunkResp.ErrCode) + } + finishedFlag = chunkResp.FinishedFlag + offset += chunkLen + up(float64(offset) * 100 / float64(size)) + } + + if finishedFlag == "" { + return "", fmt.Errorf("upload finished without server code") + } + return finishedFlag, nil +} + +func (d *BitQiu) completeChunkUpload(ctx context.Context, initData UploadInitData, parentID, parentPath, name string, size int64, md5sum, serverCode string) (model.Obj, error) { + form := map[string]string{ + "currentPage": "1", + "limit": "1", + "userId": strconv.FormatInt(initData.UserID, 10), + "status": "0", + "parentId": parentID, + "name": name, + "fileUid": initData.FileUID, + "fileSid": initData.FileSID, + "size": strconv.FormatInt(size, 10), + "serverCode": serverCode, + "snapTime": "", + "hash": md5sum, + "sampleMd5": md5sum, + "org_channel": orgChannel, + } + + var resp Response[Resource] + if err := d.postForm(ctx, uploadCompleteURL, form, &resp); err != nil { + return nil, err + } + if resp.Code != successCode { + return nil, fmt.Errorf("complete upload failed: %s", resp.Message) + } + + return resp.Data.toObject(parentID, parentPath) +} + +func (d *BitQiu) login(ctx context.Context) error { + if d.client == nil { + return fmt.Errorf("client not initialized") + } + + form := map[string]string{ + "passport": d.Username, + "password": utils.GetMD5EncodeStr(d.Password), + "remember": "0", + "captcha": "", + "org_channel": orgChannel, + } + var resp Response[LoginData] + if err := d.postForm(ctx, loginURL, form, &resp); err != nil { + return err + } + if resp.Code != successCode { + return fmt.Errorf("login failed: %s", resp.Message) + } + d.userID = strconv.FormatInt(resp.Data.UserID, 10) + return d.ensureRootFolderID(ctx) +} + +func (d *BitQiu) ensureRootFolderID(ctx context.Context) error { + rootID := d.Addition.GetRootId() + if rootID != "" && rootID != "0" { + return nil + } + + form := map[string]string{ + "org_channel": orgChannel, + } + var resp Response[UserInfoData] + if err := d.postForm(ctx, userInfoURL, form, &resp); err != nil { + return err + } + if resp.Code != successCode { + return fmt.Errorf("get user info failed: %s", resp.Message) + } + if resp.Data.RootDirID == "" { + return fmt.Errorf("get user info failed: empty root dir id") + } + if d.Addition.RootFolderID != resp.Data.RootDirID { + d.Addition.RootFolderID = resp.Data.RootDirID + op.MustSaveDriverStorage(d) + } + return nil +} + +func (d *BitQiu) postForm(ctx context.Context, url string, form map[string]string, result interface{}) error { + if d.client == nil { + return fmt.Errorf("client not initialized") + } + req := d.client.R(). + SetContext(ctx). + SetHeaders(d.commonHeaders()). + SetFormData(form) + if result != nil { + req = req.SetResult(result) + } + _, err := req.Post(url) + return err +} + +func (d *BitQiu) waitForCopiedObject(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + expectedName := srcObj.GetName() + expectedIsDir := srcObj.IsDir() + var lastListErr error + + for attempt := 0; attempt < copyPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, copyPollInterval); err != nil { + return nil, err + } + } + + if err := d.checkCopyFailure(ctx); err != nil { + return nil, err + } + + obj, err := d.findObjectInDir(ctx, dstDir, expectedName, expectedIsDir) + if err != nil { + lastListErr = err + continue + } + if obj != nil { + return obj, nil + } + } + if lastListErr != nil { + return nil, lastListErr + } + return nil, fmt.Errorf("copy task timed out waiting for completion") +} + +func (d *BitQiu) checkCopyFailure(ctx context.Context) error { + form := map[string]string{ + "org_channel": orgChannel, + } + for attempt := 0; attempt < 2; attempt++ { + var resp Response[AsyncManagerData] + if err := d.postForm(ctx, copyManagerURL, form, &resp); err != nil { + return err + } + switch resp.Code { + case successCode: + if len(resp.Data.FailTasks) > 0 { + return fmt.Errorf("copy failed: %s", resp.Data.FailTasks[0].ErrorMessage()) + } + return nil + case "10401", "10404": + if err := d.login(ctx); err != nil { + return err + } + default: + return fmt.Errorf("query copy status failed: %s", resp.Message) + } + } + return fmt.Errorf("query copy status failed: retry limit reached") +} + +func (d *BitQiu) findObjectInDir(ctx context.Context, dir model.Obj, name string, isDir bool) (model.Obj, error) { + objs, err := d.List(ctx, dir, model.ListArgs{}) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name && obj.IsDir() == isDir { + return obj, nil + } + } + return nil, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (d *BitQiu) commonHeaders() map[string]string { + headers := map[string]string{ + "accept": "application/json, text/plain, */*", + "accept-language": "en-US,en;q=0.9", + "cache-control": "no-cache", + "pragma": "no-cache", + "user-platform": d.Addition.UserPlatform, + "x-kl-saas-ajax-request": "Ajax_Request", + "x-requested-with": "XMLHttpRequest", + "referer": baseURL + "/", + "origin": baseURL, + "user-agent": d.userAgent(), + } + return headers +} + +func (d *BitQiu) userAgent() string { + if ua := strings.TrimSpace(d.Addition.UserAgent); ua != "" { + return ua + } + return defaultUserAgent +} + +func (d *BitQiu) resolveParentID(dir model.Obj) string { + if dir != nil && dir.GetID() != "" { + return dir.GetID() + } + if root := d.Addition.GetRootId(); root != "" { + return root + } + return config.DefaultRoot +} + +func (d *BitQiu) pageSize() int { + if size, err := strconv.Atoi(d.Addition.PageSize); err == nil && size > 0 { + return size + } + return 24 +} + +func (d *BitQiu) orderType() string { + if d.Addition.OrderType != "" { + return d.Addition.OrderType + } + return "updateTime" +} + +func (d *BitQiu) orderDesc() string { + if d.Addition.OrderDesc { + return "1" + } + return "0" +} + +var _ driver.Driver = (*BitQiu)(nil) +var _ driver.PutResult = (*BitQiu)(nil) diff --git a/drivers/bitqiu/meta.go b/drivers/bitqiu/meta.go new file mode 100644 index 00000000000..63cb03344c4 --- /dev/null +++ b/drivers/bitqiu/meta.go @@ -0,0 +1,28 @@ +package bitqiu + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + UserPlatform string `json:"user_platform" help:"Optional device identifier; auto-generated if empty."` + OrderType string `json:"order_type" type:"select" options:"updateTime,createTime,name,size" default:"updateTime"` + OrderDesc bool `json:"order_desc"` + PageSize string `json:"page_size" default:"24" help:"Number of entries to request per page."` + UserAgent string `json:"user_agent" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36"` +} + +var config = driver.Config{ + Name: "BitQiu", + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &BitQiu{} + }) +} diff --git a/drivers/bitqiu/types.go b/drivers/bitqiu/types.go new file mode 100644 index 00000000000..8fbec989135 --- /dev/null +++ b/drivers/bitqiu/types.go @@ -0,0 +1,107 @@ +package bitqiu + +import "encoding/json" + +type Response[T any] struct { + Code string `json:"code"` + Message string `json:"message"` + Data T `json:"data"` +} + +type LoginData struct { + UserID int64 `json:"userId"` +} + +type ResourcePage struct { + CurrentPage int `json:"currentPage"` + PageSize int `json:"pageSize"` + TotalCount int `json:"totalCount"` + TotalPageCount int `json:"totalPageCount"` + Data []Resource `json:"data"` + HasNext bool `json:"hasNext"` +} + +type Resource struct { + ResourceID string `json:"resourceId"` + ResourceUID string `json:"resourceUid"` + ResourceType int `json:"resourceType"` + ParentID string `json:"parentId"` + Name string `json:"name"` + ExtName string `json:"extName"` + Size *json.Number `json:"size"` + CreateTime *string `json:"createTime"` + UpdateTime *string `json:"updateTime"` + FileMD5 string `json:"fileMd5"` +} + +type DownloadData struct { + URL string `json:"url"` + MD5 string `json:"md5"` + Size int64 `json:"size"` +} + +type UserInfoData struct { + RootDirID string `json:"rootDirId"` +} + +type CreateDirData struct { + DirID string `json:"dirId"` + Name string `json:"name"` + ParentID string `json:"parentId"` +} + +type AsyncManagerData struct { + WaitTasks []AsyncTask `json:"waitTaskList"` + RunningTasks []AsyncTask `json:"runningTaskList"` + SuccessTasks []AsyncTask `json:"successTaskList"` + FailTasks []AsyncTask `json:"failTaskList"` + TaskList []AsyncTask `json:"taskList"` +} + +type AsyncTask struct { + TaskID string `json:"taskId"` + Status int `json:"status"` + ErrorMsg string `json:"errorMsg"` + Message string `json:"message"` + Result *AsyncTaskInfo `json:"result"` + TargetName string `json:"targetName"` + TargetDirID string `json:"parentId"` +} + +type AsyncTaskInfo struct { + Resource Resource `json:"resource"` + DirID string `json:"dirId"` + FileID string `json:"fileId"` + Name string `json:"name"` + ParentID string `json:"parentId"` +} + +func (t AsyncTask) ErrorMessage() string { + if t.ErrorMsg != "" { + return t.ErrorMsg + } + if t.Message != "" { + return t.Message + } + return "unknown error" +} + +type UploadInitData struct { + Name string `json:"name"` + Size int64 `json:"size"` + Token string `json:"token"` + FileUID string `json:"fileUid"` + FileSID string `json:"fileSid"` + ParentID string `json:"parentId"` + UserID int64 `json:"userId"` + SerialNumber string `json:"serialNumber"` + UploadURL string `json:"uploadUrl"` + AppID string `json:"appId"` +} + +type ChunkUploadResponse struct { + ErrCode int `json:"errCode"` + Offset int64 `json:"offset"` + Finished int `json:"finished"` + FinishedFlag string `json:"finishedFlag"` +} diff --git a/drivers/bitqiu/util.go b/drivers/bitqiu/util.go new file mode 100644 index 00000000000..bccd6815e9b --- /dev/null +++ b/drivers/bitqiu/util.go @@ -0,0 +1,102 @@ +package bitqiu + +import ( + "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Object struct { + model.Object + ParentID string +} + +func (r Resource) toObject(parentID, parentPath string) (model.Obj, error) { + id := r.ResourceID + if id == "" { + id = r.ResourceUID + } + obj := &Object{ + Object: model.Object{ + ID: id, + Name: r.Name, + IsFolder: r.ResourceType == 1, + }, + ParentID: parentID, + } + if r.Size != nil { + if size, err := (*r.Size).Int64(); err == nil { + obj.Size = size + } + } + if ct := parseBitQiuTime(r.CreateTime); !ct.IsZero() { + obj.Ctime = ct + } + if mt := parseBitQiuTime(r.UpdateTime); !mt.IsZero() { + obj.Modified = mt + } + if r.FileMD5 != "" { + obj.HashInfo = utils.NewHashInfo(utils.MD5, strings.ToLower(r.FileMD5)) + } + obj.SetPath(path.Join(parentPath, obj.Name)) + return obj, nil +} + +func parseBitQiuTime(value *string) time.Time { + if value == nil { + return time.Time{} + } + trimmed := strings.TrimSpace(*value) + if trimmed == "" { + return time.Time{} + } + if ts, err := time.ParseInLocation("2006-01-02 15:04:05", trimmed, time.Local); err == nil { + return ts + } + return time.Time{} +} + +func updateObjectName(obj model.Obj, newName string) model.Obj { + newPath := path.Join(parentPathOf(obj.GetPath()), newName) + + switch o := obj.(type) { + case *Object: + o.Name = newName + o.Object.Name = newName + o.SetPath(newPath) + return o + case *model.Object: + o.Name = newName + o.SetPath(newPath) + return o + } + + if setter, ok := obj.(model.SetPath); ok { + setter.SetPath(newPath) + } + + return &model.Object{ + ID: obj.GetID(), + Path: newPath, + Name: newName, + Size: obj.GetSize(), + Modified: obj.ModTime(), + Ctime: obj.CreateTime(), + IsFolder: obj.IsDir(), + HashInfo: obj.GetHash(), + } +} + +func parentPathOf(p string) string { + if p == "" { + return "" + } + dir := path.Dir(p) + if dir == "." { + return "" + } + return dir +} diff --git a/drivers/chaoxing/driver.go b/drivers/chaoxing/driver.go new file mode 100644 index 00000000000..bf01a83b732 --- /dev/null +++ b/drivers/chaoxing/driver.go @@ -0,0 +1,306 @@ +package chaoxing + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "google.golang.org/appengine/log" +) + +type ChaoXing struct { + model.Storage + Addition + cron *cron.Cron + config driver.Config + conf Conf +} + +func (d *ChaoXing) Config() driver.Config { + return d.config +} + +func (d *ChaoXing) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ChaoXing) refreshCookie() error { + cookie, err := d.Login() + if err != nil { + d.Status = err.Error() + op.MustSaveDriverStorage(d) + return nil + } + d.Addition.Cookie = cookie + op.MustSaveDriverStorage(d) + return nil +} + +func (d *ChaoXing) Init(ctx context.Context) error { + err := d.refreshCookie() + if err != nil { + log.Errorf(ctx, err.Error()) + } + d.cron = cron.NewCron(time.Hour * 12) + d.cron.Do(func() { + err = d.refreshCookie() + if err != nil { + log.Errorf(ctx, err.Error()) + } + }) + return nil +} + +func (d *ChaoXing) Drop(ctx context.Context) error { + if d.cron != nil { + d.cron.Stop() + } + return nil +} + +func (d *ChaoXing) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.GetFiles(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *ChaoXing) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var resp DownResp + ua := d.conf.ua + fileId := strings.Split(file.GetID(), "$")[1] + _, err := d.requestDownload("/screen/note_note/files/status/"+fileId, http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua) + }, &resp) + if err != nil { + return nil, err + } + u := resp.Download + return &model.Link{ + URL: u, + Header: http.Header{ + "Cookie": []string{d.Cookie}, + "Referer": []string{d.conf.referer}, + "User-Agent": []string{ua}, + }, + Concurrency: 2, + PartSize: 10 * utils.MB, + }, nil +} + +func (d *ChaoXing) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "name": dirName, + "pid": parentDir.GetID(), + } + var resp ListFileResp + _, err := d.request("/pc/resource/addResourceFolder", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return err + } + if resp.Result != 1 { + msg := fmt.Sprintf("error:%s", resp.Msg) + return errors.New(msg) + } + return nil +} + +func (d *ChaoXing) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderIds": srcObj.GetID(), + "targetId": dstDir.GetID(), + } + if !srcObj.IsDir() { + query = map[string]string{ + "bbsid": d.Addition.Bbsid, + "recIds": strings.Split(srcObj.GetID(), "$")[0], + "targetId": dstDir.GetID(), + } + } + var resp ListFileResp + _, err := d.request("/pc/resource/moveResource", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return err + } + if !resp.Status { + msg := fmt.Sprintf("error:%s", resp.Msg) + return errors.New(msg) + } + return nil +} + +func (d *ChaoXing) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderId": srcObj.GetID(), + "name": newName, + } + path := "/pc/resource/updateResourceFolderName" + if !srcObj.IsDir() { + // path = "/pc/resource/updateResourceFileName" + // query = map[string]string{ + // "bbsid": d.Addition.Bbsid, + // "recIds": strings.Split(srcObj.GetID(), "$")[0], + // "name": newName, + // } + return errors.New("此网盘不支持修改文件名") + } + var resp ListFileResp + _, err := d.request(path, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return err + } + if resp.Result != 1 { + msg := fmt.Sprintf("error:%s", resp.Msg) + return errors.New(msg) + } + return nil +} + +func (d *ChaoXing) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + // TODO copy obj, optional + return errs.NotImplement +} + +func (d *ChaoXing) Remove(ctx context.Context, obj model.Obj) error { + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderIds": obj.GetID(), + } + path := "/pc/resource/deleteResourceFolder" + var resp ListFileResp + if !obj.IsDir() { + path = "/pc/resource/deleteResourceFile" + query = map[string]string{ + "bbsid": d.Addition.Bbsid, + "recIds": strings.Split(obj.GetID(), "$")[0], + } + } + _, err := d.request(path, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return err + } + if resp.Result != 1 { + msg := fmt.Sprintf("error:%s", resp.Msg) + return errors.New(msg) + } + return nil +} + +func (d *ChaoXing) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + var resp UploadDataRsp + _, err := d.request("https://noteyd.chaoxing.com/pc/files/getUploadConfig", http.MethodGet, func(req *resty.Request) { + }, &resp) + if err != nil { + return err + } + if resp.Result != 1 { + return errors.New("get upload data error") + } + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + filePart, err := writer.CreateFormFile("file", file.GetName()) + if err != nil { + return err + } + _, err = utils.CopyWithBuffer(filePart, file) + if err != nil { + return err + } + err = writer.WriteField("_token", resp.Msg.Token) + if err != nil { + return err + } + err = writer.WriteField("puid", fmt.Sprintf("%d", resp.Msg.Puid)) + if err != nil { + fmt.Println("Error writing param2 to request body:", err) + return err + } + err = writer.Close() + if err != nil { + return err + } + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: body, + Size: int64(body.Len()), + }, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, "POST", "https://pan-yz.chaoxing.com/upload", r) + if err != nil { + return err + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len())) + resps, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resps.Body.Close() + bodys, err := io.ReadAll(resps.Body) + if err != nil { + return err + } + var fileRsp UploadFileDataRsp + err = json.Unmarshal(bodys, &fileRsp) + if err != nil { + return err + } + if fileRsp.Msg != "success" { + return errors.New(fileRsp.Msg) + } + uploadDoneParam := UploadDoneParam{Key: fileRsp.ObjectID, Cataid: "100000019", Param: fileRsp.Data} + params, err := json.Marshal(uploadDoneParam) + if err != nil { + return err + } + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "pid": dstDir.GetID(), + "type": "yunpan", + "params": url.QueryEscape("[" + string(params) + "]"), + } + var respd ListFileResp + _, err = d.request("/pc/resource/addResource", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &respd) + if err != nil { + return err + } + if respd.Result != 1 { + msg := fmt.Sprintf("error:%v", resp.Msg) + return errors.New(msg) + } + return nil +} + +var _ driver.Driver = (*ChaoXing)(nil) diff --git a/drivers/chaoxing/meta.go b/drivers/chaoxing/meta.go new file mode 100644 index 00000000000..c0500629cf3 --- /dev/null +++ b/drivers/chaoxing/meta.go @@ -0,0 +1,47 @@ +package chaoxing + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +// 此程序挂载的是超星小组网盘,需要代理才能使用; +// 登录超星后进入个人空间,进入小组,新建小组,点击进去。 +// url中就有bbsid的参数,系统限制单文件大小2G,没有总容量限制 +type Addition struct { + // 超星用户名及密码 + UserName string `json:"user_name" required:"true"` + Password string `json:"password" required:"true"` + // 从自己新建的小组url里获取 + Bbsid string `json:"bbsid" required:"true"` + driver.RootID + // 可不填,程序会自动登录获取 + Cookie string `json:"cookie"` +} + +type Conf struct { + ua string + referer string + api string + DowloadApi string +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ChaoXing{ + config: driver.Config{ + Name: "ChaoXingGroupDrive", + OnlyProxy: true, + OnlyLocal: false, + DefaultRoot: "-1", + NoOverwriteUpload: true, + }, + conf: Conf{ + ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch", + referer: "https://chaoxing.com/", + api: "https://groupweb.chaoxing.com", + DowloadApi: "https://noteyd.chaoxing.com", + }, + } + }) +} diff --git a/drivers/chaoxing/types.go b/drivers/chaoxing/types.go new file mode 100644 index 00000000000..71a59e15be6 --- /dev/null +++ b/drivers/chaoxing/types.go @@ -0,0 +1,276 @@ +package chaoxing + +import ( + "bytes" + "fmt" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Resp struct { + Result int `json:"result"` +} + +type UserAuth struct { + GroupAuth struct { + AddData int `json:"addData"` + AddDataFolder int `json:"addDataFolder"` + AddLebel int `json:"addLebel"` + AddManager int `json:"addManager"` + AddMem int `json:"addMem"` + AddTopicFolder int `json:"addTopicFolder"` + AnonymousAddReply int `json:"anonymousAddReply"` + AnonymousAddTopic int `json:"anonymousAddTopic"` + BatchOperation int `json:"batchOperation"` + DelData int `json:"delData"` + DelDataFolder int `json:"delDataFolder"` + DelMem int `json:"delMem"` + DelTopicFolder int `json:"delTopicFolder"` + Dismiss int `json:"dismiss"` + ExamEnc string `json:"examEnc"` + GroupChat int `json:"groupChat"` + IsShowCircleChatButton int `json:"isShowCircleChatButton"` + IsShowCircleCloudButton int `json:"isShowCircleCloudButton"` + IsShowCompanyButton int `json:"isShowCompanyButton"` + Join int `json:"join"` + MemberShowRankSet int `json:"memberShowRankSet"` + ModifyDataFolder int `json:"modifyDataFolder"` + ModifyExpose int `json:"modifyExpose"` + ModifyName int `json:"modifyName"` + ModifyShowPic int `json:"modifyShowPic"` + ModifyTopicFolder int `json:"modifyTopicFolder"` + ModifyVisibleState int `json:"modifyVisibleState"` + OnlyMgrScoreSet int `json:"onlyMgrScoreSet"` + Quit int `json:"quit"` + SendNotice int `json:"sendNotice"` + ShowActivityManage int `json:"showActivityManage"` + ShowActivitySet int `json:"showActivitySet"` + ShowAttentionSet int `json:"showAttentionSet"` + ShowAutoClearStatus int `json:"showAutoClearStatus"` + ShowBarcode int `json:"showBarcode"` + ShowChatRoomSet int `json:"showChatRoomSet"` + ShowCircleActivitySet int `json:"showCircleActivitySet"` + ShowCircleSet int `json:"showCircleSet"` + ShowCmem int `json:"showCmem"` + ShowDataFolder int `json:"showDataFolder"` + ShowDelReason int `json:"showDelReason"` + ShowForward int `json:"showForward"` + ShowGroupChat int `json:"showGroupChat"` + ShowGroupChatSet int `json:"showGroupChatSet"` + ShowGroupSquareSet int `json:"showGroupSquareSet"` + ShowLockAddSet int `json:"showLockAddSet"` + ShowManager int `json:"showManager"` + ShowManagerIdentitySet int `json:"showManagerIdentitySet"` + ShowNeedDelReasonSet int `json:"showNeedDelReasonSet"` + ShowNotice int `json:"showNotice"` + ShowOnlyManagerReplySet int `json:"showOnlyManagerReplySet"` + ShowRank int `json:"showRank"` + ShowRank2 int `json:"showRank2"` + ShowRecycleBin int `json:"showRecycleBin"` + ShowReplyByClass int `json:"showReplyByClass"` + ShowReplyNeedCheck int `json:"showReplyNeedCheck"` + ShowSignbanSet int `json:"showSignbanSet"` + ShowSpeechSet int `json:"showSpeechSet"` + ShowTopicCheck int `json:"showTopicCheck"` + ShowTopicNeedCheck int `json:"showTopicNeedCheck"` + ShowTransferSet int `json:"showTransferSet"` + } `json:"groupAuth"` + OperationAuth struct { + Add int `json:"add"` + AddTopicToFolder int `json:"addTopicToFolder"` + ChoiceSet int `json:"choiceSet"` + DelTopicFromFolder int `json:"delTopicFromFolder"` + Delete int `json:"delete"` + Reply int `json:"reply"` + ScoreSet int `json:"scoreSet"` + TopSet int `json:"topSet"` + Update int `json:"update"` + } `json:"operationAuth"` +} + +// 手机端学习通上传的文件的json内容(content字段)与网页端上传的有所不同 +// 网页端json `"puid": 54321, "size": 12345` +// 手机端json `"puid": "54321". "size": "12345"` +type int_str int + +// json 字符串数字和纯数字解析 +func (ios *int_str) UnmarshalJSON(data []byte) error { + intValue, err := strconv.Atoi(string(bytes.Trim(data, "\""))) + if err != nil { + return err + } + *ios = int_str(intValue) + return nil +} + +type File struct { + Cataid int `json:"cataid"` + Cfid int `json:"cfid"` + Content struct { + Cfid int `json:"cfid"` + Pid int `json:"pid"` + FolderName string `json:"folderName"` + ShareType int `json:"shareType"` + Preview string `json:"preview"` + Filetype string `json:"filetype"` + PreviewURL string `json:"previewUrl"` + IsImg bool `json:"isImg"` + ParentPath string `json:"parentPath"` + Icon string `json:"icon"` + Suffix string `json:"suffix"` + Duration int `json:"duration"` + Pantype string `json:"pantype"` + Puid int_str `json:"puid"` + Filepath string `json:"filepath"` + Crc string `json:"crc"` + Isfile bool `json:"isfile"` + Residstr string `json:"residstr"` + ObjectID string `json:"objectId"` + Extinfo string `json:"extinfo"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + ResTypeValue int `json:"resTypeValue"` + UploadDateFormat string `json:"uploadDateFormat"` + DisableOpt bool `json:"disableOpt"` + DownPath string `json:"downPath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + Restype string `json:"restype"` + Size int_str `json:"size"` + UploadDate int64 `json:"uploadDate"` + FileSize string `json:"fileSize"` + Name string `json:"name"` + FileID string `json:"fileId"` + } `json:"content"` + CreatorID int `json:"creatorId"` + DesID string `json:"des_id"` + ID int `json:"id"` + Inserttime int64 `json:"inserttime"` + Key string `json:"key"` + Norder int `json:"norder"` + OwnerID int `json:"ownerId"` + OwnerType int `json:"ownerType"` + Path string `json:"path"` + Rid int `json:"rid"` + Status int `json:"status"` + Topsign int `json:"topsign"` +} + +type ListFileResp struct { + Msg string `json:"msg"` + Result int `json:"result"` + Status bool `json:"status"` + UserAuth UserAuth `json:"userAuth"` + List []File `json:"list"` +} + +type DownResp struct { + Msg string `json:"msg"` + Duration int `json:"duration"` + Download string `json:"download"` + FileStatus string `json:"fileStatus"` + URL string `json:"url"` + Status bool `json:"status"` +} + +type UploadDataRsp struct { + Result int `json:"result"` + Msg struct { + Puid int `json:"puid"` + Token string `json:"token"` + } `json:"msg"` +} + +type UploadFileDataRsp struct { + Result bool `json:"result"` + Msg string `json:"msg"` + Crc string `json:"crc"` + ObjectID string `json:"objectId"` + Resid int64 `json:"resid"` + Puid int `json:"puid"` + Data struct { + DisableOpt bool `json:"disableOpt"` + Resid int64 `json:"resid"` + Crc string `json:"crc"` + Puid int `json:"puid"` + Isfile bool `json:"isfile"` + Pantype string `json:"pantype"` + Size int `json:"size"` + Name string `json:"name"` + ObjectID string `json:"objectId"` + Restype string `json:"restype"` + UploadDate int64 `json:"uploadDate"` + ModifyDate int64 `json:"modifyDate"` + UploadDateFormat string `json:"uploadDateFormat"` + Residstr string `json:"residstr"` + Suffix string `json:"suffix"` + Preview string `json:"preview"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + Duration int `json:"duration"` + IsImg bool `json:"isImg"` + PreviewURL string `json:"previewUrl"` + Filetype string `json:"filetype"` + Filepath string `json:"filepath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + ResTypeValue int `json:"resTypeValue"` + Extinfo string `json:"extinfo"` + } `json:"data"` +} + +type UploadDoneParam struct { + Cataid string `json:"cataid"` + Key string `json:"key"` + Param struct { + DisableOpt bool `json:"disableOpt"` + Resid int64 `json:"resid"` + Crc string `json:"crc"` + Puid int `json:"puid"` + Isfile bool `json:"isfile"` + Pantype string `json:"pantype"` + Size int `json:"size"` + Name string `json:"name"` + ObjectID string `json:"objectId"` + Restype string `json:"restype"` + UploadDate int64 `json:"uploadDate"` + ModifyDate int64 `json:"modifyDate"` + UploadDateFormat string `json:"uploadDateFormat"` + Residstr string `json:"residstr"` + Suffix string `json:"suffix"` + Preview string `json:"preview"` + Thumbnail string `json:"thumbnail"` + Creator int `json:"creator"` + Duration int `json:"duration"` + IsImg bool `json:"isImg"` + PreviewURL string `json:"previewUrl"` + Filetype string `json:"filetype"` + Filepath string `json:"filepath"` + Sort int `json:"sort"` + Topsort int `json:"topsort"` + ResTypeValue int `json:"resTypeValue"` + Extinfo string `json:"extinfo"` + } `json:"param"` +} + +func fileToObj(f File) *model.Object { + if len(f.Content.FolderName) > 0 { + return &model.Object{ + ID: fmt.Sprintf("%d", f.ID), + Name: f.Content.FolderName, + Size: 0, + Modified: time.UnixMilli(f.Inserttime), + IsFolder: true, + } + } + paserTime := time.UnixMilli(f.Content.UploadDate) + return &model.Object{ + ID: fmt.Sprintf("%d$%s", f.ID, f.Content.FileID), + Name: f.Content.Name, + Size: int64(f.Content.Size), + Modified: paserTime, + IsFolder: false, + } +} diff --git a/drivers/chaoxing/util.go b/drivers/chaoxing/util.go new file mode 100644 index 00000000000..b6725804e6c --- /dev/null +++ b/drivers/chaoxing/util.go @@ -0,0 +1,183 @@ +package chaoxing + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "errors" + "fmt" + "mime/multipart" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" +) + +func (d *ChaoXing) requestDownload(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + u := d.conf.DowloadApi + pathname + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "Accept": "application/json, text/plain, */*", + "Referer": d.conf.referer, + }) + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Resp + req.SetError(&e) + res, err := req.Execute(method, u) + if err != nil { + return nil, err + } + return res.Body(), nil +} + +func (d *ChaoXing) request(pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + u := d.conf.api + pathname + if strings.Contains(pathname, "getUploadConfig") { + u = pathname + } + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "Accept": "application/json, text/plain, */*", + "Referer": d.conf.referer, + }) + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Resp + req.SetError(&e) + res, err := req.Execute(method, u) + if err != nil { + return nil, err + } + return res.Body(), nil +} + +func (d *ChaoXing) GetFiles(parent string) ([]File, error) { + files := make([]File, 0) + query := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderId": parent, + "recType": "1", + } + var resp ListFileResp + _, err := d.request("/pc/resource/getResourceList", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return nil, err + } + if resp.Result != 1 { + msg := fmt.Sprintf("error code is:%d", resp.Result) + return nil, errors.New(msg) + } + if len(resp.List) > 0 { + files = append(files, resp.List...) + } + querys := map[string]string{ + "bbsid": d.Addition.Bbsid, + "folderId": parent, + "recType": "2", + } + var resps ListFileResp + _, err = d.request("/pc/resource/getResourceList", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(querys) + }, &resps) + if err != nil { + return nil, err + } + for _, file := range resps.List { + // 手机端超星上传的文件没有fileID字段,但ObjectID与fileID相同,可代替 + if file.Content.FileID == "" { + file.Content.FileID = file.Content.ObjectID + } + files = append(files, file) + } + return files, nil +} + +func EncryptByAES(message, key string) (string, error) { + aesKey := []byte(key) + plainText := []byte(message) + block, err := aes.NewCipher(aesKey) + if err != nil { + return "", err + } + iv := aesKey[:aes.BlockSize] + mode := cipher.NewCBCEncrypter(block, iv) + padding := aes.BlockSize - len(plainText)%aes.BlockSize + paddedText := append(plainText, byte(padding)) + for i := 0; i < padding-1; i++ { + paddedText = append(paddedText, byte(padding)) + } + ciphertext := make([]byte, len(paddedText)) + mode.CryptBlocks(ciphertext, paddedText) + encrypted := base64.StdEncoding.EncodeToString(ciphertext) + return encrypted, nil +} + +func CookiesToString(cookies []*http.Cookie) string { + var cookieStr string + for _, cookie := range cookies { + cookieStr += cookie.Name + "=" + cookie.Value + "; " + } + if len(cookieStr) > 2 { + cookieStr = cookieStr[:len(cookieStr)-2] + } + return cookieStr +} + +func (d *ChaoXing) Login() (string, error) { + transferKey := "u2oh6Vu^HWe4_AES" + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + uname, err := EncryptByAES(d.Addition.UserName, transferKey) + if err != nil { + return "", err + } + password, err := EncryptByAES(d.Addition.Password, transferKey) + if err != nil { + return "", err + } + err = writer.WriteField("uname", uname) + if err != nil { + return "", err + } + err = writer.WriteField("password", password) + if err != nil { + return "", err + } + err = writer.WriteField("t", "true") + if err != nil { + return "", err + } + err = writer.Close() + if err != nil { + return "", err + } + // Create the request + req, err := http.NewRequest("POST", "https://passport2.chaoxing.com/fanyalogin", body) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Content-Length", fmt.Sprintf("%d", body.Len())) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + return CookiesToString(resp.Cookies()), nil + +} diff --git a/drivers/chunker/driver.go b/drivers/chunker/driver.go new file mode 100644 index 00000000000..f05d4a9f89a --- /dev/null +++ b/drivers/chunker/driver.go @@ -0,0 +1,459 @@ +package chunker + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "hash" + "io" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Chunker) Config() driver.Config { + return config +} + +func (d *Chunker) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Chunker) Init(ctx context.Context) error { + if d.ChunkSize == 0 { + d.ChunkSize = defaultChunkSize + } + if d.StartFrom == 0 { + d.StartFrom = defaultStartFrom + } + d.NameFormat = utils.GetNoneEmpty(d.NameFormat, defaultChunkNameFmt) + d.MetaFormat = utils.GetNoneEmpty(d.MetaFormat, defaultMetaFormat) + d.HashType = utils.GetNoneEmpty(d.HashType, defaultHashType) + + if err := d.setChunkNameFormat(d.NameFormat); err != nil { + return fmt.Errorf("invalid name_format: %w", err) + } + if err := d.validateOptions(); err != nil { + return err + } + + targetPaths := d.configuredRemotePaths() + d.remoteTargets = make([]remoteTarget, 0, len(targetPaths)) + for _, targetPath := range targetPaths { + storage, err := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if err != nil { + return fmt.Errorf("can't find remote storage %q: %w", targetPath, err) + } + d.remoteTargets = append(d.remoteTargets, remoteTarget{ + MountPath: targetPath, + Storage: storage, + }) + } + if len(d.remoteTargets) == 0 { + return fmt.Errorf("can't find remote storage: %w", errs.ObjectNotFound) + } + d.remoteStorage = d.remoteTargets[0].Storage + return nil +} + +func (d *Chunker) Drop(ctx context.Context) error { + d.remoteStorage = nil + d.remoteTargets = nil + return nil +} + +func (d *Chunker) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.listDirObjects(ctx, dir.GetPath(), args.Refresh) +} + +func (d *Chunker) Get(ctx context.Context, pathStr string) (model.Obj, error) { + if utils.PathEqual(pathStr, "/") { + return &model.Object{ + Name: "Root", + Path: "/", + IsFolder: true, + }, nil + } + parent, name := path.Split(utils.FixAndCleanPath(pathStr)) + if parent == "" { + parent = "/" + } + objs, err := d.listDirObjects(ctx, parent, false) + if err != nil { + return nil, err + } + for _, obj := range objs { + if obj.GetName() == name { + return obj, nil + } + } + return nil, errs.ObjectNotFound +} + +func (d *Chunker) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj := d.linkedObject(file) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + actualPath, err := d.getActualPathForRemoteOnTarget(file.GetPath(), remoteIndex) + if err != nil { + return nil, fmt.Errorf("failed to convert path to remote path: %w", err) + } + link, _, err := op.Link(ctx, d.remoteTargets[remoteIndex].Storage, actualPath, args) + return link, err + } + + linkedParts := make([]linkedPart, 0, len(obj.Parts)) + baseClosers := utils.EmptyClosers() + for _, part := range obj.Parts { + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID, part.RemoteIndex) + if err != nil { + return nil, fmt.Errorf("failed to convert chunk path: %w", err) + } + link, _, err := op.Link(ctx, d.remoteTargets[part.RemoteIndex].Storage, actualPath, args) + if err != nil { + return nil, err + } + if link.MFile != nil { + baseClosers.Add(link.MFile) + } + if link.RangeReadCloser != nil { + baseClosers.Add(link.RangeReadCloser) + } + linkedParts = append(linkedParts, linkedPart{ + part: part, + link: link, + }) + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + return d.openChunkReader(ctx, linkedParts, obj.GetSize(), httpRange) + }, + Closers: baseClosers, + }, + }, nil +} + +func (d *Chunker) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.ensureDirOnAllTargets(ctx, path.Join(parentDir.GetPath(), dirName)) +} + +func (d *Chunker) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj.IsDir() { + return d.moveDirAcrossTargets(ctx, srcObj.GetPath(), dstDir.GetPath()) + } + obj := d.linkedObject(srcObj) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDir.GetPath()); err != nil { + return err + } + srcRemoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Move(ctx, d.remoteTargets[remoteIndex].Storage, srcRemoteActualPath, dstRemoteActualPath) + } + + ensuredTargets := map[int]struct{}{} + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := ensuredTargets[location.RemoteIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, location.RemoteIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[location.RemoteIndex] = struct{}{} + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) + if err != nil { + return err + } + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), location.RemoteIndex) + if err != nil { + return err + } + if err := op.Move(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath, dstRemoteActualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if srcObj.IsDir() { + return d.renameDirAcrossTargets(ctx, srcObj.GetPath(), newName) + } + obj := d.linkedObject(srcObj) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + remoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Rename(ctx, d.remoteTargets[remoteIndex].Storage, remoteActualPath, newName) + } + + for _, part := range obj.Parts { + actualPath, err := d.getActualChunkPath(obj.GetPath(), part.No, part.XactID, part.RemoteIndex) + if err != nil { + return err + } + newChunkName := d.chunkPartBaseName(path.Join(path.Dir(obj.GetPath()), newName), part.No, part.XactID) + if err := op.Rename(ctx, d.remoteTargets[part.RemoteIndex].Storage, actualPath, newChunkName); err != nil { + return err + } + } + if obj.UsesMeta { + actualPath, err := d.getActualPathForRemoteOnTarget(obj.GetPath(), obj.MainRemoteIndex) + if err != nil { + return err + } + if err := op.Rename(ctx, d.remoteTargets[obj.MainRemoteIndex].Storage, actualPath, newName); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if srcObj.IsDir() { + return d.copyDirAcrossTargets(ctx, srcObj.GetPath(), dstDir.GetPath()) + } + obj := d.linkedObject(srcObj) + if obj == nil || !obj.Chunked { + remoteIndex := 0 + if obj != nil { + remoteIndex = obj.MainRemoteIndex + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDir.GetPath()); err != nil { + return err + } + srcRemoteActualPath, err := d.getActualPathForRemoteOnTarget(srcObj.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Copy(ctx, d.remoteTargets[remoteIndex].Storage, srcRemoteActualPath, dstRemoteActualPath) + } + + ensuredTargets := map[int]struct{}{} + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := ensuredTargets[location.RemoteIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, location.RemoteIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[location.RemoteIndex] = struct{}{} + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) + if err != nil { + return err + } + dstRemoteActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), location.RemoteIndex) + if err != nil { + return err + } + if err := op.Copy(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath, dstRemoteActualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + return d.removeDirAcrossTargets(ctx, obj.GetPath()) + } + chunkedObj := d.linkedObject(obj) + if chunkedObj == nil || !chunkedObj.Chunked { + remoteIndex := 0 + if chunkedObj != nil { + remoteIndex = chunkedObj.MainRemoteIndex + } + remoteActualPath, err := d.getActualPathForRemoteOnTarget(obj.GetPath(), remoteIndex) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + return op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, remoteActualPath) + } + + for _, location := range d.objectLocationsForObject(chunkedObj) { + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) + if err != nil { + return err + } + if err := op.Remove(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath); err != nil { + return err + } + } + return nil +} + +func (d *Chunker) Put(ctx context.Context, dstDir model.Obj, streamer model.FileStreamer, up driver.UpdateProgress) error { + primaryDirActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), 0) + if err != nil { + return fmt.Errorf("failed to convert path to remote path: %w", err) + } + if err := d.ensureDirOnTarget(ctx, 0, dstDir.GetPath()); err != nil { + return err + } + + existing := d.linkedObject(streamer.GetExist()) + logicalPath := path.Join(dstDir.GetPath(), streamer.GetName()) + if streamer.GetSize() <= d.ChunkSize { + if err := op.Put(ctx, d.remoteTargets[0].Storage, primaryDirActualPath, streamer, up, false); err != nil { + return err + } + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(d.targetLocation(logicalPath, 0))) + } + + if up == nil { + up = func(float64) {} + } + + var ( + md5Hasher hash.Hash + sha1Hasher hash.Hash + writers []io.Writer + ) + switch d.HashType { + case "md5": + md5Hasher = md5.New() + writers = append(writers, md5Hasher) + case "sha1": + sha1Hasher = sha1.New() + writers = append(writers, sha1Hasher) + } + writers = append(writers, driver.NewProgress(streamer.GetSize(), up)) + + baseReader := io.TeeReader(streamer, io.MultiWriter(writers...)) + xactID := strconv.FormatInt(time.Now().UnixNano(), 36) + if len(xactID) > 9 { + xactID = xactID[len(xactID)-9:] + } + if len(xactID) < 4 { + xactID = fmt.Sprintf("%04s", xactID) + } + + chunkCount := 0 + remaining := streamer.GetSize() + keepLocations := make([]objectLocation, 0, len(d.remoteTargets)+1) + ensuredTargets := map[int]struct{}{0: {}} + for remaining > 0 { + chunkLen := utils.Min(remaining, d.ChunkSize) + targetIndex := d.chunkTargetIndex(chunkCount) + if _, ok := ensuredTargets[targetIndex]; !ok { + if err := d.ensureDirOnTarget(ctx, targetIndex, dstDir.GetPath()); err != nil { + return err + } + ensuredTargets[targetIndex] = struct{}{} + } + dstDirActualPath, err := d.getActualPathForRemoteOnTarget(dstDir.GetPath(), targetIndex) + if err != nil { + return err + } + chunkName := d.chunkPartBaseName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) + chunkPath := d.makeChunkName(logicalPath, chunkCount, xactIDIfNeeded(d.MetaFormat, xactID)) + partReader := driver.NewLimitedUploadStream(ctx, &driver.ReaderWithCtx{ + Reader: io.LimitReader(baseReader, chunkLen), + Ctx: ctx, + }) + partStream := &stream.FileStream{ + Obj: &model.Object{ + Name: chunkName, + Size: chunkLen, + Modified: streamer.ModTime(), + Ctime: streamer.CreateTime(), + IsFolder: false, + }, + Reader: partReader, + Mimetype: "application/octet-stream", + WebPutAsTask: streamer.NeedStore(), + ForceStreamUpload: true, + } + if err := op.Put(ctx, d.remoteTargets[targetIndex].Storage, dstDirActualPath, partStream, nil, false); err != nil { + return err + } + keepLocations = append(keepLocations, d.targetLocation(chunkPath, targetIndex)) + remaining -= chunkLen + chunkCount++ + } + + if d.MetaFormat == "simplejson" { + md5Value := "" + if md5Hasher != nil { + md5Value = hex.EncodeToString(md5Hasher.Sum(nil)) + } + sha1Value := "" + if sha1Hasher != nil { + sha1Value = hex.EncodeToString(sha1Hasher.Sum(nil)) + } + txn := xactID + metaData, err := marshalMetadata(streamer.GetSize(), chunkCount, md5Value, sha1Value, txn) + if err != nil { + return err + } + metaStream := &stream.FileStream{ + Obj: &model.Object{ + Name: streamer.GetName(), + Size: int64(len(metaData)), + Modified: streamer.ModTime(), + Ctime: streamer.CreateTime(), + IsFolder: false, + }, + Reader: bytes.NewReader(metaData), + Mimetype: "application/json", + WebPutAsTask: false, + ForceStreamUpload: true, + } + if err := op.Put(ctx, d.remoteTargets[0].Storage, primaryDirActualPath, metaStream, nil, false); err != nil { + return err + } + keepLocations = append(keepLocations, d.targetLocation(logicalPath, 0)) + } else { + for remoteIndex := range d.remoteTargets { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err == nil { + _ = op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, actualPath) + } + } + } + + return d.cleanupReplacedObject(ctx, existing, d.buildKeepSet(keepLocations...)) +} + +func xactIDIfNeeded(metaFormat, xactID string) string { + if metaFormat == "simplejson" { + return xactID + } + return "" +} + +var _ driver.Driver = (*Chunker)(nil) diff --git a/drivers/chunker/meta.go b/drivers/chunker/meta.go new file mode 100644 index 00000000000..27265fe3033 --- /dev/null +++ b/drivers/chunker/meta.go @@ -0,0 +1,45 @@ +package chunker + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + defaultChunkSize int64 = 2147483648 + defaultChunkNameFmt = "{name}.rclone_chunk.{chunk:3}" + defaultMetaFormat = "simplejson" + defaultHashType = "md5" + defaultStartFrom = 1 +) + +type Addition struct { + RemotePath string `json:"remote_path" required:"true" help:"Primary AList mounted folder path used to store metadata and small files, e.g. /my-storage/chunks"` + RemotePaths string `json:"remote_paths" type:"text" help:"Additional AList mounted folder paths, one per line. Chunk files will be distributed across remote_path and these extra paths."` + StoreChunksInPrimary bool `json:"store_chunks_in_primary" type:"bool" default:"true" help:"When extra remote paths are configured, also store chunk files in remote_path"` + ChunkSize int64 `json:"chunk_size" type:"number" required:"true" default:"2147483648" help:"Files larger than this will be split into chunks"` + NameFormat string `json:"name_format" required:"true" default:"{name}.rclone_chunk.{chunk:3}" help:"Magic tokens: {name}, {chunk}, {chunk:N}. Name token must appear before chunk token."` + StartFrom int `json:"start_from" type:"number" required:"true" default:"1" help:"Chunk number base, usually 0 or 1"` + MetaFormat string `json:"meta_format" type:"select" required:"true" options:"simplejson,none" default:"simplejson" help:"simplejson is compatible with rclone chunker metadata"` + HashType string `json:"hash_type" type:"select" required:"true" options:"none,md5,sha1" default:"md5" help:"Hash stored in metadata for chunked files"` +} + +var config = driver.Config{ + Name: "Chunker", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: true, + NoCache: true, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Chunker{} + }) +} diff --git a/drivers/chunker/types.go b/drivers/chunker/types.go new file mode 100644 index 00000000000..ff0f6fc4d85 --- /dev/null +++ b/drivers/chunker/types.go @@ -0,0 +1,91 @@ +package chunker + +import ( + "regexp" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" +) + +const ( + ctrlTypeRegStr = `[a-z][a-z0-9]{2,6}` + tempSuffixFormat = `_%04s` + tempSuffixRegStr = `_([0-9a-z]{4,9})` + tempSuffixRegOld = `\.\.tmp_([0-9]{10,13})` + maxMetadataSizeRead = 1023 + maxMetadataSizeWrite = 255 + maxSafeChunkNumber = 10000000 + chunkerMetadataVerion = 2 +) + +var ctrlTypeRegexp = regexp.MustCompile(`^` + ctrlTypeRegStr + `$`) +var chunkTokenRegexp = regexp.MustCompile(`\{chunk(?::([0-9]+))?\}`) + +type Chunker struct { + model.Storage + Addition + remoteStorage driver.Driver + remoteTargets []remoteTarget + dataNameFmt string + nameRegexp *regexp.Regexp +} + +type remoteTarget struct { + MountPath string + Storage driver.Driver +} + +type locatedObj struct { + Obj model.Obj + RemoteIndex int +} + +type objectLocation struct { + LogicalPath string + RemoteIndex int +} + +type metadataJSON struct { + Version *int `json:"ver"` + Size *int64 `json:"size"` + ChunkNum *int `json:"nchunks"` + MD5 string `json:"md5,omitempty"` + SHA1 string `json:"sha1,omitempty"` + XactID string `json:"txn,omitempty"` +} + +type chunkMetadata struct { + Version int + Size int64 + NChunks int + MD5 string + SHA1 string + XactID string +} + +type chunkPart struct { + No int + Size int64 + XactID string + RemoteIndex int +} + +type groupInfo struct { + base *locatedObj + partsByXact map[string]map[int]chunkPart +} + +type Object struct { + model.Object + Main model.Obj + MainRemoteIndex int + Parts []chunkPart + Meta *chunkMetadata + Chunked bool + UsesMeta bool +} + +type linkedPart struct { + part chunkPart + link *model.Link +} diff --git a/drivers/chunker/util.go b/drivers/chunker/util.go new file mode 100644 index 00000000000..ebb740e8194 --- /dev/null +++ b/drivers/chunker/util.go @@ -0,0 +1,904 @@ +package chunker + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Chunker) validateOptions() error { + if strings.TrimSpace(d.RemotePath) == "" { + return errors.New("remote_path is required") + } + if d.ChunkSize <= 0 { + return errors.New("chunk_size must be positive") + } + switch d.MetaFormat { + case "simplejson", "none": + default: + return fmt.Errorf("unsupported meta_format: %s", d.MetaFormat) + } + switch d.HashType { + case "none", "md5", "sha1": + default: + return fmt.Errorf("unsupported hash_type: %s", d.HashType) + } + if d.MetaFormat == "none" && d.HashType != "none" { + return fmt.Errorf("hash_type %q requires meta_format=simplejson", d.HashType) + } + return nil +} + +func (d *Chunker) configuredRemotePaths() []string { + seen := map[string]struct{}{} + paths := make([]string, 0, 1) + addPath := func(p string) { + p = strings.TrimSpace(p) + if p == "" { + return + } + p = utils.FixAndCleanPath(p) + if _, ok := seen[p]; ok { + return + } + seen[p] = struct{}{} + paths = append(paths, p) + } + + addPath(d.RemotePath) + for _, line := range strings.Split(d.RemotePaths, "\n") { + addPath(line) + } + return paths +} + +func (d *Chunker) setChunkNameFormat(pattern string) error { + if dir, _ := path.Split(pattern); dir != "" { + return errors.New("directory separator prohibited") + } + + nameStart, nameEnd, err := parseNameToken(pattern) + if err != nil { + return err + } + chunkStart, chunkEnd, chunkWidth, err := parseChunkToken(pattern) + if err != nil { + return err + } + if nameStart > chunkStart { + return errors.New("name token must come before chunk token") + } + + reDigits := "[0-9]+" + if chunkWidth > 0 { + reDigits = fmt.Sprintf("[0-9]{%d,}", chunkWidth) + } + reDataOrCtrl := fmt.Sprintf("(?:(%s)|_(%s))", reDigits, ctrlTypeRegStr) + + beforeName := pattern[:nameStart] + between := pattern[nameEnd:chunkStart] + afterChunk := pattern[chunkEnd:] + + strRegex := fmt.Sprintf( + "^%s(.+?)%s%s%s(?:%s|%s)?$", + regexp.QuoteMeta(beforeName), + regexp.QuoteMeta(between), + reDataOrCtrl, + regexp.QuoteMeta(afterChunk), + tempSuffixRegStr, + tempSuffixRegOld, + ) + d.nameRegexp = regexp.MustCompile(strRegex) + + fmtDigits := "%d" + if chunkWidth > 0 { + fmtDigits = fmt.Sprintf("%%0%dd", chunkWidth) + } + d.dataNameFmt = strings.ReplaceAll(beforeName, "%", "%%") + + "%s" + + strings.ReplaceAll(between, "%", "%%") + + fmtDigits + + strings.ReplaceAll(afterChunk, "%", "%%") + return nil +} + +func parseNameToken(pattern string) (start, end int, err error) { + nameMagicCount := strings.Count(pattern, "{name}") + switch nameMagicCount { + case 0: + return 0, 0, errors.New("pattern must contain one name token: {name}") + case 1: + default: + return 0, 0, errors.New("pattern must contain exactly one name token: {name}") + } + start = strings.Index(pattern, "{name}") + return start, start + len("{name}"), nil +} + +func parseChunkToken(pattern string) (start, end, width int, err error) { + chunkMatches := chunkTokenRegexp.FindAllStringSubmatchIndex(pattern, -1) + switch len(chunkMatches) { + case 0: + return 0, 0, 0, errors.New("pattern must contain one chunk token: {chunk} or {chunk:N}") + case 1: + default: + return 0, 0, 0, errors.New("pattern must contain exactly one chunk token: {chunk} or {chunk:N}") + } + match := chunkMatches[0] + start = match[0] + end = match[1] + if match[2] >= 0 && match[3] >= 0 { + width, err = strconv.Atoi(pattern[match[2]:match[3]]) + if err != nil || width <= 0 { + return 0, 0, 0, errors.New("chunk width in {chunk:N} must be a positive integer") + } + } + return start, end, width, nil +} + +func (d *Chunker) makeChunkName(filePath string, chunkNo int, xactID string) string { + dir, baseName := path.Split(filePath) + name := fmt.Sprintf(d.dataNameFmt, baseName, chunkNo+d.StartFrom) + if xactID != "" { + name += fmt.Sprintf(tempSuffixFormat, xactID) + } + return dir + name +} + +func (d *Chunker) parseChunkName(filePath string) (parentPath string, chunkNo int, ctrlType, xactID string) { + dir, name := path.Split(filePath) + match := d.nameRegexp.FindStringSubmatch(name) + if match == nil || match[1] == "" { + return "", -1, "", "" + } + + chunkNo = -1 + if match[2] != "" { + n, err := strconv.Atoi(match[2]) + if err != nil { + return "", -1, "", "" + } + chunkNo = n - d.StartFrom + if chunkNo < 0 { + return "", -1, "", "" + } + } + + if match[4] != "" { + xactID = match[4] + } + if match[5] != "" { + oldNum, err := strconv.ParseInt(match[5], 10, 64) + if err != nil || oldNum < 0 { + return "", -1, "", "" + } + xactID = fmt.Sprintf(tempSuffixFormat, strconv.FormatInt(oldNum, 36))[1:] + } + + return dir + match[1], chunkNo, match[3], xactID +} + +func marshalMetadata(size int64, nChunks int, md5Value, sha1Value, xactID string) ([]byte, error) { + version := chunkerMetadataVerion + if xactID == "" && version == 2 { + version = 1 + } + meta := metadataJSON{ + Version: &version, + Size: &size, + ChunkNum: &nChunks, + MD5: md5Value, + SHA1: sha1Value, + XactID: xactID, + } + data, err := json.Marshal(&meta) + if err == nil && len(data) >= maxMetadataSizeWrite { + return nil, errors.New("metadata can't be this big") + } + return data, err +} + +func unmarshalMetadata(data []byte) (*chunkMetadata, error) { + if len(data) > maxMetadataSizeWrite { + return nil, errors.New("metadata is too large") + } + if data == nil || len(data) < 2 || data[0] != '{' || data[len(data)-1] != '}' { + return nil, errors.New("invalid json") + } + var meta metadataJSON + if err := json.Unmarshal(data, &meta); err != nil { + return nil, err + } + if meta.Version == nil || meta.Size == nil || meta.ChunkNum == nil { + return nil, errors.New("missing required field") + } + if *meta.Version < 1 { + return nil, errors.New("wrong version") + } + if *meta.Size < 0 { + return nil, errors.New("negative file size") + } + if *meta.ChunkNum < 1 || *meta.ChunkNum > maxSafeChunkNumber { + return nil, errors.New("wrong number of chunks") + } + if meta.MD5 != "" { + if _, err := hex.DecodeString(meta.MD5); err != nil || len(meta.MD5) != 32 { + return nil, errors.New("wrong md5 hash") + } + } + if meta.SHA1 != "" { + if _, err := hex.DecodeString(meta.SHA1); err != nil || len(meta.SHA1) != 40 { + return nil, errors.New("wrong sha1 hash") + } + } + if *meta.Version > chunkerMetadataVerion { + return nil, errors.New("unknown metadata version") + } + return &chunkMetadata{ + Version: *meta.Version, + Size: *meta.Size, + NChunks: *meta.ChunkNum, + MD5: meta.MD5, + SHA1: meta.SHA1, + XactID: meta.XactID, + }, nil +} + +func joinRemotePathWithBase(baseMountPath, logicalPath string) string { + logicalPath = utils.FixAndCleanPath(logicalPath) + if utils.PathEqual(logicalPath, "/") { + return utils.FixAndCleanPath(baseMountPath) + } + return path.Join(utils.FixAndCleanPath(baseMountPath), logicalPath) +} + +func (d *Chunker) joinRemotePath(logicalPath string) string { + return joinRemotePathWithBase(d.RemotePath, logicalPath) +} + +func (d *Chunker) joinRemotePathForTarget(logicalPath string, remoteIndex int) string { + target := d.remoteTargets[remoteIndex] + return joinRemotePathWithBase(target.MountPath, logicalPath) +} + +func (d *Chunker) getActualPathForRemote(logicalPath string) (string, error) { + return d.getActualPathForRemoteOnTarget(logicalPath, 0) +} + +func (d *Chunker) getActualPathForRemoteOnTarget(logicalPath string, remoteIndex int) (string, error) { + _, actualPath, err := op.GetStorageAndActualPath(d.joinRemotePathForTarget(logicalPath, remoteIndex)) + return actualPath, err +} + +func (d *Chunker) getActualChunkPath(filePath string, chunkNo int, xactID string, remoteIndex int) (string, error) { + return d.getActualPathForRemoteOnTarget(d.makeChunkName(filePath, chunkNo, xactID), remoteIndex) +} + +func (d *Chunker) chunkTargetIndex(chunkNo int) int { + targetIndexes := d.chunkTargetIndexes() + if len(targetIndexes) == 0 { + return 0 + } + if chunkNo < 0 { + return targetIndexes[0] + } + return targetIndexes[chunkNo%len(targetIndexes)] +} + +func (d *Chunker) chunkTargetIndexes() []int { + if len(d.remoteTargets) <= 1 { + return []int{0} + } + if d.StoreChunksInPrimary { + targets := make([]int, 0, len(d.remoteTargets)) + for i := range d.remoteTargets { + targets = append(targets, i) + } + return targets + } + targets := make([]int, 0, len(d.remoteTargets)-1) + for i := 1; i < len(d.remoteTargets); i++ { + targets = append(targets, i) + } + if len(targets) == 0 { + return []int{0} + } + return targets +} + +func (d *Chunker) targetLocation(logicalPath string, remoteIndex int) objectLocation { + return objectLocation{ + LogicalPath: utils.FixAndCleanPath(logicalPath), + RemoteIndex: remoteIndex, + } +} + +func (d *Chunker) chunkLocation(filePath string, part chunkPart) objectLocation { + return d.targetLocation(d.makeChunkName(filePath, part.No, part.XactID), part.RemoteIndex) +} + +func (d *Chunker) listDirObjects(ctx context.Context, dirPath string, refresh bool) ([]model.Obj, error) { + groups := map[string]*groupInfo{} + dirMap := map[string]model.Obj{} + found := false + + for remoteIndex := range d.remoteTargets { + remotePath := d.joinRemotePathForTarget(dirPath, remoteIndex) + entries, err := fsList(ctx, remotePath, refresh) + if err != nil { + if errs.IsObjectNotFound(err) { + continue + } + return nil, err + } + found = true + + for _, entry := range entries { + if entry.IsDir() { + if _, ok := dirMap[entry.GetName()]; !ok { + dirMap[entry.GetName()] = &model.Object{ + Name: entry.GetName(), + Path: path.Join(dirPath, entry.GetName()), + Size: 0, + Modified: entry.ModTime(), + Ctime: entry.CreateTime(), + IsFolder: true, + HashInfo: entry.GetHash(), + } + } + continue + } + + mainName, chunkNo, ctrlType, xactID := d.parseChunkName(entry.GetName()) + if mainName == "" { + g := groups[entry.GetName()] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[entry.GetName()] = g + } + if g.base == nil || remoteIndex < g.base.RemoteIndex { + g.base = &locatedObj{ + Obj: entry, + RemoteIndex: remoteIndex, + } + } + continue + } + if chunkNo < 0 || ctrlType != "" { + continue + } + g := groups[mainName] + if g == nil { + g = &groupInfo{partsByXact: map[string]map[int]chunkPart{}} + groups[mainName] = g + } + if g.partsByXact[xactID] == nil { + g.partsByXact[xactID] = map[int]chunkPart{} + } + part := chunkPart{ + No: chunkNo, + Size: entry.GetSize(), + XactID: xactID, + RemoteIndex: remoteIndex, + } + existing, ok := g.partsByXact[xactID][chunkNo] + if !ok || part.RemoteIndex < existing.RemoteIndex { + g.partsByXact[xactID][chunkNo] = part + } + } + } + + if !found && !utils.PathEqual(dirPath, "/") { + return nil, errs.ObjectNotFound + } + + dirs := make([]model.Obj, 0, len(dirMap)) + for _, obj := range dirMap { + dirs = append(dirs, obj) + } + + result := make([]model.Obj, 0, len(dirs)+len(groups)) + result = append(result, dirs...) + for name, group := range groups { + obj, ok, err := d.buildListedObject(ctx, dirPath, name, group) + if err != nil { + return nil, err + } + if ok { + result = append(result, obj) + } + } + return result, nil +} + +func (d *Chunker) targetPathExists(ctx context.Context, remoteIndex int, logicalPath string) (bool, error) { + _, err := fs.Get(ctx, d.joinRemotePathForTarget(logicalPath, remoteIndex), &fs.GetArgs{NoLog: true}) + if err == nil { + return true, nil + } + if errs.IsObjectNotFound(err) { + return false, nil + } + return false, err +} + +func (d *Chunker) ensureDirOnTarget(ctx context.Context, remoteIndex int, logicalDirPath string) error { + logicalDirPath = utils.FixAndCleanPath(logicalDirPath) + if utils.PathEqual(logicalDirPath, "/") { + return nil + } + return fs.MakeDir(ctx, d.joinRemotePathForTarget(logicalDirPath, remoteIndex)) +} + +func (d *Chunker) ensureDirOnAllTargets(ctx context.Context, logicalDirPath string) error { + var errsList []error + for remoteIndex := range d.remoteTargets { + if err := d.ensureDirOnTarget(ctx, remoteIndex, logicalDirPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) existingLocationsForDir(ctx context.Context, logicalDirPath string) ([]int, error) { + locations := make([]int, 0, len(d.remoteTargets)) + for remoteIndex := range d.remoteTargets { + exists, err := d.targetPathExists(ctx, remoteIndex, logicalDirPath) + if err != nil { + return nil, err + } + if exists { + locations = append(locations, remoteIndex) + } + } + return locations, nil +} + +func (d *Chunker) dirLocationsOrAll(ctx context.Context, logicalDirPath string) ([]int, error) { + locations, err := d.existingLocationsForDir(ctx, logicalDirPath) + if err != nil { + return nil, err + } + if len(locations) > 0 { + return locations, nil + } + all := make([]int, 0, len(d.remoteTargets)) + for remoteIndex := range d.remoteTargets { + all = append(all, remoteIndex) + } + return all, nil +} + +func (d *Chunker) moveDirAcrossTargets(ctx context.Context, srcPath, dstDirPath string) error { + locations, err := d.existingLocationsForDir(ctx, srcPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDirPath); err != nil { + errsList = append(errsList, err) + continue + } + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + dstActualPath, err := d.getActualPathForRemoteOnTarget(dstDirPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Move(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, dstActualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) copyDirAcrossTargets(ctx context.Context, srcPath, dstDirPath string) error { + locations, err := d.dirLocationsOrAll(ctx, srcPath) + if err != nil { + return err + } + var errsList []error + for _, remoteIndex := range locations { + exists, err := d.targetPathExists(ctx, remoteIndex, srcPath) + if err != nil { + errsList = append(errsList, err) + continue + } + if !exists { + continue + } + if err := d.ensureDirOnTarget(ctx, remoteIndex, dstDirPath); err != nil { + errsList = append(errsList, err) + continue + } + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + dstActualPath, err := d.getActualPathForRemoteOnTarget(dstDirPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Copy(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, dstActualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) renameDirAcrossTargets(ctx context.Context, srcPath, newName string) error { + locations, err := d.existingLocationsForDir(ctx, srcPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + srcActualPath, err := d.getActualPathForRemoteOnTarget(srcPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Rename(ctx, d.remoteTargets[remoteIndex].Storage, srcActualPath, newName); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) removeDirAcrossTargets(ctx context.Context, logicalPath string) error { + locations, err := d.existingLocationsForDir(ctx, logicalPath) + if err != nil { + return err + } + if len(locations) == 0 { + return errs.ObjectNotFound + } + var errsList []error + for _, remoteIndex := range locations { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err != nil { + errsList = append(errsList, err) + continue + } + if err := op.Remove(ctx, d.remoteTargets[remoteIndex].Storage, actualPath); err != nil { + errsList = append(errsList, err) + } + } + return errors.Join(errsList...) +} + +func (d *Chunker) buildListedObject(ctx context.Context, dirPath, name string, group *groupInfo) (model.Obj, bool, error) { + var meta *chunkMetadata + var err error + if group.base != nil && group.base.Obj.GetSize() <= maxMetadataSizeRead && len(group.partsByXact) > 0 { + meta, err = d.readMetadata(ctx, path.Join(dirPath, name), group.base.Obj.GetSize(), group.base.RemoteIndex) + if err != nil { + meta = nil + } + } + + if meta == nil && group.base != nil && len(group.partsByXact) == 0 { + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: group.base.Obj.GetSize(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), + IsFolder: false, + HashInfo: group.base.Obj.GetHash(), + }, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, + }, true, nil + } + + selected := map[int]chunkPart{} + switch { + case meta != nil: + selected = group.partsByXact[meta.XactID] + case group.base == nil: + selected = group.partsByXact[""] + default: + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: group.base.Obj.GetSize(), + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), + IsFolder: false, + HashInfo: group.base.Obj.GetHash(), + }, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, + }, true, nil + } + + parts := sortChunkParts(selected) + if len(parts) == 0 { + if meta != nil { + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: meta.Size, + Modified: group.base.Obj.ModTime(), + Ctime: group.base.Obj.CreateTime(), + IsFolder: false, + HashInfo: buildHashInfo(meta), + }, + Main: group.base.Obj, + MainRemoteIndex: group.base.RemoteIndex, + Meta: meta, + Chunked: true, + UsesMeta: true, + }, true, nil + } + return nil, false, nil + } + + size := int64(0) + for _, part := range parts { + size += part.Size + } + modified := time.Time{} + ctime := time.Time{} + if group.base != nil { + modified = group.base.Obj.ModTime() + ctime = group.base.Obj.CreateTime() + } + if meta != nil { + size = meta.Size + } + + mainRemoteIndex := 0 + var mainObj model.Obj + if group.base != nil { + mainRemoteIndex = group.base.RemoteIndex + mainObj = group.base.Obj + } + + return &Object{ + Object: model.Object{ + Name: name, + Path: path.Join(dirPath, name), + Size: size, + Modified: modified, + Ctime: ctime, + IsFolder: false, + HashInfo: buildHashInfo(meta), + }, + Main: mainObj, + MainRemoteIndex: mainRemoteIndex, + Parts: parts, + Meta: meta, + Chunked: true, + UsesMeta: meta != nil, + }, true, nil +} + +func (d *Chunker) readMetadata(ctx context.Context, logicalPath string, size int64, remoteIndex int) (*chunkMetadata, error) { + actualPath, err := d.getActualPathForRemoteOnTarget(logicalPath, remoteIndex) + if err != nil { + return nil, err + } + link, obj, err := op.Link(ctx, d.remoteTargets[remoteIndex].Storage, actualPath, model.LinkArgs{}) + if err != nil { + return nil, err + } + ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, link) + if err != nil { + return nil, err + } + defer ss.Close() + + reader, err := ss.RangeRead(http_range.Range{Start: 0, Length: size}) + if err != nil { + return nil, err + } + if closer, ok := reader.(io.Closer); ok { + defer closer.Close() + } + data, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return unmarshalMetadata(data) +} + +func sortChunkParts(parts map[int]chunkPart) []chunkPart { + result := make([]chunkPart, 0, len(parts)) + for _, part := range parts { + result = append(result, part) + } + sort.Slice(result, func(i, j int) bool { + return result[i].No < result[j].No + }) + return result +} + +func buildHashInfo(meta *chunkMetadata) utils.HashInfo { + if meta == nil { + return utils.HashInfo{} + } + hashes := map[*utils.HashType]string{} + if meta.MD5 != "" { + hashes[utils.MD5] = meta.MD5 + } + if meta.SHA1 != "" { + hashes[utils.SHA1] = meta.SHA1 + } + return utils.NewHashInfoByMap(hashes) +} + +func (d *Chunker) linkedObject(obj model.Obj) *Object { + if linked, ok := obj.(*Object); ok { + return linked + } + return nil +} + +func (d *Chunker) objectLocationsForObject(obj *Object) []objectLocation { + if obj == nil { + return nil + } + locations := make([]objectLocation, 0, len(obj.Parts)+1) + if obj.Chunked && obj.UsesMeta { + locations = append(locations, d.targetLocation(obj.GetPath(), obj.MainRemoteIndex)) + } + if !obj.Chunked { + locations = append(locations, d.targetLocation(obj.GetPath(), obj.MainRemoteIndex)) + return locations + } + for _, part := range obj.Parts { + locations = append(locations, d.chunkLocation(obj.GetPath(), part)) + } + return locations +} + +func (d *Chunker) cleanupReplacedObject(ctx context.Context, obj *Object, keep map[string]struct{}) error { + if obj == nil { + return nil + } + var errs []error + for _, location := range d.objectLocationsForObject(obj) { + if _, ok := keep[d.keepKey(location)]; ok { + continue + } + actualPath, err := d.getActualPathForRemoteOnTarget(location.LogicalPath, location.RemoteIndex) + if err != nil { + errs = append(errs, err) + continue + } + if err := op.Remove(ctx, d.remoteTargets[location.RemoteIndex].Storage, actualPath); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +func (d *Chunker) keepKey(location objectLocation) string { + return fmt.Sprintf("%d:%s", location.RemoteIndex, utils.FixAndCleanPath(location.LogicalPath)) +} + +func (d *Chunker) buildKeepSet(locations ...objectLocation) map[string]struct{} { + keep := make(map[string]struct{}, len(locations)) + for _, location := range locations { + if location.LogicalPath == "" { + continue + } + keep[d.keepKey(location)] = struct{}{} + } + return keep +} + +func (d *Chunker) chunkPartBaseName(filePath string, chunkNo int, xactID string) string { + return path.Base(d.makeChunkName(filePath, chunkNo, xactID)) +} + +func (d *Chunker) openChunkReader(ctx context.Context, parts []linkedPart, totalSize int64, req http_range.Range) (io.ReadCloser, error) { + if req.Start < 0 || req.Start > totalSize { + return nil, fmt.Errorf("range start out of bound") + } + if req.Length < 0 || req.Start+req.Length > totalSize { + req.Length = totalSize - req.Start + } + if req.Length == 0 { + return io.NopCloser(strings.NewReader("")), nil + } + + var ( + readers []io.Reader + closers = utils.EmptyClosers() + offset int64 + remaining = req.Length + ) + for _, part := range parts { + partStart := offset + partEnd := offset + part.part.Size + offset = partEnd + if req.Start >= partEnd || remaining <= 0 { + continue + } + localStart := int64(0) + if req.Start > partStart { + localStart = req.Start - partStart + } + localLength := utils.Min(part.part.Size-localStart, remaining) + rc, err := d.openPartRange(ctx, part.link, part.part.Size, localStart, localLength) + if err != nil { + _ = closers.Close() + return nil, err + } + readers = append(readers, rc) + closers.Add(rc) + remaining -= localLength + } + if remaining > 0 { + _ = closers.Close() + return nil, errors.New("missing chunk data") + } + return utils.NewReadCloser(io.MultiReader(readers...), func() error { + return closers.Close() + }), nil +} + +func (d *Chunker) openPartRange(ctx context.Context, link *model.Link, size, offset, length int64) (io.ReadCloser, error) { + httpRange := http_range.Range{Start: offset, Length: length} + switch { + case link.MFile != nil: + return io.NopCloser(io.NewSectionReader(link.MFile, offset, length)), nil + case link.RangeReadCloser != nil: + return link.RangeReadCloser.RangeRead(ctx, httpRange) + case link.URL != "": + rrc, err := stream.GetRangeReadCloserFromLink(size, link) + if err != nil { + return nil, err + } + rc, err := rrc.RangeRead(ctx, httpRange) + if err != nil { + _ = rrc.Close() + return nil, err + } + return utils.NewReadCloser(rc, func() error { + return rrc.Close() + }), nil + default: + return nil, errors.New("chunk part has no readable link") + } +} + +func fsList(ctx context.Context, remotePath string, refresh bool) ([]model.Obj, error) { + return fs.List(ctx, remotePath, &fs.ListArgs{NoLog: true, Refresh: refresh}) +} diff --git a/drivers/cloudreve/driver.go b/drivers/cloudreve/driver.go index 2a22380e314..dcde58c638d 100644 --- a/drivers/cloudreve/driver.go +++ b/drivers/cloudreve/driver.go @@ -4,11 +4,12 @@ import ( "context" "io" "net/http" - "strconv" + "path" "strings" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" @@ -17,6 +18,7 @@ import ( type Cloudreve struct { model.Storage Addition + ref *Cloudreve } func (d *Cloudreve) Config() driver.Config { @@ -36,8 +38,18 @@ func (d *Cloudreve) Init(ctx context.Context) error { return d.login() } +func (d *Cloudreve) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*Cloudreve) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + func (d *Cloudreve) Drop(ctx context.Context) error { d.Cookie = "" + d.ref = nil return nil } @@ -53,6 +65,14 @@ func (d *Cloudreve) List(ctx context.Context, dir model.Obj, args model.ListArgs if err != nil { return nil, err } + if src.Type == "dir" && d.EnableThumbAndFolderSize { + var dprop DirectoryProp + err = d.request(http.MethodGet, "/object/property/"+src.Id+"?is_folder=true", nil, &dprop) + if err != nil { + return nil, err + } + src.Size = dprop.Size + } return objectToObj(src, thumb), nil }) } @@ -63,6 +83,9 @@ func (d *Cloudreve) Link(ctx context.Context, file model.Obj, args model.LinkArg if err != nil { return nil, err } + if strings.HasPrefix(dUrl, "/api") { + dUrl = d.Address + dUrl + } return &model.Link{ URL: dUrl, }, nil @@ -79,7 +102,7 @@ func (d *Cloudreve) MakeDir(ctx context.Context, parentDir model.Obj, dirName st func (d *Cloudreve) Move(ctx context.Context, srcObj, dstDir model.Obj) error { body := base.Json{ "action": "move", - "src_dir": srcObj.GetPath(), + "src_dir": path.Dir(srcObj.GetPath()), "dst": dstDir.GetPath(), "src": convertSrc(srcObj), } @@ -101,7 +124,7 @@ func (d *Cloudreve) Rename(ctx context.Context, srcObj model.Obj, newName string func (d *Cloudreve) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { body := base.Json{ - "src_dir": srcObj.GetPath(), + "src_dir": path.Dir(srcObj.GetPath()), "dst": dstDir.GetPath(), "src": convertSrc(srcObj), } @@ -122,6 +145,8 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File if io.ReadCloser(stream) == http.NoBody { return d.create(ctx, dstDir, stream) } + + // 获取存储策略 var r DirectoryResp err := d.request(http.MethodGet, "/directory"+dstDir.GetPath(), nil, &r) if err != nil { @@ -132,8 +157,10 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File "size": stream.GetSize(), "name": stream.GetName(), "policy_id": r.Policy.Id, - "last_modified": stream.ModTime().Unix(), + "last_modified": stream.ModTime().UnixMilli(), } + + // 获取上传会话信息 var u UploadInfo err = d.request(http.MethodPut, "/file/upload", func(req *resty.Request) { req.SetBody(uploadBody) @@ -141,36 +168,26 @@ func (d *Cloudreve) Put(ctx context.Context, dstDir model.Obj, stream model.File if err != nil { return err } - var chunkSize = u.ChunkSize - var buf []byte - var chunk int - for { - var n int - buf = make([]byte, chunkSize) - n, err = io.ReadAtLeast(stream, buf, chunkSize) - if err != nil && err != io.ErrUnexpectedEOF { - if err == io.EOF { - return nil - } - return err - } - - if n == 0 { - break - } - buf = buf[:n] - err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { - req.SetHeader("Content-Type", "application/octet-stream") - req.SetHeader("Content-Length", strconv.Itoa(n)) - req.SetBody(buf) - }, nil) - if err != nil { - break - } - chunk++ + // 根据存储方式选择分片上传的方法 + switch r.Policy.Type { + case "onedrive": + err = d.upOneDrive(ctx, stream, u, up) + case "s3": + err = d.upS3(ctx, stream, u, up) + case "remote": // 从机存储 + err = d.upRemote(ctx, stream, u, up) + case "local": // 本机存储 + err = d.upLocal(ctx, stream, u, up) + default: + err = errs.NotImplement } - return err + if err != nil { + // 删除失败的会话 + _ = d.request(http.MethodDelete, "/file/upload/"+u.SessionID, nil, nil) + return err + } + return nil } func (d *Cloudreve) create(ctx context.Context, dir model.Obj, file model.Obj) error { diff --git a/drivers/cloudreve/meta.go b/drivers/cloudreve/meta.go index d01d54791d3..92c0b9fb1d7 100644 --- a/drivers/cloudreve/meta.go +++ b/drivers/cloudreve/meta.go @@ -9,11 +9,12 @@ type Addition struct { // Usually one of two driver.RootPath // define other - Address string `json:"address" required:"true"` - Username string `json:"username"` - Password string `json:"password"` - Cookie string `json:"cookie"` - CustomUA string `json:"custom_ua"` + Address string `json:"address" required:"true"` + Username string `json:"username"` + Password string `json:"password"` + Cookie string `json:"cookie"` + CustomUA string `json:"custom_ua"` + EnableThumbAndFolderSize bool `json:"enable_thumb_and_folder_size"` } var config = driver.Config{ diff --git a/drivers/cloudreve/types.go b/drivers/cloudreve/types.go index e25673829a8..8a465f01a3b 100644 --- a/drivers/cloudreve/types.go +++ b/drivers/cloudreve/types.go @@ -21,9 +21,12 @@ type Policy struct { } type UploadInfo struct { - SessionID string `json:"sessionID"` - ChunkSize int `json:"chunkSize"` - Expires int `json:"expires"` + SessionID string `json:"sessionID"` + ChunkSize int `json:"chunkSize"` + Expires int `json:"expires"` + UploadURLs []string `json:"uploadURLs"` + Credential string `json:"credential,omitempty"` // local + CompleteURL string `json:"completeURL,omitempty"` // s3 } type DirectoryResp struct { @@ -44,6 +47,10 @@ type Object struct { SourceEnabled bool `json:"source_enabled"` } +type DirectoryProp struct { + Size int `json:"size"` +} + func objectToObj(f Object, t model.Thumbnail) *model.ObjThumb { return &model.ObjThumb{ Object: model.Object{ diff --git a/drivers/cloudreve/util.go b/drivers/cloudreve/util.go index ed8794667c1..5054de6cb56 100644 --- a/drivers/cloudreve/util.go +++ b/drivers/cloudreve/util.go @@ -1,18 +1,26 @@ package cloudreve import ( + "bytes" + "context" "encoding/base64" + "encoding/json" "errors" + "fmt" + "io" "net/http" + "strconv" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/cookie" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" - json "github.com/json-iterator/go" jsoniter "github.com/json-iterator/go" ) @@ -20,17 +28,23 @@ import ( const loginPath = "/user/session" +func (d *Cloudreve) getUA() string { + if d.CustomUA != "" { + return d.CustomUA + } + return base.UserAgent +} + func (d *Cloudreve) request(method string, path string, callback base.ReqCallback, out interface{}) error { - u := d.Address + "/api/v3" + path - ua := d.CustomUA - if ua == "" { - ua = base.UserAgent + if d.ref != nil { + return d.ref.request(method, path, callback, out) } + u := d.Address + "/api/v3" + path req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": "cloudreve-session=" + d.Cookie, "Accept": "application/json, text/plain, */*", - "User-Agent": ua, + "User-Agent": d.getUA(), }) var r Resp @@ -69,11 +83,11 @@ func (d *Cloudreve) request(method string, path string, callback base.ReqCallbac } if out != nil && r.Data != nil { var marshal []byte - marshal, err = json.Marshal(r.Data) + marshal, err = jsoniter.Marshal(r.Data) if err != nil { return err } - err = json.Unmarshal(marshal, out) + err = jsoniter.Unmarshal(marshal, out) if err != nil { return err } @@ -93,7 +107,7 @@ func (d *Cloudreve) login() error { if err == nil { break } - if err != nil && err.Error() != "CAPTCHA not match." { + if err.Error() != "CAPTCHA not match." { break } } @@ -151,15 +165,14 @@ func convertSrc(obj model.Obj) map[string]interface{} { } func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { - ua := d.CustomUA - if ua == "" { - ua = base.UserAgent + if !d.Addition.EnableThumbAndFolderSize { + return model.Thumbnail{}, nil } req := base.NoRedirectClient.R() req.SetHeaders(map[string]string{ "Cookie": "cloudreve-session=" + d.Cookie, "Accept": "image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", - "User-Agent": ua, + "User-Agent": d.getUA(), }) resp, err := req.Execute(http.MethodGet, d.Address+"/api/v3/file/thumb/"+file.Id) if err != nil { @@ -169,3 +182,281 @@ func (d *Cloudreve) GetThumb(file Object) (model.Thumbnail, error) { Thumbnail: resp.Header().Get("Location"), }, nil } + +func (d *Cloudreve) upLocal(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + for finish < stream.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := stream.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Cloudreve-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { + req.SetHeader("Content-Type", "application/octet-stream") + req.SetContentLength(true) + req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) + req.SetHeader("User-Agent", d.getUA()) + req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + req.AddRetryCondition(func(r *resty.Response, err error) bool { + if err != nil { + return true + } + if r.IsError() { + return true + } + var retryResp Resp + jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) + if jErr != nil { + return true + } + if retryResp.Code != 0 { + return true + } + return false + }) + }, nil) + if err != nil { + return err + } + finish += byteSize + up(float64(finish) * 100 / float64(stream.GetSize())) + chunk++ + } + return nil +} + +func (d *Cloudreve) upRemote(ctx context.Context, stream model.FileStreamer, u UploadInfo, up driver.UpdateProgress) error { + uploadUrl := u.UploadURLs[0] + credential := u.Credential + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + retryCount := 0 + maxRetries := 3 + for finish < stream.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := stream.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Cloudreve-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(stream, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + err = func() error { + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }() + if err == nil { + retryCount = 0 + finish += byteSize + up(float64(finish) * 100 / float64(stream.GetSize())) + chunk++ + } else { + retryCount++ + if retryCount > maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1<") + for i, etag := range etags { + bodyBuilder.WriteString(fmt.Sprintf( + `%d%s`, + i+1, // PartNumber 从 1 开始 + etag, + )) + } + bodyBuilder.WriteString("") + req, err := http.NewRequest( + "POST", + u.CompleteURL, + strings.NewReader(bodyBuilder.String()), + ) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/xml") + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body)) + } + + // 上传成功发送回调请求 + err = d.request(http.MethodGet, "/callback/s3/"+u.SessionID, nil, nil) + if err != nil { + return err + } + return nil +} diff --git a/drivers/cloudreve_v4/driver.go b/drivers/cloudreve_v4/driver.go new file mode 100644 index 00000000000..32a22f62652 --- /dev/null +++ b/drivers/cloudreve_v4/driver.go @@ -0,0 +1,305 @@ +package cloudreve_v4 + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type CloudreveV4 struct { + model.Storage + Addition + ref *CloudreveV4 +} + +func (d *CloudreveV4) Config() driver.Config { + if d.ref != nil { + return d.ref.Config() + } + if d.EnableVersionUpload { + config.NoOverwriteUpload = false + } + return config +} + +func (d *CloudreveV4) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *CloudreveV4) Init(ctx context.Context) error { + // removing trailing slash + d.Address = strings.TrimSuffix(d.Address, "/") + op.MustSaveDriverStorage(d) + if d.ref != nil { + return nil + } + if d.AccessToken == "" && d.RefreshToken != "" { + return d.refreshToken() + } + if d.Username != "" { + return d.login() + } + return nil +} + +func (d *CloudreveV4) InitReference(storage driver.Driver) error { + refStorage, ok := storage.(*CloudreveV4) + if ok { + d.ref = refStorage + return nil + } + return errs.NotSupport +} + +func (d *CloudreveV4) Drop(ctx context.Context) error { + d.ref = nil + return nil +} + +func (d *CloudreveV4) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + const pageSize int = 100 + var f []File + var r FileResp + params := map[string]string{ + "page_size": strconv.Itoa(pageSize), + "uri": dir.GetPath(), + "order_by": d.OrderBy, + "order_direction": d.OrderDirection, + "page": "0", + } + + for { + err := d.request(http.MethodGet, "/file", func(req *resty.Request) { + req.SetQueryParams(params) + }, &r) + if err != nil { + return nil, err + } + f = append(f, r.Files...) + if r.Pagination.NextToken == "" || len(r.Files) < pageSize { + break + } + params["next_page_token"] = r.Pagination.NextToken + } + + return utils.SliceConvert(f, func(src File) (model.Obj, error) { + if d.EnableFolderSize && src.Type == 1 { + var ds FolderSummaryResp + err := d.request(http.MethodGet, "/file/info", func(req *resty.Request) { + req.SetQueryParam("uri", src.Path) + req.SetQueryParam("folder_summary", "true") + }, &ds) + if err == nil && ds.FolderSummary.Size > 0 { + src.Size = ds.FolderSummary.Size + } + } + var thumb model.Thumbnail + if d.EnableThumb && src.Type == 0 { + var t FileThumbResp + err := d.request(http.MethodGet, "/file/thumb", func(req *resty.Request) { + req.SetQueryParam("uri", src.Path) + }, &t) + if err == nil && t.URL != "" { + thumb = model.Thumbnail{ + Thumbnail: t.URL, + } + } + } + return &model.ObjThumb{ + Object: model.Object{ + ID: src.ID, + Path: src.Path, + Name: src.Name, + Size: src.Size, + Modified: src.UpdatedAt, + Ctime: src.CreatedAt, + IsFolder: src.Type == 1, + }, + Thumbnail: thumb, + }, nil + }) +} + +func (d *CloudreveV4) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var url FileUrlResp + err := d.request(http.MethodPost, "/file/url", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{file.GetPath()}, + "download": true, + }) + }, &url) + if err != nil { + return nil, err + } + if len(url.Urls) == 0 { + return nil, errors.New("server returns no url") + } + exp := time.Until(url.Expires) + return &model.Link{ + URL: url.Urls[0].URL, + Expiration: &exp, + }, nil +} + +func (d *CloudreveV4) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "type": "folder", + "uri": parentDir.GetPath() + "/" + dirName, + "error_on_conflict": true, + }) + }, nil) +} + +func (d *CloudreveV4) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{srcObj.GetPath()}, + "dst": dstDir.GetPath(), + "copy": false, + }) + }, nil) +} + +func (d *CloudreveV4) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "new_name": newName, + "uri": srcObj.GetPath(), + }) + }, nil) + +} + +func (d *CloudreveV4) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return d.request(http.MethodPost, "/file/move", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{srcObj.GetPath()}, + "dst": dstDir.GetPath(), + "copy": true, + }) + }, nil) +} + +func (d *CloudreveV4) Remove(ctx context.Context, obj model.Obj) error { + return d.request(http.MethodDelete, "/file", func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{obj.GetPath()}, + "unlink": false, + "skip_soft_delete": true, + }) + }, nil) +} + +func (d *CloudreveV4) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if file.GetSize() == 0 { + // 空文件使用新建文件方法,避免上传卡锁 + return d.request(http.MethodPost, "/file/create", func(req *resty.Request) { + req.SetBody(base.Json{ + "type": "file", + "uri": dstDir.GetPath() + "/" + file.GetName(), + "error_on_conflict": true, + }) + }, nil) + } + var p StoragePolicy + var r FileResp + var u FileUploadResp + var err error + params := map[string]string{ + "page_size": "10", + "uri": dstDir.GetPath(), + "order_by": "created_at", + "order_direction": "asc", + "page": "0", + } + err = d.request(http.MethodGet, "/file", func(req *resty.Request) { + req.SetQueryParams(params) + }, &r) + if err != nil { + return err + } + p = r.StoragePolicy + body := base.Json{ + "uri": dstDir.GetPath() + "/" + file.GetName(), + "size": file.GetSize(), + "policy_id": p.ID, + "last_modified": file.ModTime().UnixMilli(), + "mime_type": "", + } + if d.EnableVersionUpload { + body["entity_type"] = "version" + } + err = d.request(http.MethodPut, "/file/upload", func(req *resty.Request) { + req.SetBody(body) + }, &u) + if err != nil { + return err + } + if u.StoragePolicy.Relay { + err = d.upLocal(ctx, file, u, up) + } else { + switch u.StoragePolicy.Type { + case "local": + err = d.upLocal(ctx, file, u, up) + case "remote": + err = d.upRemote(ctx, file, u, up) + case "onedrive": + err = d.upOneDrive(ctx, file, u, up) + case "s3": + err = d.upS3(ctx, file, u, up) + default: + return errs.NotImplement + } + } + if err != nil { + // 删除失败的会话 + _ = d.request(http.MethodDelete, "/file/upload", func(req *resty.Request) { + req.SetBody(base.Json{ + "id": u.SessionID, + "uri": u.URI, + }) + }, nil) + return err + } + return nil +} + +func (d *CloudreveV4) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *CloudreveV4) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *CloudreveV4) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*CloudreveV4)(nil) diff --git a/drivers/cloudreve_v4/meta.go b/drivers/cloudreve_v4/meta.go new file mode 100644 index 00000000000..bfaa14f81e4 --- /dev/null +++ b/drivers/cloudreve_v4/meta.go @@ -0,0 +1,44 @@ +package cloudreve_v4 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // driver.RootID + // define other + Address string `json:"address" required:"true"` + Username string `json:"username"` + Password string `json:"password"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + CustomUA string `json:"custom_ua"` + EnableFolderSize bool `json:"enable_folder_size"` + EnableThumb bool `json:"enable_thumb"` + EnableVersionUpload bool `json:"enable_version_upload"` + OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at" default:"name" required:"true"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc" required:"true"` +} + +var config = driver.Config{ + Name: "Cloudreve V4", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "cloudreve://my", + CheckStatus: true, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &CloudreveV4{} + }) +} diff --git a/drivers/cloudreve_v4/types.go b/drivers/cloudreve_v4/types.go new file mode 100644 index 00000000000..e81226d3da5 --- /dev/null +++ b/drivers/cloudreve_v4/types.go @@ -0,0 +1,164 @@ +package cloudreve_v4 + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Object struct { + model.Object + StoragePolicy StoragePolicy +} + +type Resp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data any `json:"data"` +} + +type BasicConfigResp struct { + InstanceID string `json:"instance_id"` + // Title string `json:"title"` + // Themes string `json:"themes"` + // DefaultTheme string `json:"default_theme"` + User struct { + ID string `json:"id"` + // Nickname string `json:"nickname"` + // CreatedAt time.Time `json:"created_at"` + // Anonymous bool `json:"anonymous"` + Group struct { + ID string `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` + } `json:"group"` + } `json:"user"` + // Logo string `json:"logo"` + // LogoLight string `json:"logo_light"` + // CaptchaReCaptchaKey string `json:"captcha_ReCaptchaKey"` + CaptchaType string `json:"captcha_type"` // support 'normal' only + // AppPromotion bool `json:"app_promotion"` +} + +type SiteLoginConfigResp struct { + LoginCaptcha bool `json:"login_captcha"` + Authn bool `json:"authn"` +} + +type PrepareLoginResp struct { + WebauthnEnabled bool `json:"webauthn_enabled"` + PasswordEnabled bool `json:"password_enabled"` +} + +type CaptchaResp struct { + Image string `json:"image"` + Ticket string `json:"ticket"` +} + +type Token struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + AccessExpires time.Time `json:"access_expires"` + RefreshExpires time.Time `json:"refresh_expires"` +} + +type TokenResponse struct { + User struct { + ID string `json:"id"` + // Email string `json:"email"` + // Nickname string `json:"nickname"` + Status string `json:"status"` + // CreatedAt time.Time `json:"created_at"` + Group struct { + ID string `json:"id"` + Name string `json:"name"` + Permission string `json:"permission"` + // DirectLinkBatchSize int `json:"direct_link_batch_size"` + // TrashRetention int `json:"trash_retention"` + } `json:"group"` + // Language string `json:"language"` + } `json:"user"` + Token Token `json:"token"` +} + +type File struct { + Type int `json:"type"` // 0: file, 1: folder + ID string `json:"id"` + Name string `json:"name"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + Size int64 `json:"size"` + Metadata interface{} `json:"metadata"` + Path string `json:"path"` + Capability string `json:"capability"` + Owned bool `json:"owned"` + PrimaryEntity string `json:"primary_entity"` +} + +type StoragePolicy struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + MaxSize int64 `json:"max_size"` + Relay bool `json:"relay,omitempty"` +} + +type Pagination struct { + Page int `json:"page"` + PageSize int `json:"page_size"` + IsCursor bool `json:"is_cursor"` + NextToken string `json:"next_token,omitempty"` +} + +type Props struct { + Capability string `json:"capability"` + MaxPageSize int `json:"max_page_size"` + OrderByOptions []string `json:"order_by_options"` + OrderDirectionOptions []string `json:"order_direction_options"` +} + +type FileResp struct { + Files []File `json:"files"` + Parent File `json:"parent"` + Pagination Pagination `json:"pagination"` + Props Props `json:"props"` + ContextHint string `json:"context_hint"` + MixedType bool `json:"mixed_type"` + StoragePolicy StoragePolicy `json:"storage_policy"` +} + +type FileUrlResp struct { + Urls []struct { + URL string `json:"url"` + } `json:"urls"` + Expires time.Time `json:"expires"` +} + +type FileUploadResp struct { + // UploadID string `json:"upload_id"` + SessionID string `json:"session_id"` + ChunkSize int64 `json:"chunk_size"` + Expires int64 `json:"expires"` + StoragePolicy StoragePolicy `json:"storage_policy"` + URI string `json:"uri"` + CompleteURL string `json:"completeURL,omitempty"` // for S3-like + CallbackSecret string `json:"callback_secret,omitempty"` // for S3-like, OneDrive + UploadUrls []string `json:"upload_urls,omitempty"` // for not-local + Credential string `json:"credential,omitempty"` // for local +} + +type FileThumbResp struct { + URL string `json:"url"` + Expires time.Time `json:"expires"` +} + +type FolderSummaryResp struct { + File + FolderSummary struct { + Size int64 `json:"size"` + Files int64 `json:"files"` + Folders int64 `json:"folders"` + Completed bool `json:"completed"` + CalculatedAt time.Time `json:"calculated_at"` + } `json:"folder_summary"` +} diff --git a/drivers/cloudreve_v4/util.go b/drivers/cloudreve_v4/util.go new file mode 100644 index 00000000000..cf2337f279b --- /dev/null +++ b/drivers/cloudreve_v4/util.go @@ -0,0 +1,476 @@ +package cloudreve_v4 + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + jsoniter "github.com/json-iterator/go" +) + +// do others that not defined in Driver interface + +func (d *CloudreveV4) getUA() string { + if d.CustomUA != "" { + return d.CustomUA + } + return base.UserAgent +} + +func (d *CloudreveV4) request(method string, path string, callback base.ReqCallback, out any) error { + if d.ref != nil { + return d.ref.request(method, path, callback, out) + } + u := d.Address + "/api/v4" + path + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "User-Agent": d.getUA(), + }) + if d.AccessToken != "" { + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + } + + var r Resp + req.SetResult(&r) + + if callback != nil { + callback(req) + } + + resp, err := req.Execute(method, u) + if err != nil { + return err + } + if !resp.IsSuccess() { + return errors.New(resp.String()) + } + + if r.Code != 0 { + if r.Code == 401 && d.RefreshToken != "" && path != "/session/token/refresh" { + // try to refresh token + err = d.refreshToken() + if err != nil { + return err + } + return d.request(method, path, callback, out) + } + return errors.New(r.Msg) + } + + if out != nil && r.Data != nil { + var marshal []byte + marshal, err = json.Marshal(r.Data) + if err != nil { + return err + } + err = json.Unmarshal(marshal, out) + if err != nil { + return err + } + } + + return nil +} + +func (d *CloudreveV4) login() error { + var siteConfig SiteLoginConfigResp + err := d.request(http.MethodGet, "/site/config/login", nil, &siteConfig) + if err != nil { + return err + } + if !siteConfig.Authn { + return errors.New("authn not support") + } + var prepareLogin PrepareLoginResp + err = d.request(http.MethodGet, "/session/prepare?email="+d.Addition.Username, nil, &prepareLogin) + if err != nil { + return err + } + if !prepareLogin.PasswordEnabled { + return errors.New("password not enabled") + } + if prepareLogin.WebauthnEnabled { + return errors.New("webauthn not support") + } + for range 5 { + err = d.doLogin(siteConfig.LoginCaptcha) + if err == nil { + break + } + if err.Error() != "CAPTCHA not match." { + break + } + } + return err +} + +func (d *CloudreveV4) doLogin(needCaptcha bool) error { + var err error + loginBody := base.Json{ + "email": d.Username, + "password": d.Password, + } + if needCaptcha { + var config BasicConfigResp + err = d.request(http.MethodGet, "/site/config/basic", nil, &config) + if err != nil { + return err + } + if config.CaptchaType != "normal" { + return fmt.Errorf("captcha type %s not support", config.CaptchaType) + } + var captcha CaptchaResp + err = d.request(http.MethodGet, "/site/captcha", nil, &captcha) + if err != nil { + return err + } + if !strings.HasPrefix(captcha.Image, "data:image/png;base64,") { + return errors.New("can not get captcha") + } + loginBody["ticket"] = captcha.Ticket + i := strings.Index(captcha.Image, ",") + dec := base64.NewDecoder(base64.StdEncoding, strings.NewReader(captcha.Image[i+1:])) + vRes, err := base.RestyClient.R().SetMultipartField( + "image", "validateCode.png", "image/png", dec). + Post(setting.GetStr(conf.OcrApi)) + if err != nil { + return err + } + if jsoniter.Get(vRes.Body(), "status").ToInt() != 200 { + return errors.New("ocr error:" + jsoniter.Get(vRes.Body(), "msg").ToString()) + } + captchaCode := jsoniter.Get(vRes.Body(), "result").ToString() + if captchaCode == "" { + return errors.New("ocr error: empty result") + } + loginBody["captcha"] = captchaCode + } + var token TokenResponse + err = d.request(http.MethodPost, "/session/token", func(req *resty.Request) { + req.SetBody(loginBody) + }, &token) + if err != nil { + return err + } + d.AccessToken, d.RefreshToken = token.Token.AccessToken, token.Token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *CloudreveV4) refreshToken() error { + var token Token + if token.RefreshToken == "" { + if d.Username != "" { + err := d.login() + if err != nil { + return fmt.Errorf("cannot login to get refresh token, error: %s", err) + } + } + return nil + } + err := d.request(http.MethodPost, "/session/token/refresh", func(req *resty.Request) { + req.SetBody(base.Json{ + "refresh_token": d.RefreshToken, + }) + }, &token) + if err != nil { + return err + } + d.AccessToken, d.RefreshToken = token.AccessToken, token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *CloudreveV4) upLocal(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + if DEFAULT == 0 { + // support relay + DEFAULT = file.GetSize() + } + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[CloudreveV4-Local] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + err = d.request(http.MethodPost, "/file/upload/"+u.SessionID+"/"+strconv.Itoa(chunk), func(req *resty.Request) { + req.SetHeader("Content-Type", "application/octet-stream") + req.SetContentLength(true) + req.SetHeader("Content-Length", strconv.FormatInt(byteSize, 10)) + req.SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + req.AddRetryCondition(func(r *resty.Response, err error) bool { + if err != nil { + return true + } + if r.IsError() { + return true + } + var retryResp Resp + jErr := base.RestyClient.JSONUnmarshal(r.Body(), &retryResp) + if jErr != nil { + return true + } + if retryResp.Code != 0 { + return true + } + return false + }) + }, nil) + if err != nil { + return err + } + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + chunk++ + } + return nil +} + +func (d *CloudreveV4) upRemote(ctx context.Context, file model.FileStreamer, u FileUploadResp, up driver.UpdateProgress) error { + uploadUrl := u.UploadUrls[0] + credential := u.Credential + var finish int64 = 0 + var chunk int = 0 + DEFAULT := int64(u.ChunkSize) + retryCount := 0 + maxRetries := 3 + for finish < file.GetSize() { + if utils.IsCanceled(ctx) { + return ctx.Err() + } + left := file.GetSize() - finish + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[CloudreveV4-Remote] upload range: %d-%d/%d", finish, finish+byteSize-1, file.GetSize()) + byteData := make([]byte, byteSize) + n, err := io.ReadFull(file, byteData) + utils.Log.Debug(err, n) + if err != nil { + return err + } + req, err := http.NewRequest("POST", uploadUrl+"?chunk="+strconv.Itoa(chunk), + driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.Header.Set("Authorization", fmt.Sprint(credential)) + req.Header.Set("User-Agent", d.getUA()) + err = func() error { + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return errors.New(res.Status) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return err + } + var up Resp + err = json.Unmarshal(body, &up) + if err != nil { + return err + } + if up.Code != 0 { + return errors.New(up.Msg) + } + return nil + }() + if err == nil { + retryCount = 0 + finish += byteSize + up(float64(finish) * 100 / float64(file.GetSize())) + chunk++ + } else { + retryCount++ + if retryCount > maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors, error: %s", maxRetries, err) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< maxRetries { + return fmt.Errorf("upload failed after %d retries due to server errors", maxRetries) + } + backoff := time.Duration(1<") + for i, etag := range etags { + bodyBuilder.WriteString(fmt.Sprintf( + `%d%s`, + i+1, // PartNumber 从 1 开始 + etag, + )) + } + bodyBuilder.WriteString("") + req, err := http.NewRequest( + "POST", + u.CompleteURL, + strings.NewReader(bodyBuilder.String()), + ) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/xml") + req.Header.Set("User-Agent", d.getUA()) + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + body, _ := io.ReadAll(res.Body) + return fmt.Errorf("up status: %d, error: %s", res.StatusCode, string(body)) + } + + // 上传成功发送回调请求 + return d.request(http.MethodPost, "/callback/s3/"+u.SessionID+"/"+u.CallbackSecret, func(req *resty.Request) { + req.SetBody("{}") + }, nil) +} diff --git a/drivers/crypt/driver.go b/drivers/crypt/driver.go index d8783b6ea14..2330fb9782b 100644 --- a/drivers/crypt/driver.go +++ b/drivers/crypt/driver.go @@ -3,7 +3,6 @@ package crypt import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" stdpath "path" "regexp" @@ -14,6 +13,8 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" @@ -124,6 +125,9 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ //filter illegal files continue } + if !d.ShowHidden && strings.HasPrefix(name, ".") { + continue + } objRes := model.Object{ Name: name, Size: 0, @@ -145,6 +149,9 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ //filter illegal files continue } + if !d.ShowHidden && strings.HasPrefix(name, ".") { + continue + } objRes := model.Object{ Name: name, Size: size, @@ -154,7 +161,11 @@ func (d *Crypt) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ // discarding hash as it's encrypted } if d.Thumbnail && thumb == "" { - thumb = utils.EncodePath(common.GetApiUrl(nil) + stdpath.Join("/d", args.ReqPath, ".thumbnails", name+".webp"), true) + thumbPath := stdpath.Join(args.ReqPath, ".thumbnails", name+".webp") + thumb = fmt.Sprintf("%s/d%s?sign=%s", + common.GetApiUrl(common.GetHttpReq(ctx)), + utils.EncodePath(thumbPath, true), + sign.Sign(thumbPath)) } if !ok && !d.Thumbnail { result = append(result, &objRes) @@ -252,19 +263,13 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } rrc := remoteLink.RangeReadCloser if len(remoteLink.URL) > 0 { - - rangedRemoteLink := &model.Link{ - URL: remoteLink.URL, - Header: remoteLink.Header, - } - var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, rangedRemoteLink) + var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, remoteLink) if err != nil { return nil, err } rrc = converted } if rrc != nil { - //remoteRangeReader, err := remoteReader, err := rrc.RangeRead(ctx, http_range.Range{Start: underlyingOffset, Length: length}) remoteClosers.AddClosers(rrc.GetClosers()) if err != nil { @@ -277,7 +282,6 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( if err != nil { return nil, err } - //remoteClosers.Add(remoteLink.MFile) //keep reuse same MFile and close at last. remoteClosers.Add(remoteLink.MFile) return io.NopCloser(remoteLink.MFile), nil @@ -296,7 +300,6 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: remoteClosers} resultLink := &model.Link{ - Header: remoteLink.Header, RangeReadCloser: resultRangeReadCloser, Expiration: remoteLink.Expiration, } @@ -383,10 +386,11 @@ func (d *Crypt) Put(ctx context.Context, dstDir model.Obj, streamer model.FileSt Modified: streamer.ModTime(), IsFolder: streamer.IsDir(), }, - Reader: wrappedIn, - Mimetype: "application/octet-stream", - WebPutAsTask: streamer.NeedStore(), - Exist: streamer.GetExist(), + Reader: wrappedIn, + Mimetype: "application/octet-stream", + WebPutAsTask: streamer.NeedStore(), + ForceStreamUpload: true, + Exist: streamer.GetExist(), } err = op.Put(ctx, d.remoteStorage, dstDirActualPath, streamOut, up, false) if err != nil { diff --git a/drivers/crypt/meta.go b/drivers/crypt/meta.go index ffa4af71bdc..0878f63869f 100644 --- a/drivers/crypt/meta.go +++ b/drivers/crypt/meta.go @@ -13,14 +13,16 @@ type Addition struct { FileNameEnc string `json:"filename_encryption" type:"select" required:"true" options:"off,standard,obfuscate" default:"off"` DirNameEnc string `json:"directory_name_encryption" type:"select" required:"true" options:"false,true" default:"false"` - RemotePath string `json:"remote_path" required:"true" help:"This is where the encrypted data stores"` + RemotePath string `json:"remote_path" required:"true" help:"AList mounted folder path used to store encrypted data, e.g. /my-storage/secret"` Password string `json:"password" required:"true" confidential:"true" help:"the main password"` Salt string `json:"salt" confidential:"true" help:"If you don't know what is salt, treat it as a second password. Optional but recommended"` EncryptedSuffix string `json:"encrypted_suffix" required:"true" default:".bin" help:"for advanced user only! encrypted files will have this suffix"` FileNameEncoding string `json:"filename_encoding" type:"select" required:"true" options:"base64,base32,base32768" default:"base64" help:"for advanced user only!"` - Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + Thumbnail bool `json:"thumbnail" required:"true" default:"false" help:"enable thumbnail which pre-generated under .thumbnails folder"` + + ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` } var config = driver.Config{ diff --git a/drivers/darkibox/driver.go b/drivers/darkibox/driver.go new file mode 100644 index 00000000000..0e3f2e5065b --- /dev/null +++ b/drivers/darkibox/driver.go @@ -0,0 +1,299 @@ +package darkibox + +import ( + "context" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" +) + +type Darkibox struct { + model.Storage + Addition +} + +func (d *Darkibox) Config() driver.Config { + return config +} + +func (d *Darkibox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Darkibox) Init(ctx context.Context) error { + if d.APIKey == "" { + return fmt.Errorf("API key is required") + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + + // Verify API key by calling account/info + var account accountInfoResult + if err := d.callAPI(ctx, "/account/info", nil, &account); err != nil { + return fmt.Errorf("failed to verify API key: %w", err) + } + + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Darkibox) Drop(ctx context.Context) error { + return nil +} + +func (d *Darkibox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = folderIDFromObjID(dir.GetID()) + } + + var objects []model.Obj + + // List sub-folders via /api/folder/list + var folders folderListResult + if err := d.callAPI(ctx, "/folder/list", map[string]string{ + "fld_id": fldIDStr(folderID), + }, &folders); err != nil { + return nil, fmt.Errorf("list folders failed: %w", err) + } + for _, f := range folders.Folders { + objects = append(objects, &model.Object{ + ID: encodeFolderID(f.FldID), + Name: f.Name, + IsFolder: true, + }) + } + + // List files via /api/file/list (paginated) + page := 1 + for { + var files fileListResult + if err := d.callAPI(ctx, "/file/list", map[string]string{ + "fld_id": fldIDStr(folderID), + "per_page": "200", + "page": strconv.Itoa(page), + }, &files); err != nil { + return nil, fmt.Errorf("list files failed: %w", err) + } + + for _, f := range files.Files { + modified := time.Now() + if f.Uploaded != "" { + if t, err := time.Parse("2006-01-02 15:04:05", f.Uploaded); err == nil { + modified = t + } + } + name := f.Name + if name == "" { + name = f.Title + } + objects = append(objects, &model.Object{ + ID: encodeFileID(f.FileCode), + Name: name, + Size: f.Size, + Modified: modified, + IsFolder: false, + }) + } + + // Check if there are more pages + if len(files.Files) < 200 { + break + } + page++ + } + + return objects, nil +} + +func (d *Darkibox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + + fileCode := fileCodeFromObjID(file.GetID()) + if fileCode == "" { + return nil, fmt.Errorf("empty file code") + } + + var result directLinkResult + if err := d.callAPI(ctx, "/file/direct_link", map[string]string{ + "file_code": fileCode, + }, &result); err != nil { + return nil, fmt.Errorf("failed to get direct link: %w", err) + } + + // Find the original quality version, fall back to first available + var dlURL string + for _, v := range result.Versions { + if v.Name == "o" { + dlURL = v.URL + break + } + } + if dlURL == "" && len(result.Versions) > 0 { + dlURL = result.Versions[0].URL + } + if dlURL == "" { + return nil, fmt.Errorf("no download URL available for file %s", fileCode) + } + + return &model.Link{ + URL: dlURL, + }, nil +} + +func (d *Darkibox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + parentID := d.RootFolderID + if parentDir.GetID() != "" { + parentID = folderIDFromObjID(parentDir.GetID()) + } + + var result folderCreateResult + if err := d.callAPI(ctx, "/folder/create", map[string]string{ + "name": dirName, + "parent_id": fldIDStr(parentID), + }, &result); err != nil { + return nil, fmt.Errorf("create folder failed: %w", err) + } + + return &model.Object{ + ID: encodeFolderID(result.FldID), + Name: dirName, + IsFolder: true, + }, nil +} + +func (d *Darkibox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, errs.NotImplement + } + + fileCode := fileCodeFromObjID(srcObj.GetID()) + if fileCode == "" { + return nil, fmt.Errorf("empty file code") + } + + dstFolderID := d.RootFolderID + if dstDir.GetID() != "" { + dstFolderID = folderIDFromObjID(dstDir.GetID()) + } + + if err := d.callAPI(ctx, "/file/move", map[string]string{ + "file_code": fileCode, + "to_folder": fldIDStr(dstFolderID), + }, nil); err != nil { + return nil, fmt.Errorf("move file failed: %w", err) + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: false, + }, nil +} + +func (d *Darkibox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Remove(ctx context.Context, obj model.Obj) error { + if obj.IsDir() { + folderID := folderIDFromObjID(obj.GetID()) + return d.callAPI(ctx, "/folder/delete", map[string]string{ + "fld_id": folderID, + }, nil) + } + + fileCode := fileCodeFromObjID(obj.GetID()) + return d.callAPI(ctx, "/file/delete", map[string]string{ + "file_code": fileCode, + }, nil) +} + +func (d *Darkibox) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + // Step 1: Get the upload server URL + var server uploadServerResult + if err := d.callAPI(ctx, "/upload/server", nil, &server); err != nil { + return nil, fmt.Errorf("get upload server failed: %w", err) + } + if server.URL == "" { + return nil, fmt.Errorf("no upload server URL returned") + } + + // Step 2: Upload the file to the upload server + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) + + res, err := base.RestyClient.R(). + SetContext(ctx). + SetMultipartField("file", file.GetName(), "", reader). + SetMultipartFormData(map[string]string{ + "key": d.APIKey, + "fld_id": fldIDStr(folderID), + }). + Post(server.URL) + if err != nil { + return nil, fmt.Errorf("upload failed: %w", err) + } + if res.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("upload failed: http %d", res.StatusCode()) + } + + // Try to parse upload response to get the file code + var uploadResp uploadResult + if err := base.RestyClient.JSONUnmarshal(res.Body(), &uploadResp); err == nil && len(uploadResp.Files) > 0 { + uf := uploadResp.Files[0] + return &model.Object{ + ID: encodeFileID(uf.FileCode), + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil + } + + return &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Darkibox) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Darkibox) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Darkibox)(nil) diff --git a/drivers/darkibox/meta.go b/drivers/darkibox/meta.go new file mode 100644 index 00000000000..a09707707fd --- /dev/null +++ b/drivers/darkibox/meta.go @@ -0,0 +1,27 @@ +package darkibox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APIKey string `json:"api_key" required:"true" help:"API key from your Darkibox account"` +} + +var config = driver.Config{ + Name: "Darkibox", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Darkibox{} + }) +} diff --git a/drivers/darkibox/types.go b/drivers/darkibox/types.go new file mode 100644 index 00000000000..9f193f97e4f --- /dev/null +++ b/drivers/darkibox/types.go @@ -0,0 +1,79 @@ +package darkibox + +import "encoding/json" + +// apiResponse is the common wrapper for all Darkibox API responses. +type apiResponse struct { + Msg string `json:"msg"` + Result json.RawMessage `json:"result"` + ServerTime string `json:"server_time"` + Status int `json:"status"` +} + +// accountInfoResult represents the result of /api/account/info +type accountInfoResult struct { + Email string `json:"email"` + Balance string `json:"balance"` + StorageUsed string `json:"storage_used"` +} + +// fileListResult represents the result of /api/file/list +type fileListResult struct { + Results int `json:"results"` + ResultsTotal int `json:"results_total"` + Files []fileItem `json:"files"` +} + +type fileItem struct { + FileCode string `json:"file_code"` + Name string `json:"name"` + Title string `json:"title"` + Size int64 `json:"size"` + Uploaded string `json:"uploaded"` + FldID int64 `json:"fld_id"` +} + +// folderListResult represents the result of /api/folder/list +type folderListResult struct { + Folders []folderItem `json:"folders"` +} + +type folderItem struct { + FldID int64 `json:"fld_id"` + Name string `json:"name"` + Code string `json:"code"` +} + +// directLinkResult represents the result of /api/file/direct_link +type directLinkResult struct { + Versions []directLinkVersion `json:"versions"` +} + +type directLinkVersion struct { + Name string `json:"name"` + URL string `json:"url"` +} + +// uploadServerResult represents the result of /api/upload/server +type uploadServerResult struct { + URL string `json:"url"` +} + +// uploadResult represents the response from the upload endpoint +type uploadResult struct { + Files []uploadedFile `json:"files"` +} + +type uploadedFile struct { + FileCode string `json:"filecode"` + URL string `json:"url"` + Name string `json:"name"` + Size int64 `json:"size"` + Status int `json:"status"` +} + +// folderCreateResult represents the result of /api/folder/create +type folderCreateResult struct { + FldID int64 `json:"fld_id"` + Name string `json:"name"` +} diff --git a/drivers/darkibox/util.go b/drivers/darkibox/util.go new file mode 100644 index 00000000000..0032abab106 --- /dev/null +++ b/drivers/darkibox/util.go @@ -0,0 +1,88 @@ +package darkibox + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" +) + +const apiBase = "https://darkibox.com/api" + +// callAPI makes a GET request to the Darkibox API with the given endpoint and params. +// It automatically injects the API key. The result JSON is unmarshalled into out if non-nil. +func (d *Darkibox) callAPI(ctx context.Context, endpoint string, params map[string]string, out any) error { + query := map[string]string{ + "key": d.APIKey, + } + for k, v := range params { + if strings.TrimSpace(v) == "" { + continue + } + query[k] = v + } + + var resp apiResponse + r, err := base.RestyClient.R(). + SetContext(ctx). + SetQueryParams(query). + SetResult(&resp). + Get(apiBase + endpoint) + if err != nil { + return err + } + if r.StatusCode() != http.StatusOK { + return fmt.Errorf("darkibox http error: %d", r.StatusCode()) + } + if resp.Status != 200 { + return fmt.Errorf("darkibox api error: status=%d msg=%s", resp.Status, resp.Msg) + } + if out == nil || len(resp.Result) == 0 || string(resp.Result) == "null" { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("decode darkibox result failed: %w", err) + } + return nil +} + +// fldIDStr converts a folder ID (which may be the root "0") to a string suitable for API params. +func fldIDStr(id string) string { + if id == "" { + return "0" + } + return id +} + +// encodeFolderID prefixes a folder ID so we can distinguish folders from files. +func encodeFolderID(id int64) string { + return "d:" + strconv.FormatInt(id, 10) +} + +// encodeFileID prefixes a file code so we can distinguish files from folders. +func encodeFileID(code string) string { + return "f:" + code +} + +// folderIDFromObjID extracts the numeric folder ID string from an object ID. +func folderIDFromObjID(id string) string { + if strings.HasPrefix(id, "d:") { + return strings.TrimPrefix(id, "d:") + } + if id == "" || id == "/" { + return "0" + } + return id +} + +// fileCodeFromObjID extracts the file code from an object ID. +func fileCodeFromObjID(id string) string { + if strings.HasPrefix(id, "f:") { + return strings.TrimPrefix(id, "f:") + } + return id +} diff --git a/drivers/doubao/driver.go b/drivers/doubao/driver.go new file mode 100644 index 00000000000..0d421946099 --- /dev/null +++ b/drivers/doubao/driver.go @@ -0,0 +1,271 @@ +package doubao + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" +) + +type Doubao struct { + model.Storage + Addition + *UploadToken + UserId string + uploadThread int +} + +func (d *Doubao) Config() driver.Config { + return config +} + +func (d *Doubao) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Doubao) Init(ctx context.Context) error { + // TODO login / refresh token + //op.MustSaveDriverStorage(d) + uploadThread, err := strconv.Atoi(d.UploadThread) + if err != nil || uploadThread < 1 { + d.uploadThread, d.UploadThread = 3, "3" // Set default value + } else { + d.uploadThread = uploadThread + } + + if d.UserId == "" { + userInfo, err := d.getUserInfo() + if err != nil { + return err + } + + d.UserId = strconv.FormatInt(userInfo.UserID, 10) + } + + if d.UploadToken == nil { + uploadToken, err := d.initUploadToken() + if err != nil { + return err + } + + d.UploadToken = uploadToken + } + + return nil +} + +func (d *Doubao) Drop(ctx context.Context) error { + return nil +} + +func (d *Doubao) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var files []model.Obj + fileList, err := d.getFiles(dir.GetID(), "") + if err != nil { + return nil, err + } + + for _, child := range fileList { + files = append(files, &Object{ + Object: model.Object{ + ID: child.ID, + Path: child.ParentID, + Name: child.Name, + Size: child.Size, + Modified: time.Unix(child.UpdateTime, 0), + Ctime: time.Unix(child.CreateTime, 0), + IsFolder: child.NodeType == 1, + }, + Key: child.Key, + NodeType: child.NodeType, + }) + } + + return files, nil +} + +func (d *Doubao) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var downloadUrl string + + if u, ok := file.(*Object); ok { + switch d.DownloadApi { + case "get_download_info": + var r GetDownloadInfoResp + _, err := d.request("/samantha/aispace/get_download_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "requests": []base.Json{{"node_id": file.GetID()}}, + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.DownloadInfos[0].MainURL + case "get_file_url": + switch u.NodeType { + case VideoType, AudioType: + var r GetVideoFileUrlResp + _, err := d.request("/samantha/media/get_play_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "key": u.Key, + "node_id": file.GetID(), + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.OriginalMediaInfo.MainURL + default: + var r GetFileUrlResp + _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{u.Key}, + "type": FileNodeType[u.NodeType], + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.FileUrls[0].MainURL + } + default: + return nil, errs.NotImplement + } + + // 生成标准的Content-Disposition + contentDisposition := generateContentDisposition(u.Name) + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "User-Agent": []string{UserAgent}, + "Content-Disposition": []string{contentDisposition}, + }, + }, nil + } + + return nil, errors.New("can't convert obj to URL") +} + +func (d *Doubao) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + var r UploadNodeResp + _, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "node_list": []base.Json{ + { + "local_id": uuid.New().String(), + "name": dirName, + "parent_id": parentDir.GetID(), + "node_type": 1, + }, + }, + }) + }, &r) + return err +} + +func (d *Doubao) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + var r UploadNodeResp + _, err := d.request("/samantha/aispace/move_node", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "node_list": []base.Json{ + {"id": srcObj.GetID()}, + }, + "current_parent_id": srcObj.GetPath(), + "target_parent_id": dstDir.GetID(), + }) + }, &r) + return err +} + +func (d *Doubao) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + var r BaseResp + _, err := d.request("/samantha/aispace/rename_node", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "node_id": srcObj.GetID(), + "node_name": newName, + }) + }, &r) + return err +} + +func (d *Doubao) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *Doubao) Remove(ctx context.Context, obj model.Obj) error { + var r BaseResp + _, err := d.request("/samantha/aispace/delete_node", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{"node_list": []base.Json{{"id": obj.GetID()}}}) + }, &r) + return err +} + +func (d *Doubao) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // 根据MIME类型确定数据类型 + mimetype := file.GetMimetype() + dataType := FileDataType + + switch { + case strings.HasPrefix(mimetype, "video/"): + dataType = VideoDataType + case strings.HasPrefix(mimetype, "audio/"): + dataType = VideoDataType // 音频与视频使用相同的处理方式 + case strings.HasPrefix(mimetype, "image/"): + dataType = ImgDataType + } + + // 获取上传配置 + uploadConfig := UploadConfig{} + if err := d.getUploadConfig(&uploadConfig, dataType, file); err != nil { + return nil, err + } + + // 根据文件大小选择上传方式 + if file.GetSize() <= 1*utils.MB { // 小于1MB,使用普通模式上传 + return d.Upload(&uploadConfig, dstDir, file, up, dataType) + } + // 大文件使用分片上传 + return d.UploadByMultipart(ctx, &uploadConfig, file.GetSize(), dstDir, file, up, dataType) +} + +func (d *Doubao) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Doubao) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Doubao) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Doubao) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *Doubao) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Doubao)(nil) diff --git a/drivers/doubao/meta.go b/drivers/doubao/meta.go new file mode 100644 index 00000000000..7735e5ff0b0 --- /dev/null +++ b/drivers/doubao/meta.go @@ -0,0 +1,36 @@ +package doubao + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + // driver.RootPath + driver.RootID + // define other + Cookie string `json:"cookie" type:"text"` + UploadThread string `json:"upload_thread" default:"3"` + DownloadApi string `json:"download_api" type:"select" options:"get_file_url,get_download_info" default:"get_file_url"` +} + +var config = driver.Config{ + Name: "Doubao", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Doubao{} + }) +} diff --git a/drivers/doubao/types.go b/drivers/doubao/types.go new file mode 100644 index 00000000000..ae747f887cf --- /dev/null +++ b/drivers/doubao/types.go @@ -0,0 +1,415 @@ +package doubao + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +type NodeInfoResp struct { + BaseResp + Data struct { + NodeInfo File `json:"node_info"` + Children []File `json:"children"` + NextCursor string `json:"next_cursor"` + HasMore bool `json:"has_more"` + } `json:"data"` +} + +type File struct { + ID string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` // 0: 文件, 1: 文件夹 + Size int64 `json:"size"` + Source int `json:"source"` + NameReviewStatus int `json:"name_review_status"` + ContentReviewStatus int `json:"content_review_status"` + RiskReviewStatus int `json:"risk_review_status"` + ConversationID string `json:"conversation_id"` + ParentID string `json:"parent_id"` + CreateTime int64 `json:"create_time"` + UpdateTime int64 `json:"update_time"` +} + +type GetDownloadInfoResp struct { + BaseResp + Data struct { + DownloadInfos []struct { + NodeID string `json:"node_id"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"download_infos"` + } `json:"data"` +} + +type GetFileUrlResp struct { + BaseResp + Data struct { + FileUrls []struct { + URI string `json:"uri"` + MainURL string `json:"main_url"` + BackURL string `json:"back_url"` + } `json:"file_urls"` + } `json:"data"` +} + +type GetVideoFileUrlResp struct { + BaseResp + Data struct { + MediaType string `json:"media_type"` + MediaInfo []struct { + Meta struct { + Height string `json:"height"` + Width string `json:"width"` + Format string `json:"format"` + Duration float64 `json:"duration"` + CodecType string `json:"codec_type"` + Definition string `json:"definition"` + } `json:"meta"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"media_info"` + OriginalMediaInfo struct { + Meta struct { + Height string `json:"height"` + Width string `json:"width"` + Format string `json:"format"` + Duration float64 `json:"duration"` + CodecType string `json:"codec_type"` + Definition string `json:"definition"` + } `json:"meta"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"original_media_info"` + PosterURL string `json:"poster_url"` + PlayableStatus int `json:"playable_status"` + } `json:"data"` +} + +type UploadNodeResp struct { + BaseResp + Data struct { + NodeList []struct { + LocalID string `json:"local_id"` + ID string `json:"id"` + ParentID string `json:"parent_id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` // 0: 文件, 1: 文件夹 + } `json:"node_list"` + } `json:"data"` +} + +type Object struct { + model.Object + Key string + NodeType int +} + +type UserInfoResp struct { + Data UserInfo `json:"data"` + Message string `json:"message"` +} +type AppUserInfo struct { + BuiAuditInfo string `json:"bui_audit_info"` +} +type AuditInfo struct { +} +type Details struct { +} +type BuiAuditInfo struct { + AuditInfo AuditInfo `json:"audit_info"` + IsAuditing bool `json:"is_auditing"` + AuditStatus int `json:"audit_status"` + LastUpdateTime int `json:"last_update_time"` + UnpassReason string `json:"unpass_reason"` + Details Details `json:"details"` +} +type Connects struct { + Platform string `json:"platform"` + ProfileImageURL string `json:"profile_image_url"` + ExpiredTime int `json:"expired_time"` + ExpiresIn int `json:"expires_in"` + PlatformScreenName string `json:"platform_screen_name"` + UserID int64 `json:"user_id"` + PlatformUID string `json:"platform_uid"` + SecPlatformUID string `json:"sec_platform_uid"` + PlatformAppID int `json:"platform_app_id"` + ModifyTime int `json:"modify_time"` + AccessToken string `json:"access_token"` + OpenID string `json:"open_id"` +} +type OperStaffRelationInfo struct { + HasPassword int `json:"has_password"` + Mobile string `json:"mobile"` + SecOperStaffUserID string `json:"sec_oper_staff_user_id"` + RelationMobileCountryCode int `json:"relation_mobile_country_code"` +} +type UserInfo struct { + AppID int `json:"app_id"` + AppUserInfo AppUserInfo `json:"app_user_info"` + AvatarURL string `json:"avatar_url"` + BgImgURL string `json:"bg_img_url"` + BuiAuditInfo BuiAuditInfo `json:"bui_audit_info"` + CanBeFoundByPhone int `json:"can_be_found_by_phone"` + Connects []Connects `json:"connects"` + CountryCode int `json:"country_code"` + Description string `json:"description"` + DeviceID int `json:"device_id"` + Email string `json:"email"` + EmailCollected bool `json:"email_collected"` + Gender int `json:"gender"` + HasPassword int `json:"has_password"` + HmRegion int `json:"hm_region"` + IsBlocked int `json:"is_blocked"` + IsBlocking int `json:"is_blocking"` + IsRecommendAllowed int `json:"is_recommend_allowed"` + IsVisitorAccount bool `json:"is_visitor_account"` + Mobile string `json:"mobile"` + Name string `json:"name"` + NeedCheckBindStatus bool `json:"need_check_bind_status"` + OdinUserType int `json:"odin_user_type"` + OperStaffRelationInfo OperStaffRelationInfo `json:"oper_staff_relation_info"` + PhoneCollected bool `json:"phone_collected"` + RecommendHintMessage string `json:"recommend_hint_message"` + ScreenName string `json:"screen_name"` + SecUserID string `json:"sec_user_id"` + SessionKey string `json:"session_key"` + UseHmRegion bool `json:"use_hm_region"` + UserCreateTime int `json:"user_create_time"` + UserID int64 `json:"user_id"` + UserIDStr string `json:"user_id_str"` + UserVerified bool `json:"user_verified"` + VerifiedContent string `json:"verified_content"` +} + +// UploadToken 上传令牌配置 +type UploadToken struct { + Alice map[string]UploadAuthToken + Samantha MediaUploadAuthToken +} + +// UploadAuthToken 多种类型的上传配置:图片/文件 +type UploadAuthToken struct { + ServiceID string `json:"service_id"` + UploadPathPrefix string `json:"upload_path_prefix"` + Auth struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` + ExpiredTime time.Time `json:"expired_time"` + CurrentTime time.Time `json:"current_time"` + } `json:"auth"` + UploadHost string `json:"upload_host"` +} + +// MediaUploadAuthToken 媒体上传配置 +type MediaUploadAuthToken struct { + StsToken struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` + ExpiredTime time.Time `json:"expired_time"` + CurrentTime time.Time `json:"current_time"` + } `json:"sts_token"` + UploadInfo struct { + VideoHost string `json:"video_host"` + SpaceName string `json:"space_name"` + } `json:"upload_info"` +} + +type UploadAuthTokenResp struct { + BaseResp + Data UploadAuthToken `json:"data"` +} + +type MediaUploadAuthTokenResp struct { + BaseResp + Data MediaUploadAuthToken `json:"data"` +} + +type ResponseMetadata struct { + RequestID string `json:"RequestId"` + Action string `json:"Action"` + Version string `json:"Version"` + Service string `json:"Service"` + Region string `json:"Region"` + Error struct { + CodeN int `json:"CodeN,omitempty"` + Code string `json:"Code,omitempty"` + Message string `json:"Message,omitempty"` + } `json:"Error,omitempty"` +} + +type UploadConfig struct { + UploadAddress UploadAddress `json:"UploadAddress"` + FallbackUploadAddress FallbackUploadAddress `json:"FallbackUploadAddress"` + InnerUploadAddress InnerUploadAddress `json:"InnerUploadAddress"` + RequestID string `json:"RequestId"` + SDKParam interface{} `json:"SDKParam"` +} + +type UploadConfigResp struct { + ResponseMetadata `json:"ResponseMetadata"` + Result UploadConfig `json:"Result"` +} + +// StoreInfo 存储信息 +type StoreInfo struct { + StoreURI string `json:"StoreUri"` + Auth string `json:"Auth"` + UploadID string `json:"UploadID"` + UploadHeader map[string]interface{} `json:"UploadHeader,omitempty"` + StorageHeader map[string]interface{} `json:"StorageHeader,omitempty"` +} + +// UploadAddress 上传地址信息 +type UploadAddress struct { + StoreInfos []StoreInfo `json:"StoreInfos"` + UploadHosts []string `json:"UploadHosts"` + UploadHeader map[string]interface{} `json:"UploadHeader"` + SessionKey string `json:"SessionKey"` + Cloud string `json:"Cloud"` +} + +// FallbackUploadAddress 备用上传地址 +type FallbackUploadAddress struct { + StoreInfos []StoreInfo `json:"StoreInfos"` + UploadHosts []string `json:"UploadHosts"` + UploadHeader map[string]interface{} `json:"UploadHeader"` + SessionKey string `json:"SessionKey"` + Cloud string `json:"Cloud"` +} + +// UploadNode 上传节点信息 +type UploadNode struct { + Vid string `json:"Vid"` + Vids []string `json:"Vids"` + StoreInfos []StoreInfo `json:"StoreInfos"` + UploadHost string `json:"UploadHost"` + UploadHeader map[string]interface{} `json:"UploadHeader"` + Type string `json:"Type"` + Protocol string `json:"Protocol"` + SessionKey string `json:"SessionKey"` + NodeConfig struct { + UploadMode string `json:"UploadMode"` + } `json:"NodeConfig"` + Cluster string `json:"Cluster"` +} + +// AdvanceOption 高级选项 +type AdvanceOption struct { + Parallel int `json:"Parallel"` + Stream int `json:"Stream"` + SliceSize int `json:"SliceSize"` + EncryptionKey string `json:"EncryptionKey"` +} + +// InnerUploadAddress 内部上传地址 +type InnerUploadAddress struct { + UploadNodes []UploadNode `json:"UploadNodes"` + AdvanceOption AdvanceOption `json:"AdvanceOption"` +} + +// UploadPart 上传分片信息 +type UploadPart struct { + UploadId string `json:"uploadid,omitempty"` + PartNumber string `json:"part_number,omitempty"` + Crc32 string `json:"crc32,omitempty"` + Etag string `json:"etag,omitempty"` + Mode string `json:"mode,omitempty"` +} + +// UploadResp 上传响应体 +type UploadResp struct { + Code int `json:"code"` + ApiVersion string `json:"apiversion"` + Message string `json:"message"` + Data UploadPart `json:"data"` +} + +type VideoCommitUpload struct { + Vid string `json:"Vid"` + VideoMeta struct { + URI string `json:"Uri"` + Height int `json:"Height"` + Width int `json:"Width"` + OriginHeight int `json:"OriginHeight"` + OriginWidth int `json:"OriginWidth"` + Duration float64 `json:"Duration"` + Bitrate int `json:"Bitrate"` + Md5 string `json:"Md5"` + Format string `json:"Format"` + Size int `json:"Size"` + FileType string `json:"FileType"` + Codec string `json:"Codec"` + } `json:"VideoMeta"` + WorkflowInput struct { + TemplateID string `json:"TemplateId"` + } `json:"WorkflowInput"` + GetPosterMode string `json:"GetPosterMode"` +} + +type VideoCommitUploadResp struct { + ResponseMetadata ResponseMetadata `json:"ResponseMetadata"` + Result struct { + RequestID string `json:"RequestId"` + Results []VideoCommitUpload `json:"Results"` + } `json:"Result"` +} + +type CommonResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` // 错误情况下的消息 + Data json.RawMessage `json:"data,omitempty"` // 原始数据,稍后解析 + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + Locale string `json:"locale"` + } `json:"error,omitempty"` +} + +// IsSuccess 判断响应是否成功 +func (r *CommonResp) IsSuccess() bool { + return r.Code == 0 +} + +// GetError 获取错误信息 +func (r *CommonResp) GetError() error { + if r.IsSuccess() { + return nil + } + // 优先使用message字段 + errMsg := r.Message + if errMsg == "" { + errMsg = r.Msg + } + // 如果error对象存在且有详细消息,则使用error中的信息 + if r.Error != nil && r.Error.Message != "" { + errMsg = r.Error.Message + } + + return fmt.Errorf("[doubao] API error (code: %d): %s", r.Code, errMsg) +} + +// UnmarshalData 将data字段解析为指定类型 +func (r *CommonResp) UnmarshalData(v interface{}) error { + if !r.IsSuccess() { + return r.GetError() + } + + if len(r.Data) == 0 { + return nil + } + + return json.Unmarshal(r.Data, v) +} diff --git a/drivers/doubao/util.go b/drivers/doubao/util.go new file mode 100644 index 00000000000..348c0aa0c14 --- /dev/null +++ b/drivers/doubao/util.go @@ -0,0 +1,970 @@ +package doubao + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" + "github.com/google/uuid" + log "github.com/sirupsen/logrus" + "hash/crc32" + "io" + "math" + "math/rand" + "net/http" + "net/url" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +const ( + DirectoryType = 1 + FileType = 2 + LinkType = 3 + ImageType = 4 + PagesType = 5 + VideoType = 6 + AudioType = 7 + MeetingMinutesType = 8 +) + +var FileNodeType = map[int]string{ + 1: "directory", + 2: "file", + 3: "link", + 4: "image", + 5: "pages", + 6: "video", + 7: "audio", + 8: "meeting_minutes", +} + +const ( + BaseURL = "https://www.doubao.com" + FileDataType = "file" + ImgDataType = "image" + VideoDataType = "video" + DefaultChunkSize = int64(5 * 1024 * 1024) // 5MB + MaxRetryAttempts = 3 // 最大重试次数 + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" + Region = "cn-north-1" + UploadTimeout = 3 * time.Minute +) + +// do others that not defined in Driver interface +func (d *Doubao) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + reqUrl := BaseURL + path + req := base.RestyClient.R() + req.SetHeader("Cookie", d.Cookie) + if callback != nil { + callback(req) + } + + var commonResp CommonResp + + res, err := req.Execute(method, reqUrl) + log.Debugln(res.String()) + if err != nil { + return nil, err + } + + body := res.Body() + // 先解析为通用响应 + if err = json.Unmarshal(body, &commonResp); err != nil { + return nil, err + } + // 检查响应是否成功 + if !commonResp.IsSuccess() { + return body, commonResp.GetError() + } + + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func (d *Doubao) getFiles(dirId, cursor string) (resp []File, err error) { + var r NodeInfoResp + + var body = base.Json{ + "node_id": dirId, + } + // 如果有游标,则设置游标和大小 + if cursor != "" { + body["cursor"] = cursor + body["size"] = 50 + } else { + body["need_full_path"] = false + } + + _, err = d.request("/samantha/aispace/node_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &r) + if err != nil { + return nil, err + } + + if r.Data.Children != nil { + resp = r.Data.Children + } + + if r.Data.NextCursor != "-1" { + // 递归获取下一页 + nextFiles, err := d.getFiles(dirId, r.Data.NextCursor) + if err != nil { + return nil, err + } + + resp = append(r.Data.Children, nextFiles...) + } + + return resp, err +} + +func (d *Doubao) getUserInfo() (UserInfo, error) { + var r UserInfoResp + + _, err := d.request("/passport/account/info/v2/", http.MethodGet, nil, &r) + if err != nil { + return UserInfo{}, err + } + + return r.Data, err +} + +// 签名请求 +func (d *Doubao) signRequest(req *resty.Request, method, tokenType, uploadUrl string) error { + parsedUrl, err := url.Parse(uploadUrl) + if err != nil { + return fmt.Errorf("invalid URL format: %w", err) + } + + var accessKeyId, secretAccessKey, sessionToken string + var serviceName string + + if tokenType == VideoDataType { + accessKeyId = d.UploadToken.Samantha.StsToken.AccessKeyID + secretAccessKey = d.UploadToken.Samantha.StsToken.SecretAccessKey + sessionToken = d.UploadToken.Samantha.StsToken.SessionToken + serviceName = "vod" + } else { + accessKeyId = d.UploadToken.Alice[tokenType].Auth.AccessKeyID + secretAccessKey = d.UploadToken.Alice[tokenType].Auth.SecretAccessKey + sessionToken = d.UploadToken.Alice[tokenType].Auth.SessionToken + serviceName = "imagex" + } + + // 当前时间,格式为 ISO8601 + now := time.Now().UTC() + amzDate := now.Format("20060102T150405Z") + dateStamp := now.Format("20060102") + + req.SetHeader("X-Amz-Date", amzDate) + + if sessionToken != "" { + req.SetHeader("X-Amz-Security-Token", sessionToken) + } + + // 计算请求体的SHA256哈希 + var bodyHash string + if req.Body != nil { + bodyBytes, ok := req.Body.([]byte) + if !ok { + return fmt.Errorf("request body must be []byte") + } + + bodyHash = hashSHA256(string(bodyBytes)) + req.SetHeader("X-Amz-Content-Sha256", bodyHash) + } else { + bodyHash = hashSHA256("") + } + + // 创建规范请求 + canonicalURI := parsedUrl.Path + if canonicalURI == "" { + canonicalURI = "/" + } + + // 查询参数按照字母顺序排序 + canonicalQueryString := getCanonicalQueryString(req.QueryParam) + // 规范请求头 + canonicalHeaders, signedHeaders := getCanonicalHeadersFromMap(req.Header) + canonicalRequest := method + "\n" + + canonicalURI + "\n" + + canonicalQueryString + "\n" + + canonicalHeaders + "\n" + + signedHeaders + "\n" + + bodyHash + + algorithm := "AWS4-HMAC-SHA256" + credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", dateStamp, Region, serviceName) + + stringToSign := algorithm + "\n" + + amzDate + "\n" + + credentialScope + "\n" + + hashSHA256(canonicalRequest) + // 计算签名密钥 + signingKey := getSigningKey(secretAccessKey, dateStamp, Region, serviceName) + // 计算签名 + signature := hmacSHA256Hex(signingKey, stringToSign) + // 构建授权头 + authorizationHeader := fmt.Sprintf( + "%s Credential=%s/%s, SignedHeaders=%s, Signature=%s", + algorithm, + accessKeyId, + credentialScope, + signedHeaders, + signature, + ) + + req.SetHeader("Authorization", authorizationHeader) + + return nil +} + +func (d *Doubao) requestApi(url, method, tokenType string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "user-agent": UserAgent, + }) + + if method == http.MethodPost { + req.SetHeader("Content-Type", "text/plain;charset=UTF-8") + } + + if callback != nil { + callback(req) + } + + if resp != nil { + req.SetResult(resp) + } + + // 使用自定义AWS SigV4签名 + err := d.signRequest(req, method, tokenType, url) + if err != nil { + return nil, err + } + + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Doubao) initUploadToken() (*UploadToken, error) { + uploadToken := &UploadToken{ + Alice: make(map[string]UploadAuthToken), + Samantha: MediaUploadAuthToken{}, + } + + fileAuthToken, err := d.getUploadAuthToken(FileDataType) + if err != nil { + return nil, err + } + + imgAuthToken, err := d.getUploadAuthToken(ImgDataType) + if err != nil { + return nil, err + } + + mediaAuthToken, err := d.getSamantaUploadAuthToken() + if err != nil { + return nil, err + } + + uploadToken.Alice[FileDataType] = fileAuthToken + uploadToken.Alice[ImgDataType] = imgAuthToken + uploadToken.Samantha = mediaAuthToken + + return uploadToken, nil +} + +func (d *Doubao) getUploadAuthToken(dataType string) (ut UploadAuthToken, err error) { + var r UploadAuthTokenResp + _, err = d.request("/alice/upload/auth_token", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "scene": "bot_chat", + "data_type": dataType, + }) + }, &r) + + return r.Data, err +} + +func (d *Doubao) getSamantaUploadAuthToken() (mt MediaUploadAuthToken, err error) { + var r MediaUploadAuthTokenResp + _, err = d.request("/samantha/media/get_upload_token", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{}) + }, &r) + + return r.Data, err +} + +// getUploadConfig 获取上传配置信息 +func (d *Doubao) getUploadConfig(upConfig *UploadConfig, dataType string, file model.FileStreamer) error { + tokenType := dataType + // 配置参数函数 + configureParams := func() (string, map[string]string) { + var uploadUrl string + var params map[string]string + // 根据数据类型设置不同的上传参数 + switch dataType { + case VideoDataType: + // 音频/视频类型 - 使用uploadToken.Samantha的配置 + uploadUrl = d.UploadToken.Samantha.UploadInfo.VideoHost + params = map[string]string{ + "Action": "ApplyUploadInner", + "Version": "2020-11-19", + "SpaceName": d.UploadToken.Samantha.UploadInfo.SpaceName, + "FileType": "video", + "IsInner": "1", + "NeedFallback": "true", + "FileSize": strconv.FormatInt(file.GetSize(), 10), + "s": randomString(), + } + case ImgDataType, FileDataType: + // 图片或其他文件类型 - 使用uploadToken.Alice对应配置 + uploadUrl = "https://" + d.UploadToken.Alice[dataType].UploadHost + params = map[string]string{ + "Action": "ApplyImageUpload", + "Version": "2018-08-01", + "ServiceId": d.UploadToken.Alice[dataType].ServiceID, + "NeedFallback": "true", + "FileSize": strconv.FormatInt(file.GetSize(), 10), + "FileExtension": filepath.Ext(file.GetName()), + "s": randomString(), + } + } + return uploadUrl, params + } + + // 获取初始参数 + uploadUrl, params := configureParams() + + tokenRefreshed := false + var configResp UploadConfigResp + + err := d._retryOperation("get upload_config", func() error { + configResp = UploadConfigResp{} + + _, err := d.requestApi(uploadUrl, http.MethodGet, tokenType, func(req *resty.Request) { + req.SetQueryParams(params) + }, &configResp) + if err != nil { + return err + } + + if configResp.ResponseMetadata.Error.Code == "" { + *upConfig = configResp.Result + return nil + } + + // 100028 凭证过期 + if configResp.ResponseMetadata.Error.CodeN == 100028 && !tokenRefreshed { + log.Debugln("[doubao] Upload token expired, re-fetching...") + newToken, err := d.initUploadToken() + if err != nil { + return fmt.Errorf("failed to refresh token: %w", err) + } + + d.UploadToken = newToken + tokenRefreshed = true + uploadUrl, params = configureParams() + + return retry.Error{errors.New("token refreshed, retry needed")} + } + + return fmt.Errorf("get upload_config failed: %s", configResp.ResponseMetadata.Error.Message) + }) + + return err +} + +// uploadNode 上传 文件信息 +func (d *Doubao) uploadNode(uploadConfig *UploadConfig, dir model.Obj, file model.FileStreamer, dataType string) (UploadNodeResp, error) { + reqUuid := uuid.New().String() + var key string + var nodeType int + + mimetype := file.GetMimetype() + switch dataType { + case VideoDataType: + key = uploadConfig.InnerUploadAddress.UploadNodes[0].Vid + if strings.HasPrefix(mimetype, "audio/") { + nodeType = AudioType // 音频类型 + } else { + nodeType = VideoType // 视频类型 + } + case ImgDataType: + key = uploadConfig.InnerUploadAddress.UploadNodes[0].StoreInfos[0].StoreURI + nodeType = ImageType // 图片类型 + default: // FileDataType + key = uploadConfig.InnerUploadAddress.UploadNodes[0].StoreInfos[0].StoreURI + nodeType = FileType // 文件类型 + } + + var r UploadNodeResp + _, err := d.request("/samantha/aispace/upload_node", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "node_list": []base.Json{ + { + "local_id": reqUuid, + "parent_id": dir.GetID(), + "name": file.GetName(), + "key": key, + "node_content": base.Json{}, + "node_type": nodeType, + "size": file.GetSize(), + }, + }, + "request_id": reqUuid, + }) + }, &r) + + return r, err +} + +// Upload 普通上传实现 +func (d *Doubao) Upload(config *UploadConfig, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, dataType string) (model.Obj, error) { + data, err := io.ReadAll(file) + if err != nil { + return nil, err + } + + // 计算CRC32 + crc32Hash := crc32.NewIEEE() + crc32Hash.Write(data) + crc32Value := hex.EncodeToString(crc32Hash.Sum(nil)) + + // 构建请求路径 + uploadNode := config.InnerUploadAddress.UploadNodes[0] + storeInfo := uploadNode.StoreInfos[0] + uploadUrl := fmt.Sprintf("https://%s/upload/v1/%s", uploadNode.UploadHost, storeInfo.StoreURI) + + uploadResp := UploadResp{} + + if _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Content-Type": "application/octet-stream", + "Content-Crc32": crc32Value, + "Content-Length": fmt.Sprintf("%d", len(data)), + "Content-Disposition": fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI)), + }) + + req.SetBody(data) + }, &uploadResp); err != nil { + return nil, err + } + + if uploadResp.Code != 2000 { + return nil, fmt.Errorf("upload failed: %s", uploadResp.Message) + } + + uploadNodeResp, err := d.uploadNode(config, dstDir, file, dataType) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: uploadNodeResp.Data.NodeList[0].ID, + Name: uploadNodeResp.Data.NodeList[0].Name, + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +// UploadByMultipart 分片上传 +func (d *Doubao) UploadByMultipart(ctx context.Context, config *UploadConfig, fileSize int64, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress, dataType string) (model.Obj, error) { + // 构建请求路径 + uploadNode := config.InnerUploadAddress.UploadNodes[0] + storeInfo := uploadNode.StoreInfos[0] + uploadUrl := fmt.Sprintf("https://%s/upload/v1/%s", uploadNode.UploadHost, storeInfo.StoreURI) + // 初始化分片上传 + var uploadID string + err := d._retryOperation("Initialize multipart upload", func() error { + var err error + uploadID, err = d.initMultipartUpload(config, uploadUrl, storeInfo) + return err + }) + if err != nil { + return nil, fmt.Errorf("failed to initialize multipart upload: %w", err) + } + // 准备分片参数 + chunkSize := DefaultChunkSize + if config.InnerUploadAddress.AdvanceOption.SliceSize > 0 { + chunkSize = int64(config.InnerUploadAddress.AdvanceOption.SliceSize) + } + totalParts := (fileSize + chunkSize - 1) / chunkSize + // 创建分片信息组 + parts := make([]UploadPart, totalParts) + // 缓存文件 + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, fmt.Errorf("failed to cache file: %w", err) + } + defer tempFile.Close() + up(10.0) // 更新进度 + // 设置并行上传 + threadG, uploadCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, + retry.Attempts(1), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + var partsMutex sync.Mutex + // 并行上传所有分片 + for partIndex := int64(0); partIndex < totalParts; partIndex++ { + if utils.IsCanceled(uploadCtx) { + break + } + partIndex := partIndex + partNumber := partIndex + 1 // 分片编号从1开始 + + threadG.Go(func(ctx context.Context) error { + // 计算此分片的大小和偏移 + offset := partIndex * chunkSize + size := chunkSize + if partIndex == totalParts-1 { + size = fileSize - offset + } + + limitedReader := driver.NewLimitedUploadStream(ctx, io.NewSectionReader(tempFile, offset, size)) + // 读取数据到内存 + data, err := io.ReadAll(limitedReader) + if err != nil { + return fmt.Errorf("failed to read part %d: %w", partNumber, err) + } + // 计算CRC32 + crc32Value := calculateCRC32(data) + // 使用_retryOperation上传分片 + var uploadPart UploadPart + if err = d._retryOperation(fmt.Sprintf("Upload part %d", partNumber), func() error { + var err error + uploadPart, err = d.uploadPart(config, uploadUrl, uploadID, partNumber, data, crc32Value) + return err + }); err != nil { + return fmt.Errorf("part %d upload failed: %w", partNumber, err) + } + // 记录成功上传的分片 + partsMutex.Lock() + parts[partIndex] = UploadPart{ + PartNumber: strconv.FormatInt(partNumber, 10), + Etag: uploadPart.Etag, + Crc32: crc32Value, + } + partsMutex.Unlock() + // 更新进度 + progress := 10.0 + 90.0*float64(threadG.Success()+1)/float64(totalParts) + up(math.Min(progress, 95.0)) + + return nil + }) + } + + if err = threadG.Wait(); err != nil { + return nil, err + } + // 完成上传-分片合并 + if err = d._retryOperation("Complete multipart upload", func() error { + return d.completeMultipartUpload(config, uploadUrl, uploadID, parts) + }); err != nil { + return nil, fmt.Errorf("failed to complete multipart upload: %w", err) + } + // 提交上传 + if err = d._retryOperation("Commit upload", func() error { + return d.commitMultipartUpload(config) + }); err != nil { + return nil, fmt.Errorf("failed to commit upload: %w", err) + } + + up(98.0) // 更新到98% + // 上传节点信息 + var uploadNodeResp UploadNodeResp + + if err = d._retryOperation("Upload node", func() error { + var err error + uploadNodeResp, err = d.uploadNode(config, dstDir, file, dataType) + return err + }); err != nil { + return nil, fmt.Errorf("failed to upload node: %w", err) + } + + up(100.0) // 完成上传 + + return &model.Object{ + ID: uploadNodeResp.Data.NodeList[0].ID, + Name: uploadNodeResp.Data.NodeList[0].Name, + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +// 统一上传请求方法 +func (d *Doubao) uploadRequest(uploadUrl string, method string, storeInfo StoreInfo, callback base.ReqCallback, resp interface{}) ([]byte, error) { + client := resty.New() + client.SetTransport(&http.Transport{ + DisableKeepAlives: true, // 禁用连接复用 + ForceAttemptHTTP2: false, // 强制使用HTTP/1.1 + }) + client.SetTimeout(UploadTimeout) + + req := client.R() + req.SetHeaders(map[string]string{ + "Host": strings.Split(uploadUrl, "/")[2], + "Referer": BaseURL + "/", + "Origin": BaseURL, + "User-Agent": UserAgent, + "X-Storage-U": d.UserId, + "Authorization": storeInfo.Auth, + }) + + if method == http.MethodPost { + req.SetHeader("Content-Type", "text/plain;charset=UTF-8") + } + + if callback != nil { + callback(req) + } + + if resp != nil { + req.SetResult(resp) + } + + res, err := req.Execute(method, uploadUrl) + if err != nil && err != io.EOF { + return nil, fmt.Errorf("upload request failed: %w", err) + } + + return res.Body(), nil +} + +// 初始化分片上传 +func (d *Doubao) initMultipartUpload(config *UploadConfig, uploadUrl string, storeInfo StoreInfo) (uploadId string, err error) { + uploadResp := UploadResp{} + + _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "uploadmode": "part", + "phase": "init", + }) + }, &uploadResp) + + if err != nil { + return uploadId, err + } + + if uploadResp.Code != 2000 { + return uploadId, fmt.Errorf("init upload failed: %s", uploadResp.Message) + } + + return uploadResp.Data.UploadId, nil +} + +// 分片上传实现 +func (d *Doubao) uploadPart(config *UploadConfig, uploadUrl, uploadID string, partNumber int64, data []byte, crc32Value string) (resp UploadPart, err error) { + uploadResp := UploadResp{} + storeInfo := config.InnerUploadAddress.UploadNodes[0].StoreInfos[0] + + _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Content-Type": "application/octet-stream", + "Content-Crc32": crc32Value, + "Content-Length": fmt.Sprintf("%d", len(data)), + "Content-Disposition": fmt.Sprintf("attachment; filename=%s", url.QueryEscape(storeInfo.StoreURI)), + }) + + req.SetQueryParams(map[string]string{ + "uploadid": uploadID, + "part_number": strconv.FormatInt(partNumber, 10), + "phase": "transfer", + }) + + req.SetBody(data) + req.SetContentLength(true) + }, &uploadResp) + + if err != nil { + return resp, err + } + + if uploadResp.Code != 2000 { + return resp, fmt.Errorf("upload part failed: %s", uploadResp.Message) + } else if uploadResp.Data.Crc32 != crc32Value { + return resp, fmt.Errorf("upload part failed: crc32 mismatch, expected %s, got %s", crc32Value, uploadResp.Data.Crc32) + } + + return uploadResp.Data, nil +} + +// 完成分片上传 +func (d *Doubao) completeMultipartUpload(config *UploadConfig, uploadUrl, uploadID string, parts []UploadPart) error { + uploadResp := UploadResp{} + + storeInfo := config.InnerUploadAddress.UploadNodes[0].StoreInfos[0] + + body := _convertUploadParts(parts) + + err := utils.Retry(MaxRetryAttempts, time.Second, func() (err error) { + _, err = d.uploadRequest(uploadUrl, http.MethodPost, storeInfo, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "uploadid": uploadID, + "phase": "finish", + "uploadmode": "part", + }) + req.SetBody(body) + }, &uploadResp) + + if err != nil { + return err + } + // 检查响应状态码 2000 成功 4024 分片合并中 + if uploadResp.Code != 2000 && uploadResp.Code != 4024 { + return fmt.Errorf("finish upload failed: %s", uploadResp.Message) + } + + return err + }) + + if err != nil { + return fmt.Errorf("failed to complete multipart upload: %w", err) + } + + return nil +} + +func (d *Doubao) commitMultipartUpload(uploadConfig *UploadConfig) error { + uploadUrl := d.UploadToken.Samantha.UploadInfo.VideoHost + params := map[string]string{ + "Action": "CommitUploadInner", + "Version": "2020-11-19", + "SpaceName": d.UploadToken.Samantha.UploadInfo.SpaceName, + } + tokenType := VideoDataType + + videoCommitUploadResp := VideoCommitUploadResp{} + + jsonBytes, err := json.Marshal(base.Json{ + "SessionKey": uploadConfig.InnerUploadAddress.UploadNodes[0].SessionKey, + "Functions": []base.Json{}, + }) + if err != nil { + return fmt.Errorf("failed to marshal request data: %w", err) + } + + _, err = d.requestApi(uploadUrl, http.MethodPost, tokenType, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetQueryParams(params) + req.SetBody(jsonBytes) + + }, &videoCommitUploadResp) + if err != nil { + return err + } + + return nil +} + +// 计算CRC32 +func calculateCRC32(data []byte) string { + hash := crc32.NewIEEE() + hash.Write(data) + return hex.EncodeToString(hash.Sum(nil)) +} + +// _retryOperation 操作重试 +func (d *Doubao) _retryOperation(operation string, fn func() error) error { + return retry.Do( + fn, + retry.Attempts(MaxRetryAttempts), + retry.Delay(500*time.Millisecond), + retry.DelayType(retry.BackOffDelay), + retry.MaxJitter(200*time.Millisecond), + retry.OnRetry(func(n uint, err error) { + log.Debugf("[doubao] %s retry #%d: %v", operation, n+1, err) + }), + ) +} + +// _convertUploadParts 将分片信息转换为字符串 +func _convertUploadParts(parts []UploadPart) string { + if len(parts) == 0 { + return "" + } + + var result strings.Builder + + for i, part := range parts { + if i > 0 { + result.WriteString(",") + } + result.WriteString(fmt.Sprintf("%s:%s", part.PartNumber, part.Crc32)) + } + + return result.String() +} + +// 获取规范查询字符串 +func getCanonicalQueryString(query url.Values) string { + if len(query) == 0 { + return "" + } + + keys := make([]string, 0, len(query)) + for k := range query { + keys = append(keys, k) + } + sort.Strings(keys) + + parts := make([]string, 0, len(keys)) + for _, k := range keys { + values := query[k] + for _, v := range values { + parts = append(parts, urlEncode(k)+"="+urlEncode(v)) + } + } + + return strings.Join(parts, "&") +} + +func urlEncode(s string) string { + s = url.QueryEscape(s) + s = strings.ReplaceAll(s, "+", "%20") + return s +} + +// 获取规范头信息和已签名头列表 +func getCanonicalHeadersFromMap(headers map[string][]string) (string, string) { + // 不可签名的头部列表 + unsignableHeaders := map[string]bool{ + "authorization": true, + "content-type": true, + "content-length": true, + "user-agent": true, + "presigned-expires": true, + "expect": true, + "x-amzn-trace-id": true, + } + headerValues := make(map[string]string) + var signedHeadersList []string + + for k, v := range headers { + if len(v) == 0 { + continue + } + + lowerKey := strings.ToLower(k) + // 检查是否可签名 + if strings.HasPrefix(lowerKey, "x-amz-") || !unsignableHeaders[lowerKey] { + value := strings.TrimSpace(v[0]) + value = strings.Join(strings.Fields(value), " ") + headerValues[lowerKey] = value + signedHeadersList = append(signedHeadersList, lowerKey) + } + } + + sort.Strings(signedHeadersList) + + var canonicalHeadersStr strings.Builder + for _, key := range signedHeadersList { + canonicalHeadersStr.WriteString(key) + canonicalHeadersStr.WriteString(":") + canonicalHeadersStr.WriteString(headerValues[key]) + canonicalHeadersStr.WriteString("\n") + } + + signedHeaders := strings.Join(signedHeadersList, ";") + + return canonicalHeadersStr.String(), signedHeaders +} + +// 计算HMAC-SHA256 +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + h.Write([]byte(data)) + return h.Sum(nil) +} + +// 计算HMAC-SHA256并返回十六进制字符串 +func hmacSHA256Hex(key []byte, data string) string { + return hex.EncodeToString(hmacSHA256(key, data)) +} + +// 计算SHA256哈希并返回十六进制字符串 +func hashSHA256(data string) string { + h := sha256.New() + h.Write([]byte(data)) + return hex.EncodeToString(h.Sum(nil)) +} + +// 获取签名密钥 +func getSigningKey(secretKey, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secretKey), dateStamp) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + kSigning := hmacSHA256(kService, "aws4_request") + return kSigning +} + +// generateContentDisposition 生成符合RFC 5987标准的Content-Disposition头部 +func generateContentDisposition(filename string) string { + // 按照RFC 2047进行编码,用于filename部分 + encodedName := urlEncode(filename) + + // 按照RFC 5987进行编码,用于filename*部分 + encodedNameRFC5987 := encodeRFC5987(filename) + + return fmt.Sprintf("attachment; filename=\"%s\"; filename*=utf-8''%s", + encodedName, encodedNameRFC5987) +} + +// encodeRFC5987 按照RFC 5987规范编码字符串,适用于HTTP头部参数中的非ASCII字符 +func encodeRFC5987(s string) string { + var buf strings.Builder + for _, r := range []byte(s) { + // 根据RFC 5987,只有字母、数字和部分特殊符号可以不编码 + if (r >= 'a' && r <= 'z') || + (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || + r == '-' || r == '.' || r == '_' || r == '~' { + buf.WriteByte(r) + } else { + // 其他字符都需要百分号编码 + fmt.Fprintf(&buf, "%%%02X", r) + } + } + return buf.String() +} + +func randomString() string { + const charset = "0123456789abcdefghijklmnopqrstuvwxyz" + const length = 11 // 11位随机字符串 + + var sb strings.Builder + sb.Grow(length) + + for i := 0; i < length; i++ { + sb.WriteByte(charset[rand.Intn(len(charset))]) + } + + return sb.String() +} diff --git a/drivers/doubao_new/driver.go b/drivers/doubao_new/driver.go new file mode 100644 index 00000000000..af0f9dd2ac0 --- /dev/null +++ b/drivers/doubao_new/driver.go @@ -0,0 +1,598 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type DoubaoNew struct { + model.Storage + Addition + TtLogid string +} + +func (d *DoubaoNew) Config() driver.Config { + return config +} + +func (d *DoubaoNew) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *DoubaoNew) Init(ctx context.Context) error { + // TODO login / refresh token + //op.MustSaveDriverStorage(d) + return nil +} + +func (d *DoubaoNew) Drop(ctx context.Context) error { + return nil +} + +func (d *DoubaoNew) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + nodes, err := d.listAllChildren(ctx, dir.GetID()) + if err != nil { + return nil, err + } + + objs := make([]model.Obj, 0, len(nodes)) + for _, node := range nodes { + size := parseSize(node.Extra.Size) + isFolder := node.Type == 0 + obj := &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: dir.GetID(), + Name: node.Name, + Size: size, + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: isFolder, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + } + objs = append(objs, obj) + } + + return objs, nil +} + +func (d *DoubaoNew) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, ok := file.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + if obj.IsFolder { + return nil, errs.LinkIsDir + } + if args.Type == "preview" || args.Type == "thumb" { + if link, err := d.previewLink(ctx, obj, args); err == nil { + return link, nil + } + } + auth := d.resolveAuthorization() + dpop := d.resolveDpop() + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + + query := url.Values{} + query.Set("authorization", auth) + query.Set("dpop", dpop) + + downloadURL := DownloadBaseURL + "/space/api/box/stream/download/all/" + obj.ObjToken + "/?" + query.Encode() + + headers := http.Header{ + "Referer": []string{"https://www.doubao.com/"}, + "User-Agent": []string{base.UserAgent}, + } + + return &model.Link{ + URL: downloadURL, + Header: headers, + }, nil +} + +func (d *DoubaoNew) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + node, err := d.createFolder(ctx, parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + return &Object{ + Object: model.Object{ + ID: node.NodeToken, + Path: parentDir.GetID(), + Name: node.Name, + Size: parseSize(node.Extra.Size), + Modified: time.Unix(node.EditTime, 0), + Ctime: time.Unix(node.CreateTime, 0), + IsFolder: true, + }, + ObjToken: node.ObjToken, + NodeType: node.NodeType, + ObjType: node.Type, + URL: node.URL, + }, nil +} + +func (d *DoubaoNew) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if dstDir == nil { + return nil, errors.New("nil destination dir") + } + srcToken := srcObj.GetID() + if srcToken == "" { + if obj, ok := srcObj.(*Object); ok { + srcToken = obj.ObjToken + } + } + if srcToken == "" { + return nil, errors.New("missing source token") + } + if err := d.moveObj(ctx, srcToken, dstDir.GetID()); err != nil { + return nil, err + } + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Path = dstDir.GetID() + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if srcObj == nil { + return nil, errors.New("nil source object") + } + if srcObj.IsDir() { + if err := d.renameFolder(ctx, srcObj.GetID(), newName); err != nil { + return nil, err + } + } else { + fileToken := "" + if obj, ok := srcObj.(*Object); ok { + fileToken = obj.ObjToken + } + if fileToken == "" { + fileToken = srcObj.GetID() + } + if err := d.renameFile(ctx, fileToken, newName); err != nil { + return nil, err + } + } + + if obj, ok := srcObj.(*Object); ok { + clone := *obj + clone.Name = newName + return &clone, nil + } + return srcObj, nil +} + +func (d *DoubaoNew) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Remove(ctx context.Context, obj model.Obj) error { + if obj == nil { + return errors.New("nil object") + } + token := obj.GetID() + if token == "" { + if o, ok := obj.(*Object); ok { + token = o.ObjToken + } + } + if token == "" { + return errors.New("missing object token") + } + return d.removeObj(ctx, []string{token}) +} + +func (d *DoubaoNew) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if file == nil { + return nil, errors.New("nil file") + } + if file.GetSize() <= 0 { + return nil, errors.New("invalid file size") + } + + uploadPrep, err := d.prepareUpload(ctx, file.GetName(), file.GetSize(), dstDir.GetID()) + if err != nil { + return nil, err + } + if uploadPrep.BlockSize <= 0 { + return nil, errors.New("invalid block size from prepare") + } + + tmpFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer tmpFile.Close() + + blockSize := uploadPrep.BlockSize + totalSize := file.GetSize() + numBlocks := int((totalSize + blockSize - 1) / blockSize) + blocks := make([]UploadBlock, 0, numBlocks) + blockMeta := make(map[int]UploadBlock, numBlocks) + + for seq := 0; seq < numBlocks; seq++ { + offset := int64(seq) * blockSize + length := blockSize + if remain := totalSize - offset; remain < length { + length = remain + } + buf := make([]byte, int(length)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + buf = buf[:n] + sum := sha256.Sum256(buf) + hash := base64.StdEncoding.EncodeToString(sum[:]) + checksum := adler32String(buf) + + block := UploadBlock{ + Hash: hash, + Seq: seq, + Size: int64(n), + Checksum: checksum, + IsUploaded: true, + } + blocks = append(blocks, block) + blockMeta[seq] = block + } + + needed, err := d.uploadBlocks(ctx, uploadPrep.UploadID, blocks, "explorer") + if err != nil { + return nil, err + } + + if len(needed.NeededUploadBlocks) > 0 { + sort.Slice(needed.NeededUploadBlocks, func(i, j int) bool { + return needed.NeededUploadBlocks[i].Seq < needed.NeededUploadBlocks[j].Seq + }) + const maxMergeBlockCount = 20 + var ( + groupSeqs []int + groupChecksums []string + groupSizes []int64 + groupRealSize int64 + groupExpectSum int64 + groupBuf bytes.Buffer + uploadedBytes int64 + ) + + flushGroup := func() error { + if len(groupSeqs) == 0 { + return nil + } + data := groupBuf.Bytes() + expectLen := groupExpectSum + if len(data) > 0 { + headLen := 32 + if len(data) < headLen { + headLen = len(data) + } + tailLen := 32 + if len(data) < tailLen { + tailLen = len(data) + } + } + if int64(len(data)) != expectLen { + return fmt.Errorf("[doubao_new] merge blocks invalid body len: got=%d expect=%d seqs=%v", len(data), expectLen, groupSeqs) + } + mergeResp, err := d.mergeUploadBlocks(ctx, uploadPrep.UploadID, groupSeqs, groupChecksums, groupSizes, blockSize, data) + if err != nil { + return err + } + if len(mergeResp.SuccessSeqList) != len(groupSeqs) { + return fmt.Errorf("[doubao_new] merge blocks incomplete: %v", mergeResp.SuccessSeqList) + } + success := make(map[int]bool, len(mergeResp.SuccessSeqList)) + for _, seq := range mergeResp.SuccessSeqList { + success[seq] = true + } + for _, seq := range groupSeqs { + if !success[seq] { + return fmt.Errorf("[doubao_new] merge blocks missing seq %d", seq) + } + } + + uploadedBytes += groupRealSize + groupSeqs = groupSeqs[:0] + groupChecksums = groupChecksums[:0] + groupSizes = groupSizes[:0] + groupRealSize = 0 + groupExpectSum = 0 + groupBuf.Reset() + if up != nil { + percent := float64(uploadedBytes) / float64(totalSize) * 100 + up(percent) + } + return nil + } + + for _, item := range needed.NeededUploadBlocks { + if _, ok := blockMeta[item.Seq]; !ok { + return nil, fmt.Errorf("[doubao_new] missing block meta for seq %d", item.Seq) + } + if item.Size <= 0 { + return nil, fmt.Errorf("[doubao_new] invalid block size from needed list: seq=%d size=%d", item.Seq, item.Size) + } + offset := int64(item.Seq) * blockSize + buf := make([]byte, int(item.Size)) + n, err := tmpFile.ReadAt(buf, offset) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + return nil, err + } + if n != len(buf) { + return nil, fmt.Errorf("[doubao_new] short read: seq=%d want=%d got=%d", item.Seq, len(buf), n) + } + buf = buf[:n] + realAdler := adler32String(buf) + if realAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] block checksum mismatch: seq=%d offset=%d adler32=%s step2=%s", item.Seq, offset, realAdler, item.Checksum) + } + payloadStart := groupBuf.Len() + groupBuf.Write(buf) + payloadEnd := groupBuf.Len() + payloadAdler := adler32String(groupBuf.Bytes()[payloadStart:payloadEnd]) + if payloadAdler != item.Checksum { + return nil, fmt.Errorf("[doubao_new] payload checksum mismatch: seq=%d start=%d end=%d adler32=%s step2=%s", item.Seq, payloadStart, payloadEnd, payloadAdler, item.Checksum) + } + groupSeqs = append(groupSeqs, item.Seq) + groupChecksums = append(groupChecksums, item.Checksum) + groupSizes = append(groupSizes, item.Size) + groupRealSize += int64(n) + groupExpectSum += item.Size + if len(groupSeqs) >= maxMergeBlockCount { + if err := flushGroup(); err != nil { + return nil, err + } + } + } + + if err := flushGroup(); err != nil { + return nil, err + } + if up != nil { + up(100) + } + } else if up != nil { + up(100) + } + + numBlocksFinish := uploadPrep.NumBlocks + if numBlocksFinish <= 0 { + numBlocksFinish = numBlocks + } + finish, err := d.finishUpload(ctx, uploadPrep.UploadID, numBlocksFinish, "explorer") + if err != nil { + return nil, err + } + + nodeToken := finish.Extra.NodeToken + if nodeToken == "" { + nodeToken = finish.FileToken + } + now := time.Now() + return &Object{ + Object: model.Object{ + ID: nodeToken, + Path: dstDir.GetID(), + Name: file.GetName(), + Size: file.GetSize(), + Modified: now, + Ctime: now, + IsFolder: false, + }, + ObjToken: finish.FileToken, + }, nil +} + +func (d *DoubaoNew) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoNew) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +func (d *DoubaoNew) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch args.Method { + case "doubao_preview", "preview": + obj, ok := args.Obj.(*Object) + if !ok { + return nil, errors.New("unsupported object type") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errs.NotSupport + } + + imgExt := ".webp" + pageNums := 1 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + if extra.PageNums > 0 { + pageNums = extra.PageNums + } + } + } + + return base.Json{ + "version": info.Version, + "img_ext": imgExt, + "page_nums": pageNums, + }, nil + default: + return nil, errs.NotSupport + } +} + +func (d *DoubaoNew) listAllChildren(ctx context.Context, parentToken string) ([]Node, error) { + nodes := make([]Node, 0, 50) + lastLabel := "" + for page := 0; page < 100; page++ { + data, err := d.listChildren(ctx, parentToken, lastLabel) + if err != nil { + return nil, err + } + + if len(data.NodeList) > 0 { + for _, token := range data.NodeList { + node, ok := data.Entities.Nodes[token] + if !ok { + continue + } + nodes = append(nodes, node) + } + } else { + for _, node := range data.Entities.Nodes { + nodes = append(nodes, node) + } + } + + if !data.HasMore || data.LastLabel == "" || data.LastLabel == lastLabel { + break + } + lastLabel = data.LastLabel + } + + if len(nodes) == 0 { + return nil, nil + } + return nodes, nil +} + +func (d *DoubaoNew) previewLink(ctx context.Context, obj *Object, args model.LinkArgs) (*model.Link, error) { + auth := d.resolveAuthorization() + dpop := d.resolveDpop() + if auth == "" || dpop == "" { + return nil, errors.New("missing authorization or dpop") + } + if obj.ObjToken == "" { + return nil, errors.New("missing obj_token") + } + info, err := d.getFileInfo(ctx, obj.ObjToken) + if err != nil { + return nil, err + } + + entry, ok := info.PreviewMeta.Data["22"] + if !ok || entry.Status != 0 { + return nil, errors.New("preview not available") + } + + subID := "" + pageIndex := 0 + if args.HttpReq != nil { + query := args.HttpReq.URL.Query() + if v := query.Get("sub_id"); v != "" { + subID = v + } else if v := query.Get("page"); v != "" { + if p, err := strconv.Atoi(v); err == nil && p >= 0 { + pageIndex = p + } + } + } + if subID == "" { + imgExt := ".webp" + pageNums := 0 + if entry.Extra != "" { + var extra PreviewImageExtra + if err := json.Unmarshal([]byte(entry.Extra), &extra); err == nil { + if extra.ImgExt != "" { + imgExt = extra.ImgExt + } + pageNums = extra.PageNums + } + } + if pageNums > 0 && pageIndex >= pageNums { + pageIndex = pageNums - 1 + } + subID = fmt.Sprintf("img_%d%s", pageIndex, imgExt) + } + + query := url.Values{} + query.Set("preview_type", "22") + query.Set("sub_id", subID) + if info.Version != "" { + query.Set("version", info.Version) + } + previewURL := fmt.Sprintf("%s/space/api/box/stream/download/preview_sub/%s?%s", BaseURL, obj.ObjToken, query.Encode()) + + headers := http.Header{ + "Referer": []string{"https://www.doubao.com/"}, + "User-Agent": []string{base.UserAgent}, + "Authorization": []string{auth}, + "Dpop": []string{dpop}, + } + + return &model.Link{ + URL: previewURL, + Header: headers, + }, nil +} + +func parseSize(size string) int64 { + if size == "" { + return 0 + } + val, err := strconv.ParseInt(size, 10, 64) + if err != nil { + return 0 + } + return val +} + +var _ driver.Driver = (*DoubaoNew)(nil) diff --git a/drivers/doubao_new/meta.go b/drivers/doubao_new/meta.go new file mode 100644 index 00000000000..5a8050a2653 --- /dev/null +++ b/drivers/doubao_new/meta.go @@ -0,0 +1,36 @@ +package doubao_new + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + Authorization string `json:"authorization" help:"DPoP access token (Authorization header value); optional if present in cookie"` + Dpop string `json:"dpop" help:"DPoP header value; optional if present in cookie"` + Cookie string `json:"cookie" help:"Optional cookie; only used to extract authorization/dpop tokens"` + Debug bool `json:"debug" help:"Enable debug logs for upload"` +} + +var config = driver.Config{ + Name: "DoubaoNew", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &DoubaoNew{} + }) +} diff --git a/drivers/doubao_new/types.go b/drivers/doubao_new/types.go new file mode 100644 index 00000000000..ea8acfc5b18 --- /dev/null +++ b/drivers/doubao_new/types.go @@ -0,0 +1,182 @@ +package doubao_new + +import "github.com/alist-org/alist/v3/internal/model" + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` +} + +type ListResp struct { + BaseResp + Data ListData `json:"data"` +} + +type ListData struct { + HasMore bool `json:"has_more"` + LastLabel string `json:"last_label"` + NodeList []string `json:"node_list"` + Entities struct { + Nodes map[string]Node `json:"nodes"` + Users map[string]User `json:"users"` + } `json:"entities"` +} + +type Node struct { + Token string `json:"token"` + NodeToken string `json:"node_token"` + ObjToken string `json:"obj_token"` + Name string `json:"name"` + Type int `json:"type"` + NodeType int `json:"node_type"` + OwnerID string `json:"owner_id"` + EditUID string `json:"edit_uid"` + CreateTime int64 `json:"create_time"` + EditTime int64 `json:"edit_time"` + URL string `json:"url"` + Extra struct { + Size string `json:"size"` + } `json:"extra"` +} + +type User struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type Object struct { + model.Object + ObjToken string + NodeType int + ObjType int + URL string +} + +type CreateFolderResp struct { + BaseResp + Data struct { + Entities struct { + Nodes map[string]Node `json:"nodes"` + } `json:"entities"` + NodeList []string `json:"node_list"` + } `json:"data"` +} + +type FileInfoResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data FileInfo `json:"data"` +} + +type FileInfo struct { + Name string `json:"name"` + NumBlocks int `json:"num_blocks"` + Version string `json:"version"` + MimeType string `json:"mime_type"` + MountPoint string `json:"mount_point"` + PreviewMeta PreviewMeta `json:"preview_meta"` +} + +type PreviewMeta struct { + Data map[string]PreviewMetaEntry `json:"data"` +} + +type PreviewMetaEntry struct { + Status int `json:"status"` + Extra string `json:"extra"` + PreviewFileSize int64 `json:"preview_file_size"` +} + +type PreviewImageExtra struct { + ImgExt string `json:"img_ext"` + PageNums int `json:"page_nums"` +} + +type UserStorageResp struct { + BaseResp + Data UserStorageData `json:"data"` +} + +type UserStorageData struct { + ShowSizeLimit bool `json:"show_size_limit"` + TotalSizeLimitBytes int64 `json:"total_size_limit_bytes"` + UsedSizeBytes int64 `json:"used_size_bytes"` +} + +type UploadPrepareResp struct { + BaseResp + Data UploadPrepareData `json:"data"` +} + +type UploadPrepareData struct { + BlockSize int64 `json:"block_size"` + NumBlocks int `json:"num_blocks"` + OptionBlockSize int64 `json:"option_block_size"` + DedupeSupport bool `json:"dedupe_support"` + UploadID string `json:"upload_id"` +} + +type UploadBlock struct { + Hash string `json:"hash"` + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + IsUploaded bool `json:"isUploaded"` +} + +type UploadBlocksResp struct { + BaseResp + Data UploadBlocksData `json:"data"` +} + +type UploadBlocksData struct { + NeededUploadBlocks []UploadBlockNeed `json:"needed_upload_blocks"` +} + +type UploadBlockNeed struct { + Seq int `json:"seq"` + Size int64 `json:"size"` + Checksum string `json:"checksum"` + Hash string `json:"hash"` +} + +type UploadMergeResp struct { + BaseResp + Data UploadMergeData `json:"data"` +} + +type UploadMergeData struct { + SuccessSeqList []int `json:"success_seq_list"` +} + +type UploadFinishResp struct { + BaseResp + Data UploadFinishData `json:"data"` +} + +type UploadFinishData struct { + Version string `json:"version"` + DataVersion string `json:"data_version"` + Extra struct { + NodeToken string `json:"node_token"` + } `json:"extra"` + FileToken string `json:"file_token"` +} + +type RemoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + } `json:"data"` +} + +type TaskStatusResp struct { + BaseResp + Data TaskStatusData `json:"data"` +} + +type TaskStatusData struct { + IsFinish bool `json:"is_finish"` + IsFail bool `json:"is_fail"` +} diff --git a/drivers/doubao_new/util.go b/drivers/doubao_new/util.go new file mode 100644 index 00000000000..5c21ca090db --- /dev/null +++ b/drivers/doubao_new/util.go @@ -0,0 +1,909 @@ +package doubao_new + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "hash/adler32" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" +) + +const ( + BaseURL = "https://my.feishu.cn" + DownloadBaseURL = "https://internal-api-drive-stream.feishu.cn" +) + +var defaultObjTypes = []string{"124", "0", "12", "30", "123", "22"} + +func (d *DoubaoNew) request(ctx context.Context, path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, BaseURL+path) + if err != nil { + return nil, err + } + if res != nil { + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + } + + body := res.Body() + var common BaseResp + if err = json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return body, fmt.Errorf(msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return body, fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func getCookieValue(cookie, name string) string { + parts := strings.Split(cookie, ";") + prefix := name + "=" + for _, part := range parts { + part = strings.TrimSpace(part) + if strings.HasPrefix(part, prefix) { + return strings.TrimPrefix(part, prefix) + } + } + return "" +} + +func adler32String(data []byte) string { + sum := adler32.Checksum(data) + return strconv.FormatUint(uint64(sum), 10) +} + +func buildCommaHeader(items []string) string { + return strings.Join(items, ",") +} + +func joinIntComma(items []int) string { + if len(items) == 0 { + return "" + } + var sb strings.Builder + for i, v := range items { + if i > 0 { + sb.WriteByte(',') + } + sb.WriteString(strconv.Itoa(v)) + } + return sb.String() +} + +func previewList(items []string, n int) string { + if n <= 0 || len(items) == 0 { + return "" + } + if len(items) < n { + n = len(items) + } + return strings.Join(items[:n], ",") +} + +func (d *DoubaoNew) resolveAuthorization() string { + auth := strings.TrimSpace(d.Authorization) + if auth == "" && d.Cookie != "" { + if token := getCookieValue(d.Cookie, "LARK_SUITE_ACCESS_TOKEN"); token != "" { + auth = token + } + } + if auth == "" { + return "" + } + if !strings.HasPrefix(auth, "DPoP ") && !strings.HasPrefix(auth, "dpop ") { + auth = "DPoP " + auth + } + return auth +} + +func (d *DoubaoNew) resolveDpop() string { + dpop := strings.TrimSpace(d.Dpop) + if dpop == "" && d.Cookie != "" { + dpop = getCookieValue(d.Cookie, "LARK_SUITE_DPOP") + } + return dpop +} + +func (d *DoubaoNew) listChildren(ctx context.Context, parentToken string, lastLabel string) (ListData, error) { + var resp ListResp + _, err := d.request(ctx, "/space/api/explorer/doubao/children/list/", http.MethodGet, func(req *resty.Request) { + values := url.Values{} + for _, t := range defaultObjTypes { + values.Add("obj_type", t) + } + values.Set("length", "50") + values.Set("rank", "0") + values.Set("asc", "0") + values.Set("min_length", "40") + values.Set("thumbnail_width", "1028") + values.Set("thumbnail_height", "1028") + values.Set("thumbnail_policy", "4") + if parentToken != "" { + values.Set("token", parentToken) + } + if lastLabel != "" { + values.Set("last_label", lastLabel) + } + req.SetQueryParamsFromValues(values) + }, &resp) + if err != nil { + return ListData{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) getFileInfo(ctx context.Context, fileToken string) (FileInfo, error) { + var resp FileInfoResp + _, err := d.request(ctx, "/space/api/box/file/info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "caller": "explorer", + "file_token": fileToken, + "mount_point": "explorer", + "option_params": []string{"preview_meta", "check_cipher"}, + }) + }, &resp) + if err != nil { + return FileInfo{}, err + } + + return resp.Data, nil +} + +func (d *DoubaoNew) createFolder(ctx context.Context, parentToken, name string) (Node, error) { + data := url.Values{} + data.Set("name", name) + data.Set("source", "0") + if parentToken != "" { + data.Set("parent_token", parentToken) + } + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/create/folder/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return Node{}, err + } + if err := decodeBaseResp(body, res); err != nil { + return Node{}, err + } + + var resp CreateFolderResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return Node{}, fmt.Errorf(msg) + } + + var node Node + if len(resp.Data.NodeList) > 0 { + if n, ok := resp.Data.Entities.Nodes[resp.Data.NodeList[0]]; ok { + node = n + } + } + if node.Token == "" { + for _, n := range resp.Data.Entities.Nodes { + node = n + break + } + } + if node.Token == "" && node.ObjToken == "" && node.NodeToken == "" { + return Node{}, fmt.Errorf("[doubao_new] create folder failed: empty response") + } + if node.NodeToken == "" { + if node.Token != "" { + node.NodeToken = node.Token + } else if node.ObjToken != "" { + node.NodeToken = node.ObjToken + } + } + if node.ObjToken == "" && node.Token != "" { + node.ObjToken = node.Token + } + return node, nil +} + +func (d *DoubaoNew) renameFolder(ctx context.Context, token, name string) error { + if token == "" { + return fmt.Errorf("[doubao_new] rename folder missing token") + } + data := url.Values{} + data.Set("token", token) + data.Set("name", name) + + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/rename/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func isCsrfTokenError(body []byte, res *resty.Response) bool { + if len(body) == 0 { + return false + } + if strings.Contains(strings.ToLower(string(body)), "csrf token error") { + return true + } + if res != nil && res.StatusCode() == http.StatusForbidden { + return true + } + return false +} + +func doRequestWithCsrf(doRequest func(csrfToken string) (*resty.Response, []byte, error)) (*resty.Response, []byte, error) { + res, body, err := doRequest("") + if err != nil { + return res, body, err + } + if isCsrfTokenError(body, res) { + csrfToken := extractCsrfTokenFromResponse(res) + if csrfToken != "" { + return doRequest(csrfToken) + } + } + return res, body, err +} + +func extractCsrfTokenFromResponse(res *resty.Response) string { + if res == nil || res.Request == nil { + return "" + } + if res.Request.RawRequest != nil { + if csrf := getCookieValue(res.Request.RawRequest.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + } + if csrf := getCookieValue(res.Request.Header.Get("Cookie"), "_csrf_token"); csrf != "" { + return csrf + } + for _, c := range res.Cookies() { + if c.Name == "_csrf_token" { + return c.Value + } + } + return "" +} + +func decodeBaseResp(body []byte, res *resty.Response) error { + var common BaseResp + if err := json.Unmarshal(body, &common); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf(msg) + } + if common.Code != 0 { + errMsg := common.Msg + if errMsg == "" { + errMsg = common.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", common.Code, errMsg) + } + return nil +} + +func (d *DoubaoNew) renameFile(ctx context.Context, fileToken, name string) error { + if fileToken == "" { + return fmt.Errorf("[doubao_new] rename file missing file token") + } + _, err := d.request(ctx, "/space/api/box/file/update_info/", http.MethodPost, func(req *resty.Request) { + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "file_token": fileToken, + "name": name, + }) + }, nil) + return err +} + +func (d *DoubaoNew) moveObj(ctx context.Context, srcToken, destToken string) error { + if srcToken == "" { + return fmt.Errorf("[doubao_new] move missing src token") + } + data := url.Values{} + data.Set("src_token", srcToken) + if destToken != "" { + data.Set("dest_token", destToken) + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/x-www-form-urlencoded") + req.SetBody(data.Encode()) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v2/move/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + return decodeBaseResp(body, res) +} + +func (d *DoubaoNew) removeObj(ctx context.Context, tokens []string) error { + if len(tokens) == 0 { + return fmt.Errorf("[doubao_new] remove missing tokens") + } + doRequest := func(csrfToken string) (*resty.Response, []byte, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if csrfToken != "" { + req.SetHeader("x-csrftoken", csrfToken) + } + req.SetHeader("Content-Type", "application/json") + req.SetBody(base.Json{ + "tokens": tokens, + "apply": 1, + }) + res, err := req.Execute(http.MethodPost, BaseURL+"/space/api/explorer/v3/remove/") + if err != nil { + return nil, nil, err + } + return res, res.Body(), nil + } + + res, body, err := doRequestWithCsrf(doRequest) + if err != nil { + return err + } + var resp RemoveResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + if resp.Data.TaskID == "" { + return nil + } + return d.waitTask(ctx, resp.Data.TaskID) +} + +func (d *DoubaoNew) getUserStorage(ctx context.Context) (UserStorageData, error) { + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("agw-js-conv", "str") + req.SetHeader("content-type", "application/json") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + } + req.SetBody(base.Json{}) + + res, err := req.Execute(http.MethodPost, "https://www.doubao.com/alice/aispace/facade/get_user_storage") + if err != nil { + return UserStorageData{}, err + } + + body := res.Body() + var resp UserStorageResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UserStorageData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UserStorageData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) waitTask(ctx context.Context, taskID string) error { + const ( + taskPollInterval = time.Second + taskPollMaxAttempts = 120 + ) + var lastErr error + for attempt := 0; attempt < taskPollMaxAttempts; attempt++ { + if attempt > 0 { + if err := waitWithContext(ctx, taskPollInterval); err != nil { + return err + } + } + status, err := d.getTaskStatus(ctx, taskID) + if err != nil { + lastErr = err + continue + } + if status.IsFail { + return fmt.Errorf("[doubao_new] remove task failed: %s", taskID) + } + if status.IsFinish { + return nil + } + } + if lastErr != nil { + return lastErr + } + return fmt.Errorf("[doubao_new] remove task timed out: %s", taskID) +} + +func (d *DoubaoNew) getTaskStatus(ctx context.Context, taskID string) (TaskStatusData, error) { + if taskID == "" { + return TaskStatusData{}, fmt.Errorf("[doubao_new] task status missing task_id") + } + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + req.SetQueryParam("task_id", taskID) + res, err := req.Execute(http.MethodGet, BaseURL+"/space/api/explorer/v2/task/") + if err != nil { + return TaskStatusData{}, err + } + body := res.Body() + var resp TaskStatusResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return TaskStatusData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return TaskStatusData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + return resp.Data, nil +} + +func waitWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (d *DoubaoNew) prepareUpload(ctx context.Context, name string, size int64, mountNodeToken string) (UploadPrepareData, error) { + var resp UploadPrepareResp + _, err := d.request(ctx, "/space/api/box/upload/prepare/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.prepare") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + body := base.Json{ + "mount_point": "explorer", + "mount_node_token": "", + "name": name, + "size": size, + "size_checker": true, + } + if mountNodeToken != "" { + body["mount_node_token"] = mountNodeToken + } + req.SetBody(body) + }, &resp) + if err != nil { + return UploadPrepareData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlocks(ctx context.Context, uploadID string, blocks []UploadBlock, mountPoint string) (UploadBlocksData, error) { + if uploadID == "" { + return UploadBlocksData{}, fmt.Errorf("[doubao_new] upload blocks missing upload_id") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadBlocksResp + _, err := d.request(ctx, "/space/api/box/upload/blocks/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.blocks") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetBody(base.Json{ + "blocks": blocks, + "upload_id": uploadID, + "mount_point": mountPoint, + }) + }, &resp) + if err != nil { + return UploadBlocksData{}, err + } + return resp.Data, nil +} + +func (d *DoubaoNew) mergeUploadBlocks(ctx context.Context, uploadID string, seqList []int, checksumList []string, sizeList []int64, blockOriginSize int64, data []byte) (UploadMergeData, error) { + if uploadID == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing upload_id") + } + if len(seqList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty seq list") + } + if len(checksumList) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty checksum list") + } + if len(sizeList) != len(seqList) { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks size list mismatch") + } + if blockOriginSize <= 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks invalid block origin size") + } + if len(data) == 0 { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks empty data") + } + + seqHeader := joinIntComma(seqList) + checksumHeader := buildCommaHeader(checksumList) + + client := base.NewRestyClient() + client.SetCookieJar(nil) + req := client.R() + req.SetContext(ctx) + req.SetHeader("accept", "application/json, text/plain, */*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("content-type", "application/octet-stream") + req.Header.Set("x-block-list-checksum", checksumHeader) + req.Header.Set("x-seq-list", seqHeader) + req.SetHeader("x-block-origin-size", strconv.FormatInt(blockOriginSize, 10)) + req.SetHeader("x-command", "space.api.box.stream.upload.merge_block") + req.SetHeader("x-csrftoken", "") + reqID := "" + if buf := make([]byte, 16); true { + if _, err := rand.Read(buf); err == nil { + reqID = hex.EncodeToString(buf) + } + } + if reqID != "" { + req.SetHeader("x-request-id", reqID) + } + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + req.Header.Del("Cookie") + req.Header.Del("cookie") + if req.Header.Get("x-command") == "" { + return UploadMergeData{}, fmt.Errorf("[doubao_new] merge blocks missing x-command header") + } + req.SetBody(data) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + urlStr := "https://internal-api-drive-stream.feishu.cn/space/api/box/stream/upload/merge_block/?" + values.Encode() + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return UploadMergeData{}, err + } + if v := res.Header().Get("X-Tt-Logid"); v != "" { + d.TtLogid = v + } else if v := res.Header().Get("x-tt-logid"); v != "" { + d.TtLogid = v + } + body := res.Body() + var resp UploadMergeResp + if err := json.Unmarshal(body, &resp); err != nil { + msg := fmt.Sprintf("[doubao_new] decode response failed (status: %s, content-type: %s, body: %s): %v", + res.Status(), + res.Header().Get("Content-Type"), + string(body), + err, + ) + return UploadMergeData{}, fmt.Errorf(msg) + } + if resp.Code != 0 { + if res != nil && res.StatusCode() == http.StatusBadRequest && resp.Code == 2 { + success := make([]int, 0, len(seqList)) + offset := 0 + for i, seq := range seqList { + size := sizeList[i] + if size <= 0 { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback invalid size: seq=%d size=%d", seq, size) + } + if offset+int(size) > len(data) { + return UploadMergeData{SuccessSeqList: success}, fmt.Errorf("[doubao_new] v3 fallback payload out of range: seq=%d offset=%d size=%d total=%d", seq, offset, size, len(data)) + } + payload := data[offset : offset+int(size)] + block := UploadBlockNeed{ + Seq: seq, + Size: size, + Checksum: checksumList[i], + } + if err := d.uploadBlockV3(ctx, uploadID, block, payload); err != nil { + return UploadMergeData{SuccessSeqList: success}, err + } + success = append(success, seq) + offset += int(size) + } + return UploadMergeData{SuccessSeqList: success}, nil + } + errMsg := resp.Msg + if errMsg == "" { + errMsg = resp.Message + } + return UploadMergeData{}, fmt.Errorf("[doubao_new] API error (code: %d): %s", resp.Code, errMsg) + } + + return resp.Data, nil +} + +func (d *DoubaoNew) uploadBlockV3(ctx context.Context, uploadID string, block UploadBlockNeed, data []byte) error { + if uploadID == "" { + return fmt.Errorf("[doubao_new] upload v3 block missing upload_id") + } + if block.Seq < 0 { + return fmt.Errorf("[doubao_new] upload v3 block invalid seq") + } + if len(data) == 0 { + return fmt.Errorf("[doubao_new] upload v3 block empty data") + } + + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeader("accept", "*/*") + req.SetHeader("origin", "https://www.doubao.com") + req.SetHeader("referer", "https://www.doubao.com/") + req.SetHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("x-block-seq", strconv.Itoa(block.Seq)) + req.SetHeader("x-block-checksum", block.Checksum) + if auth := d.resolveAuthorization(); auth != "" { + req.SetHeader("authorization", auth) + } + if dpop := d.resolveDpop(); dpop != "" { + req.SetHeader("dpop", dpop) + } + + req.SetMultipartFormData(map[string]string{ + "upload_id": uploadID, + "size": strconv.FormatInt(int64(len(data)), 10), + }) + req.SetMultipartField("file", "blob", "application/octet-stream", bytes.NewReader(data)) + + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("upload_id", uploadID) + values.Set("seq", strconv.Itoa(block.Seq)) + values.Set("size", strconv.FormatInt(int64(len(data)), 10)) + values.Set("checksum", block.Checksum) + values.Set("mount_point", "explorer") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + urlStr := "https://internal-api-drive-stream.feishu.cn/space/api/box/stream/upload/v3/block/?" + values.Encode() + + res, err := req.Execute(http.MethodPost, urlStr) + if err != nil { + return err + } + body := res.Body() + if err := decodeBaseResp(body, res); err != nil { + return err + } + return nil +} + +func (d *DoubaoNew) finishUpload(ctx context.Context, uploadID string, numBlocks int, mountPoint string) (UploadFinishData, error) { + if uploadID == "" { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload missing upload_id") + } + if numBlocks <= 0 { + return UploadFinishData{}, fmt.Errorf("[doubao_new] finish upload invalid num_blocks") + } + if mountPoint == "" { + mountPoint = "explorer" + } + var resp UploadFinishResp + _, err := d.request(ctx, "/space/api/box/upload/finish/", http.MethodPost, func(req *resty.Request) { + values := url.Values{} + values.Set("shouldBypassScsDialog", "true") + values.Set("doubao_storage", "imagex_other") + values.Set("doubao_app_id", "497858") + req.SetQueryParamsFromValues(values) + req.SetHeader("Content-Type", "application/json") + req.SetHeader("x-command", "space.api.box.upload.finish") + req.SetHeader("rpc-persist-doubao-pan", "true") + req.SetHeader("cache-control", "no-cache") + req.SetHeader("pragma", "no-cache") + req.SetHeader("biz-scene", "file_upload") + req.SetHeader("biz-ua-type", "Web") + req.SetBody(base.Json{ + "upload_id": uploadID, + "num_blocks": numBlocks, + "mount_point": mountPoint, + "push_open_history_record": 1, + }) + }, &resp) + if err != nil { + return UploadFinishData{}, err + } + return resp.Data, nil +} diff --git a/drivers/doubao_share/driver.go b/drivers/doubao_share/driver.go new file mode 100644 index 00000000000..61076d1ed17 --- /dev/null +++ b/drivers/doubao_share/driver.go @@ -0,0 +1,177 @@ +package doubao_share + +import ( + "context" + "errors" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" + "net/http" +) + +type DoubaoShare struct { + model.Storage + Addition + RootFiles []RootFileList +} + +func (d *DoubaoShare) Config() driver.Config { + return config +} + +func (d *DoubaoShare) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *DoubaoShare) Init(ctx context.Context) error { + // 初始化 虚拟分享列表 + if err := d.initShareList(); err != nil { + return err + } + + return nil +} + +func (d *DoubaoShare) Drop(ctx context.Context) error { + return nil +} + +func (d *DoubaoShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + // 检查是否为根目录 + if dir.GetID() == "" && dir.GetPath() == "/" { + return d.listRootDirectory(ctx) + } + + // 非根目录,处理不同情况 + if fo, ok := dir.(*FileObject); ok { + if fo.ShareID == "" { + // 虚拟目录,需要列出子目录 + return d.listVirtualDirectoryContent(dir) + } else { + // 具有分享ID的目录,获取此分享下的文件 + shareId, relativePath, err := d._findShareAndPath(dir) + if err != nil { + return nil, err + } + return d.getFilesInPath(ctx, shareId, dir.GetID(), relativePath) + } + } + + // 使用通用方法 + shareId, relativePath, err := d._findShareAndPath(dir) + if err != nil { + return nil, err + } + + // 获取指定路径下的文件 + return d.getFilesInPath(ctx, shareId, dir.GetID(), relativePath) +} + +func (d *DoubaoShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var downloadUrl string + + if u, ok := file.(*FileObject); ok { + switch u.NodeType { + case VideoType, AudioType: + var r GetVideoFileUrlResp + _, err := d.request("/samantha/media/get_play_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "key": u.Key, + "share_id": u.ShareID, + "node_id": file.GetID(), + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.OriginalMediaInfo.MainURL + default: + var r GetFileUrlResp + _, err := d.request("/alice/message/get_file_url", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "uris": []string{u.Key}, + "type": FileNodeType[u.NodeType], + }) + }, &r) + if err != nil { + return nil, err + } + + downloadUrl = r.Data.FileUrls[0].MainURL + } + + // 生成标准的Content-Disposition + contentDisposition := generateContentDisposition(u.Name) + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "User-Agent": []string{UserAgent}, + "Content-Disposition": []string{contentDisposition}, + }, + }, nil + } + + return nil, errors.New("can't convert obj to URL") +} + +func (d *DoubaoShare) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + // TODO create folder, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO move obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + // TODO rename obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Remove(ctx context.Context, obj model.Obj) error { + // TODO remove obj, optional + return errs.NotImplement +} + +func (d *DoubaoShare) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // TODO upload file, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *DoubaoShare) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *DoubaoShare) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*DoubaoShare)(nil) diff --git a/drivers/doubao_share/meta.go b/drivers/doubao_share/meta.go new file mode 100644 index 00000000000..a749eefb975 --- /dev/null +++ b/drivers/doubao_share/meta.go @@ -0,0 +1,32 @@ +package doubao_share + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Cookie string `json:"cookie" type:"text"` + ShareIds string `json:"share_ids" type:"text" required:"true"` +} + +var config = driver.Config{ + Name: "DoubaoShare", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: true, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &DoubaoShare{} + }) +} diff --git a/drivers/doubao_share/types.go b/drivers/doubao_share/types.go new file mode 100644 index 00000000000..46f226fa5a7 --- /dev/null +++ b/drivers/doubao_share/types.go @@ -0,0 +1,207 @@ +package doubao_share + +import ( + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/internal/model" +) + +type BaseResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +type NodeInfoData struct { + Share ShareInfo `json:"share,omitempty"` + Creator CreatorInfo `json:"creator,omitempty"` + NodeList []File `json:"node_list,omitempty"` + NodeInfo File `json:"node_info,omitempty"` + Children []File `json:"children,omitempty"` + Path FilePath `json:"path,omitempty"` + NextCursor string `json:"next_cursor,omitempty"` + HasMore bool `json:"has_more,omitempty"` +} + +type NodeInfoResp struct { + BaseResp + NodeInfoData `json:"data"` +} + +type RootFileList struct { + ShareID string + VirtualPath string + NodeInfo NodeInfoData + Child *[]RootFileList +} + +type File struct { + ID string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` + Size int64 `json:"size"` + Source int `json:"source"` + NameReviewStatus int `json:"name_review_status"` + ContentReviewStatus int `json:"content_review_status"` + RiskReviewStatus int `json:"risk_review_status"` + ConversationID string `json:"conversation_id"` + ParentID string `json:"parent_id"` + CreateTime int64 `json:"create_time"` + UpdateTime int64 `json:"update_time"` +} + +type FileObject struct { + model.Object + ShareID string + Key string + NodeID string + NodeType int +} + +type ShareInfo struct { + ShareID string `json:"share_id"` + FirstNode struct { + ID string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` + Size int `json:"size"` + Source int `json:"source"` + Content struct { + LinkFileType string `json:"link_file_type"` + ImageWidth int `json:"image_width"` + ImageHeight int `json:"image_height"` + AiSkillStatus int `json:"ai_skill_status"` + } `json:"content"` + NameReviewStatus int `json:"name_review_status"` + ContentReviewStatus int `json:"content_review_status"` + RiskReviewStatus int `json:"risk_review_status"` + ConversationID string `json:"conversation_id"` + ParentID string `json:"parent_id"` + CreateTime int `json:"create_time"` + UpdateTime int `json:"update_time"` + } `json:"first_node"` + NodeCount int `json:"node_count"` + CreateTime int `json:"create_time"` + Channel string `json:"channel"` + InfluencerType int `json:"influencer_type"` +} + +type CreatorInfo struct { + EntityID string `json:"entity_id"` + UserName string `json:"user_name"` + NickName string `json:"nick_name"` + Avatar struct { + OriginURL string `json:"origin_url"` + TinyURL string `json:"tiny_url"` + URI string `json:"uri"` + } `json:"avatar"` +} + +type FilePath []struct { + ID string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + NodeType int `json:"node_type"` + Size int `json:"size"` + Source int `json:"source"` + NameReviewStatus int `json:"name_review_status"` + ContentReviewStatus int `json:"content_review_status"` + RiskReviewStatus int `json:"risk_review_status"` + ConversationID string `json:"conversation_id"` + ParentID string `json:"parent_id"` + CreateTime int `json:"create_time"` + UpdateTime int `json:"update_time"` +} + +type GetFileUrlResp struct { + BaseResp + Data struct { + FileUrls []struct { + URI string `json:"uri"` + MainURL string `json:"main_url"` + BackURL string `json:"back_url"` + } `json:"file_urls"` + } `json:"data"` +} + +type GetVideoFileUrlResp struct { + BaseResp + Data struct { + MediaType string `json:"media_type"` + MediaInfo []struct { + Meta struct { + Height string `json:"height"` + Width string `json:"width"` + Format string `json:"format"` + Duration float64 `json:"duration"` + CodecType string `json:"codec_type"` + Definition string `json:"definition"` + } `json:"meta"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"media_info"` + OriginalMediaInfo struct { + Meta struct { + Height string `json:"height"` + Width string `json:"width"` + Format string `json:"format"` + Duration float64 `json:"duration"` + CodecType string `json:"codec_type"` + Definition string `json:"definition"` + } `json:"meta"` + MainURL string `json:"main_url"` + BackupURL string `json:"backup_url"` + } `json:"original_media_info"` + PosterURL string `json:"poster_url"` + PlayableStatus int `json:"playable_status"` + } `json:"data"` +} + +type CommonResp struct { + Code int `json:"code"` + Msg string `json:"msg,omitempty"` + Message string `json:"message,omitempty"` // 错误情况下的消息 + Data json.RawMessage `json:"data,omitempty"` // 原始数据,稍后解析 + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + Locale string `json:"locale"` + } `json:"error,omitempty"` +} + +// IsSuccess 判断响应是否成功 +func (r *CommonResp) IsSuccess() bool { + return r.Code == 0 +} + +// GetError 获取错误信息 +func (r *CommonResp) GetError() error { + if r.IsSuccess() { + return nil + } + // 优先使用message字段 + errMsg := r.Message + if errMsg == "" { + errMsg = r.Msg + } + // 如果error对象存在且有详细消息,则使用error中的信息 + if r.Error != nil && r.Error.Message != "" { + errMsg = r.Error.Message + } + + return fmt.Errorf("[doubao] API error (code: %d): %s", r.Code, errMsg) +} + +// UnmarshalData 将data字段解析为指定类型 +func (r *CommonResp) UnmarshalData(v interface{}) error { + if !r.IsSuccess() { + return r.GetError() + } + + if len(r.Data) == 0 { + return nil + } + + return json.Unmarshal(r.Data, v) +} diff --git a/drivers/doubao_share/util.go b/drivers/doubao_share/util.go new file mode 100644 index 00000000000..e0fc526e9fb --- /dev/null +++ b/drivers/doubao_share/util.go @@ -0,0 +1,744 @@ +package doubao_share + +import ( + "context" + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" + "net/http" + "net/url" + "path" + "regexp" + "strings" + "time" +) + +const ( + DirectoryType = 1 + FileType = 2 + LinkType = 3 + ImageType = 4 + PagesType = 5 + VideoType = 6 + AudioType = 7 + MeetingMinutesType = 8 +) + +var FileNodeType = map[int]string{ + 1: "directory", + 2: "file", + 3: "link", + 4: "image", + 5: "pages", + 6: "video", + 7: "audio", + 8: "meeting_minutes", +} + +const ( + BaseURL = "https://www.doubao.com" + FileDataType = "file" + ImgDataType = "image" + VideoDataType = "video" + UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36" +) + +func (d *DoubaoShare) request(path string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + reqUrl := BaseURL + path + req := base.RestyClient.R() + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "User-Agent": UserAgent, + }) + + req.SetQueryParams(map[string]string{ + "version_code": "20800", + "device_platform": "web", + }) + + if callback != nil { + callback(req) + } + + var commonResp CommonResp + + res, err := req.Execute(method, reqUrl) + log.Debugln(res.String()) + if err != nil { + return nil, err + } + + body := res.Body() + // 先解析为通用响应 + if err = json.Unmarshal(body, &commonResp); err != nil { + return nil, err + } + // 检查响应是否成功 + if !commonResp.IsSuccess() { + return body, commonResp.GetError() + } + + if resp != nil { + if err = json.Unmarshal(body, resp); err != nil { + return body, err + } + } + + return body, nil +} + +func (d *DoubaoShare) getFiles(dirId, nodeId, cursor string) (resp []File, err error) { + var r NodeInfoResp + + var body = base.Json{ + "share_id": dirId, + "node_id": nodeId, + } + // 如果有游标,则设置游标和大小 + if cursor != "" { + body["cursor"] = cursor + body["size"] = 50 + } else { + body["need_full_path"] = false + } + + _, err = d.request("/samantha/aispace/share/node_info", http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &r) + if err != nil { + return nil, err + } + + if r.NodeInfoData.Children != nil { + resp = r.NodeInfoData.Children + } + + if r.NodeInfoData.NextCursor != "-1" { + // 递归获取下一页 + nextFiles, err := d.getFiles(dirId, nodeId, r.NodeInfoData.NextCursor) + if err != nil { + return nil, err + } + + resp = append(r.NodeInfoData.Children, nextFiles...) + } + + return resp, err +} + +func (d *DoubaoShare) getShareOverview(shareId, cursor string) (resp []File, err error) { + return d.getShareOverviewWithHistory(shareId, cursor, make(map[string]bool)) +} + +func (d *DoubaoShare) getShareOverviewWithHistory(shareId, cursor string, cursorHistory map[string]bool) (resp []File, err error) { + var r NodeInfoResp + + var body = base.Json{ + "share_id": shareId, + } + // 如果有游标,则设置游标和大小 + if cursor != "" { + body["cursor"] = cursor + body["size"] = 50 + } else { + body["need_full_path"] = false + } + + _, err = d.request("/samantha/aispace/share/overview", http.MethodPost, func(req *resty.Request) { + req.SetBody(body) + }, &r) + if err != nil { + return nil, err + } + + if r.NodeInfoData.NodeList != nil { + resp = r.NodeInfoData.NodeList + } + + if r.NodeInfoData.NextCursor != "-1" { + // 检查游标是否重复出现,防止无限循环 + if cursorHistory[r.NodeInfoData.NextCursor] { + return resp, nil + } + + // 记录当前游标 + cursorHistory[r.NodeInfoData.NextCursor] = true + + // 递归获取下一页 + nextFiles, err := d.getShareOverviewWithHistory(shareId, r.NodeInfoData.NextCursor, cursorHistory) + if err != nil { + return nil, err + } + + resp = append(resp, nextFiles...) + } + + return resp, nil +} + +func (d *DoubaoShare) initShareList() error { + if d.Addition.ShareIds == "" { + return fmt.Errorf("share_ids is empty") + } + + // 解析分享配置 + shareConfigs, rootShares, err := d._parseShareConfigs() + if err != nil { + return err + } + + // 检查路径冲突 + if err := d._detectPathConflicts(shareConfigs); err != nil { + return err + } + + // 构建树形结构 + rootMap := d._buildTreeStructure(shareConfigs, rootShares) + + // 提取顶级节点 + topLevelNodes := d._extractTopLevelNodes(rootMap, rootShares) + if len(topLevelNodes) == 0 { + return fmt.Errorf("no valid share_ids found") + } + + // 存储结果 + d.RootFiles = topLevelNodes + + return nil +} + +// 从配置中解析分享ID和路径 +func (d *DoubaoShare) _parseShareConfigs() (map[string]string, []string, error) { + shareConfigs := make(map[string]string) // 路径 -> 分享ID + rootShares := make([]string, 0) // 根目录显示的分享ID + + lines := strings.Split(strings.TrimSpace(d.Addition.ShareIds), "\n") + if len(lines) == 0 { + return nil, nil, fmt.Errorf("no share_ids found") + } + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // 解析分享ID和路径 + parts := strings.Split(line, "|") + var shareId, sharePath string + + if len(parts) == 1 { + // 无路径分享,直接在根目录显示 + shareId = _extractShareId(parts[0]) + if shareId != "" { + rootShares = append(rootShares, shareId) + } + continue + } else if len(parts) >= 2 { + shareId = _extractShareId(parts[0]) + sharePath = strings.Trim(parts[1], "/") + } + + if shareId == "" { + log.Warnf("[doubao_share] Invalid Share_id Format: %s", line) + continue + } + + // 空路径也加入根目录显示 + if sharePath == "" { + rootShares = append(rootShares, shareId) + continue + } + + // 添加到路径映射 + shareConfigs[sharePath] = shareId + } + + return shareConfigs, rootShares, nil +} + +// 检测路径冲突 +func (d *DoubaoShare) _detectPathConflicts(shareConfigs map[string]string) error { + // 检查直接路径冲突 + pathToShareIds := make(map[string][]string) + for sharePath, id := range shareConfigs { + pathToShareIds[sharePath] = append(pathToShareIds[sharePath], id) + } + + for sharePath, ids := range pathToShareIds { + if len(ids) > 1 { + return fmt.Errorf("路径冲突: 路径 '%s' 被多个不同的分享ID使用: %s", + sharePath, strings.Join(ids, ", ")) + } + } + + // 检查层次冲突 + for path1, id1 := range shareConfigs { + for path2, id2 := range shareConfigs { + if path1 == path2 || id1 == id2 { + continue + } + + // 检查前缀冲突 + if strings.HasPrefix(path2, path1+"/") || strings.HasPrefix(path1, path2+"/") { + return fmt.Errorf("路径冲突: 路径 '%s' (ID: %s) 与路径 '%s' (ID: %s) 存在层次冲突", + path1, id1, path2, id2) + } + } + } + + return nil +} + +// 构建树形结构 +func (d *DoubaoShare) _buildTreeStructure(shareConfigs map[string]string, rootShares []string) map[string]*RootFileList { + rootMap := make(map[string]*RootFileList) + + // 添加所有分享节点 + for sharePath, shareId := range shareConfigs { + children := make([]RootFileList, 0) + rootMap[sharePath] = &RootFileList{ + ShareID: shareId, + VirtualPath: sharePath, + NodeInfo: NodeInfoData{}, + Child: &children, + } + } + + // 构建父子关系 + for sharePath, node := range rootMap { + if sharePath == "" { + continue + } + + pathParts := strings.Split(sharePath, "/") + if len(pathParts) > 1 { + parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") + + // 确保所有父级路径都已创建 + _ensurePathExists(rootMap, parentPath) + + // 添加当前节点到父节点 + if parent, exists := rootMap[parentPath]; exists { + *parent.Child = append(*parent.Child, *node) + } + } + } + + return rootMap +} + +// 提取顶级节点 +func (d *DoubaoShare) _extractTopLevelNodes(rootMap map[string]*RootFileList, rootShares []string) []RootFileList { + var topLevelNodes []RootFileList + + // 添加根目录分享 + for _, shareId := range rootShares { + children := make([]RootFileList, 0) + topLevelNodes = append(topLevelNodes, RootFileList{ + ShareID: shareId, + VirtualPath: "", + NodeInfo: NodeInfoData{}, + Child: &children, + }) + } + + // 添加顶级目录 + for rootPath, node := range rootMap { + if rootPath == "" { + continue + } + + isTopLevel := true + pathParts := strings.Split(rootPath, "/") + + if len(pathParts) > 1 { + parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") + if _, exists := rootMap[parentPath]; exists { + isTopLevel = false + } + } + + if isTopLevel { + topLevelNodes = append(topLevelNodes, *node) + } + } + + return topLevelNodes +} + +// 确保路径存在,创建所有必要的中间节点 +func _ensurePathExists(rootMap map[string]*RootFileList, path string) { + if path == "" { + return + } + + // 如果路径已存在,不需要再处理 + if _, exists := rootMap[path]; exists { + return + } + + // 创建当前路径节点 + children := make([]RootFileList, 0) + rootMap[path] = &RootFileList{ + ShareID: "", + VirtualPath: path, + NodeInfo: NodeInfoData{}, + Child: &children, + } + + // 处理父路径 + pathParts := strings.Split(path, "/") + if len(pathParts) > 1 { + parentPath := strings.Join(pathParts[:len(pathParts)-1], "/") + + // 确保父路径存在 + _ensurePathExists(rootMap, parentPath) + + // 将当前节点添加为父节点的子节点 + if parent, exists := rootMap[parentPath]; exists { + *parent.Child = append(*parent.Child, *rootMap[path]) + } + } +} + +// _extractShareId 从URL或直接ID中提取分享ID +func _extractShareId(input string) string { + input = strings.TrimSpace(input) + if strings.HasPrefix(input, "http") { + regex := regexp.MustCompile(`/drive/s/([a-zA-Z0-9]+)`) + if matches := regex.FindStringSubmatch(input); len(matches) > 1 { + return matches[1] + } + return "" + } + return input // 直接返回ID +} + +// _findRootFileByShareID 查找指定ShareID的配置 +func _findRootFileByShareID(rootFiles []RootFileList, shareID string) *RootFileList { + for i, rf := range rootFiles { + if rf.ShareID == shareID { + return &rootFiles[i] + } + if rf.Child != nil && len(*rf.Child) > 0 { + if found := _findRootFileByShareID(*rf.Child, shareID); found != nil { + return found + } + } + } + return nil +} + +// _findNodeByPath 查找指定路径的节点 +func _findNodeByPath(rootFiles []RootFileList, path string) *RootFileList { + for i, rf := range rootFiles { + if rf.VirtualPath == path { + return &rootFiles[i] + } + if rf.Child != nil && len(*rf.Child) > 0 { + if found := _findNodeByPath(*rf.Child, path); found != nil { + return found + } + } + } + return nil +} + +// _findShareByPath 根据路径查找分享和相对路径 +func _findShareByPath(rootFiles []RootFileList, path string) (*RootFileList, string) { + // 完全匹配或子路径匹配 + for i, rf := range rootFiles { + if rf.VirtualPath == path { + return &rootFiles[i], "" + } + + if rf.VirtualPath != "" && strings.HasPrefix(path, rf.VirtualPath+"/") { + relPath := strings.TrimPrefix(path, rf.VirtualPath+"/") + + // 先检查子节点 + if rf.Child != nil && len(*rf.Child) > 0 { + if child, childPath := _findShareByPath(*rf.Child, path); child != nil { + return child, childPath + } + } + + return &rootFiles[i], relPath + } + + // 递归检查子节点 + if rf.Child != nil && len(*rf.Child) > 0 { + if child, childPath := _findShareByPath(*rf.Child, path); child != nil { + return child, childPath + } + } + } + + // 检查根目录分享 + for i, rf := range rootFiles { + if rf.VirtualPath == "" && rf.ShareID != "" { + parts := strings.SplitN(path, "/", 2) + if len(parts) > 0 && parts[0] == rf.ShareID { + if len(parts) > 1 { + return &rootFiles[i], parts[1] + } + return &rootFiles[i], "" + } + } + } + + return nil, "" +} + +// _findShareAndPath 根据给定路径查找对应的ShareID和相对路径 +func (d *DoubaoShare) _findShareAndPath(dir model.Obj) (string, string, error) { + dirPath := dir.GetPath() + + // 如果是根目录,返回空值表示需要列出所有分享 + if dirPath == "/" || dirPath == "" { + return "", "", nil + } + + // 检查是否是 FileObject 类型,并获取 ShareID + if fo, ok := dir.(*FileObject); ok && fo.ShareID != "" { + // 直接使用对象中存储的 ShareID + // 计算相对路径(移除前导斜杠) + relativePath := strings.TrimPrefix(dirPath, "/") + + // 递归查找对应的 RootFile + found := _findRootFileByShareID(d.RootFiles, fo.ShareID) + if found != nil { + if found.VirtualPath != "" { + // 如果此分享配置了路径前缀,需要考虑相对路径的计算 + if strings.HasPrefix(relativePath, found.VirtualPath) { + return fo.ShareID, strings.TrimPrefix(relativePath, found.VirtualPath+"/"), nil + } + } + return fo.ShareID, relativePath, nil + } + + // 如果找不到对应的 RootFile 配置,仍然使用对象中的 ShareID + return fo.ShareID, relativePath, nil + } + + // 移除开头的斜杠 + cleanPath := strings.TrimPrefix(dirPath, "/") + + // 先检查是否有直接匹配的根目录分享 + for _, rootFile := range d.RootFiles { + if rootFile.VirtualPath == "" && rootFile.ShareID != "" { + // 检查是否匹配当前路径的第一部分 + parts := strings.SplitN(cleanPath, "/", 2) + if len(parts) > 0 && parts[0] == rootFile.ShareID { + if len(parts) > 1 { + return rootFile.ShareID, parts[1], nil + } + return rootFile.ShareID, "", nil + } + } + } + + // 查找匹配此路径的分享或虚拟目录 + share, relPath := _findShareByPath(d.RootFiles, cleanPath) + if share != nil { + return share.ShareID, relPath, nil + } + + log.Warnf("[doubao_share] No matching share path found: %s", dirPath) + return "", "", fmt.Errorf("no matching share path found: %s", dirPath) +} + +// convertToFileObject 将File转换为FileObject +func (d *DoubaoShare) convertToFileObject(file File, shareId string, relativePath string) *FileObject { + // 构建文件对象 + obj := &FileObject{ + Object: model.Object{ + ID: file.ID, + Name: file.Name, + Size: file.Size, + Modified: time.Unix(file.UpdateTime, 0), + Ctime: time.Unix(file.CreateTime, 0), + IsFolder: file.NodeType == DirectoryType, + Path: path.Join(relativePath, file.Name), + }, + ShareID: shareId, + Key: file.Key, + NodeID: file.ID, + NodeType: file.NodeType, + } + + return obj +} + +// getFilesInPath 获取指定分享和路径下的文件 +func (d *DoubaoShare) getFilesInPath(ctx context.Context, shareId, nodeId, relativePath string) ([]model.Obj, error) { + var ( + files []File + err error + ) + + // 调用overview接口获取分享链接信息 nodeId + if nodeId == "" { + files, err = d.getShareOverview(shareId, "") + if err != nil { + return nil, fmt.Errorf("failed to get share link information: %w", err) + } + + result := make([]model.Obj, 0, len(files)) + for _, file := range files { + result = append(result, d.convertToFileObject(file, shareId, "/")) + } + + return result, nil + + } else { + files, err = d.getFiles(shareId, nodeId, "") + if err != nil { + return nil, fmt.Errorf("failed to get share file: %w", err) + } + + result := make([]model.Obj, 0, len(files)) + for _, file := range files { + result = append(result, d.convertToFileObject(file, shareId, path.Join("/", relativePath))) + } + + return result, nil + } +} + +// listRootDirectory 处理根目录的内容展示 +func (d *DoubaoShare) listRootDirectory(ctx context.Context) ([]model.Obj, error) { + objects := make([]model.Obj, 0) + + // 分组处理:直接显示的分享内容 vs 虚拟目录 + var directShareIDs []string + addedDirs := make(map[string]bool) + + // 处理所有根节点 + for _, rootFile := range d.RootFiles { + if rootFile.VirtualPath == "" && rootFile.ShareID != "" { + // 无路径分享,记录ShareID以便后续获取内容 + directShareIDs = append(directShareIDs, rootFile.ShareID) + } else { + // 有路径的分享,显示第一级目录 + parts := strings.SplitN(rootFile.VirtualPath, "/", 2) + firstLevel := parts[0] + + // 避免重复添加同名目录 + if _, exists := addedDirs[firstLevel]; exists { + continue + } + + // 创建虚拟目录对象 + obj := &FileObject{ + Object: model.Object{ + ID: "", + Name: firstLevel, + Modified: time.Now(), + Ctime: time.Now(), + IsFolder: true, + Path: path.Join("/", firstLevel), + }, + ShareID: rootFile.ShareID, + Key: "", + NodeID: "", + NodeType: DirectoryType, + } + objects = append(objects, obj) + addedDirs[firstLevel] = true + } + } + + // 处理直接显示的分享内容 + for _, shareID := range directShareIDs { + shareFiles, err := d.getFilesInPath(ctx, shareID, "", "") + if err != nil { + log.Warnf("[doubao_share] Failed to get list of files in share %s: %s", shareID, err) + continue + } + objects = append(objects, shareFiles...) + } + + return objects, nil +} + +// listVirtualDirectoryContent 列出虚拟目录的内容 +func (d *DoubaoShare) listVirtualDirectoryContent(dir model.Obj) ([]model.Obj, error) { + dirPath := strings.TrimPrefix(dir.GetPath(), "/") + objects := make([]model.Obj, 0) + + // 递归查找此路径的节点 + node := _findNodeByPath(d.RootFiles, dirPath) + + if node != nil && node.Child != nil { + // 显示此节点的所有子节点 + for _, child := range *node.Child { + // 计算显示名称(取路径的最后一部分) + displayName := child.VirtualPath + if child.VirtualPath != "" { + parts := strings.Split(child.VirtualPath, "/") + displayName = parts[len(parts)-1] + } else if child.ShareID != "" { + displayName = child.ShareID + } + + obj := &FileObject{ + Object: model.Object{ + ID: "", + Name: displayName, + Modified: time.Now(), + Ctime: time.Now(), + IsFolder: true, + Path: path.Join("/", child.VirtualPath), + }, + ShareID: child.ShareID, + Key: "", + NodeID: "", + NodeType: DirectoryType, + } + objects = append(objects, obj) + } + } + + return objects, nil +} + +// generateContentDisposition 生成符合RFC 5987标准的Content-Disposition头部 +func generateContentDisposition(filename string) string { + // 按照RFC 2047进行编码,用于filename部分 + encodedName := urlEncode(filename) + + // 按照RFC 5987进行编码,用于filename*部分 + encodedNameRFC5987 := encodeRFC5987(filename) + + return fmt.Sprintf("attachment; filename=\"%s\"; filename*=utf-8''%s", + encodedName, encodedNameRFC5987) +} + +// encodeRFC5987 按照RFC 5987规范编码字符串,适用于HTTP头部参数中的非ASCII字符 +func encodeRFC5987(s string) string { + var buf strings.Builder + for _, r := range []byte(s) { + // 根据RFC 5987,只有字母、数字和部分特殊符号可以不编码 + if (r >= 'a' && r <= 'z') || + (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || + r == '-' || r == '.' || r == '_' || r == '~' { + buf.WriteByte(r) + } else { + // 其他字符都需要百分号编码 + fmt.Fprintf(&buf, "%%%02X", r) + } + } + return buf.String() +} + +func urlEncode(s string) string { + s = url.QueryEscape(s) + s = strings.ReplaceAll(s, "+", "%20") + return s +} diff --git a/drivers/dropbox/driver.go b/drivers/dropbox/driver.go index 7559d645858..fbaecc4a991 100644 --- a/drivers/dropbox/driver.go +++ b/drivers/dropbox/driver.go @@ -45,7 +45,25 @@ func (d *Dropbox) Init(ctx context.Context) error { if result != query { return fmt.Errorf("failed to check user: %s", string(res)) } - return nil + d.RootNamespaceId, err = d.GetRootNamespaceId(ctx) + + return err +} + +func (d *Dropbox) GetRootNamespaceId(ctx context.Context) (string, error) { + res, err := d.request("/2/users/get_current_account", http.MethodPost, func(req *resty.Request) { + req.SetBody(nil) + }) + if err != nil { + return "", err + } + var currentAccountResp CurrentAccountResp + err = utils.Json.Unmarshal(res, ¤tAccountResp) + if err != nil { + return "", err + } + rootNamespaceId := currentAccountResp.RootInfo.RootNamespaceId + return rootNamespaceId, nil } func (d *Dropbox) Drop(ctx context.Context) error { @@ -173,7 +191,7 @@ func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt } url := d.contentBase + "/2/files/upload_session/append_v2" - reader := io.LimitReader(stream, PartSize) + reader := driver.NewLimitedUploadStream(ctx, io.LimitReader(stream, PartSize)) req, err := http.NewRequest(http.MethodPost, url, reader) if err != nil { log.Errorf("failed to update file when append to upload session, err: %+v", err) @@ -201,13 +219,8 @@ func (d *Dropbox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt return err } _ = res.Body.Close() - - if count > 0 { - up((i + 1) * 100 / count) - } - + up(float64(i+1) * 100 / float64(count)) offset += byteSize - } // 3.finish toPath := dstDir.GetPath() + "/" + stream.GetName() diff --git a/drivers/dropbox/meta.go b/drivers/dropbox/meta.go index 07ab44695c1..6e7bc014790 100644 --- a/drivers/dropbox/meta.go +++ b/drivers/dropbox/meta.go @@ -17,7 +17,8 @@ type Addition struct { ClientID string `json:"client_id" required:"false" help:"Keep it empty if you don't have one"` ClientSecret string `json:"client_secret" required:"false" help:"Keep it empty if you don't have one"` - AccessToken string + AccessToken string + RootNamespaceId string } var config = driver.Config{ diff --git a/drivers/dropbox/types.go b/drivers/dropbox/types.go index 7ddbb9683c1..f2ec4cb7e7d 100644 --- a/drivers/dropbox/types.go +++ b/drivers/dropbox/types.go @@ -23,6 +23,13 @@ type RefreshTokenErrorResp struct { ErrorDescription string `json:"error_description"` } +type CurrentAccountResp struct { + RootInfo struct { + RootNamespaceId string `json:"root_namespace_id"` + HomeNamespaceId string `json:"home_namespace_id"` + } `json:"root_info"` +} + type File struct { Tag string `json:".tag"` Name string `json:"name"` diff --git a/drivers/dropbox/util.go b/drivers/dropbox/util.go index 14a5c6c6fc6..5065f08d394 100644 --- a/drivers/dropbox/util.go +++ b/drivers/dropbox/util.go @@ -46,12 +46,22 @@ func (d *Dropbox) refreshToken() error { func (d *Dropbox) request(uri, method string, callback base.ReqCallback, retry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) - if method == http.MethodPost { - req.SetHeader("Content-Type", "application/json") + if d.RootNamespaceId != "" { + apiPathRootJson, err := utils.Json.MarshalToString(map[string]interface{}{ + ".tag": "root", + "root": d.RootNamespaceId, + }) + if err != nil { + return nil, err + } + req.SetHeader("Dropbox-API-Path-Root", apiPathRootJson) } if callback != nil { callback(req) } + if method == http.MethodPost && req.Body != nil { + req.SetHeader("Content-Type", "application/json") + } var e ErrorResp req.SetError(&e) res, err := req.Execute(method, d.base+uri) diff --git a/drivers/febbox/driver.go b/drivers/febbox/driver.go new file mode 100644 index 00000000000..55c3aa211fe --- /dev/null +++ b/drivers/febbox/driver.go @@ -0,0 +1,132 @@ +package febbox + +import ( + "context" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type FebBox struct { + model.Storage + Addition + accessToken string + oauth2Token oauth2.TokenSource +} + +func (d *FebBox) Config() driver.Config { + return config +} + +func (d *FebBox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *FebBox) Init(ctx context.Context) error { + // 初始化 oauth2Config + oauth2Config := &clientcredentials.Config{ + ClientID: d.ClientID, + ClientSecret: d.ClientSecret, + AuthStyle: oauth2.AuthStyleInParams, + TokenURL: "https://api.febbox.com/oauth/token", + } + + d.initializeOAuth2Token(ctx, oauth2Config, d.Addition.RefreshToken) + + token, err := d.oauth2Token.Token() + if err != nil { + return err + } + d.accessToken = token.AccessToken + d.Addition.RefreshToken = token.RefreshToken + op.MustSaveDriverStorage(d) + + return nil +} + +func (d *FebBox) Drop(ctx context.Context) error { + return nil +} + +func (d *FebBox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFilesList(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *FebBox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var ip string + if d.Addition.UserIP != "" { + ip = d.Addition.UserIP + } else { + ip = args.IP + } + + url, err := d.getDownloadLink(file.GetID(), ip) + if err != nil { + return nil, err + } + return &model.Link{ + URL: url, + }, nil +} + +func (d *FebBox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + err := d.makeDir(parentDir.GetID(), dirName) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (d *FebBox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + err := d.move(srcObj.GetID(), dstDir.GetID()) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (d *FebBox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + err := d.rename(srcObj.GetID(), newName) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (d *FebBox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + err := d.copy(srcObj.GetID(), dstDir.GetID()) + if err != nil { + return nil, err + } + + return nil, nil +} + +func (d *FebBox) Remove(ctx context.Context, obj model.Obj) error { + err := d.remove(obj.GetID()) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*FebBox)(nil) diff --git a/drivers/febbox/meta.go b/drivers/febbox/meta.go new file mode 100644 index 00000000000..1daeeea8e52 --- /dev/null +++ b/drivers/febbox/meta.go @@ -0,0 +1,36 @@ +package febbox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + ClientID string `json:"client_id" required:"true" default:""` + ClientSecret string `json:"client_secret" required:"true" default:""` + RefreshToken string + SortRule string `json:"sort_rule" required:"true" type:"select" options:"size_asc,size_desc,name_asc,name_desc,update_asc,update_desc,ext_asc,ext_desc" default:"name_asc"` + PageSize int64 `json:"page_size" required:"true" type:"number" default:"100" help:"list api per page size of FebBox driver"` + UserIP string `json:"user_ip" default:"" help:"user ip address for download link which can speed up the download"` +} + +var config = driver.Config{ + Name: "FebBox", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: true, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &FebBox{} + }) +} diff --git a/drivers/febbox/oauth2.go b/drivers/febbox/oauth2.go new file mode 100644 index 00000000000..6345d1a711e --- /dev/null +++ b/drivers/febbox/oauth2.go @@ -0,0 +1,88 @@ +package febbox + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/oauth2" + "golang.org/x/oauth2/clientcredentials" +) + +type customTokenSource struct { + config *clientcredentials.Config + ctx context.Context + refreshToken string +} + +func (c *customTokenSource) Token() (*oauth2.Token, error) { + v := url.Values{} + if c.refreshToken != "" { + v.Set("grant_type", "refresh_token") + v.Set("refresh_token", c.refreshToken) + } else { + v.Set("grant_type", "client_credentials") + } + + v.Set("client_id", c.config.ClientID) + v.Set("client_secret", c.config.ClientSecret) + + req, err := http.NewRequest("POST", c.config.TokenURL, strings.NewReader(v.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := http.DefaultClient.Do(req.WithContext(c.ctx)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.New("oauth2: cannot fetch token") + } + + var tokenResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + AccessToken string `json:"access_token"` + ExpiresIn int64 `json:"expires_in"` + TokenType string `json:"token_type"` + Scope string `json:"scope"` + RefreshToken string `json:"refresh_token"` + } `json:"data"` + } + + if err := json.NewDecoder(resp.Body).Decode(&tokenResp); err != nil { + return nil, err + } + + if tokenResp.Code != 1 { + return nil, errors.New("oauth2: server response error") + } + + c.refreshToken = tokenResp.Data.RefreshToken + + token := &oauth2.Token{ + AccessToken: tokenResp.Data.AccessToken, + TokenType: tokenResp.Data.TokenType, + RefreshToken: tokenResp.Data.RefreshToken, + Expiry: time.Now().Add(time.Duration(tokenResp.Data.ExpiresIn) * time.Second), + } + + return token, nil +} + +func (d *FebBox) initializeOAuth2Token(ctx context.Context, oauth2Config *clientcredentials.Config, refreshToken string) { + d.oauth2Token = oauth2.ReuseTokenSource(nil, &customTokenSource{ + config: oauth2Config, + ctx: ctx, + refreshToken: refreshToken, + }) +} diff --git a/drivers/febbox/types.go b/drivers/febbox/types.go new file mode 100644 index 00000000000..2ac6d6b76cc --- /dev/null +++ b/drivers/febbox/types.go @@ -0,0 +1,123 @@ +package febbox + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" + "strconv" + "time" +) + +type ErrResp struct { + ErrorCode int64 `json:"code"` + ErrorMsg string `json:"msg"` + ServerRunTime float64 `json:"server_runtime"` + ServerName string `json:"server_name"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ServerRunTime != 0 || e.ServerName != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ServerRunTime: %f ,ServerName: %s", e.ErrorCode, e.ErrorMsg, e.ServerRunTime, e.ServerName) +} + +type FileListResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileList []File `json:"file_list"` + ShowType string `json:"show_type"` + } `json:"data"` +} + +type Rules struct { + AllowCopy int64 `json:"allow_copy"` + AllowDelete int64 `json:"allow_delete"` + AllowDownload int64 `json:"allow_download"` + AllowComment int64 `json:"allow_comment"` + HideLocation int64 `json:"hide_location"` +} + +type File struct { + Fid int64 `json:"fid"` + UID int64 `json:"uid"` + FileSize int64 `json:"file_size"` + Path string `json:"path"` + FileName string `json:"file_name"` + Ext string `json:"ext"` + AddTime int64 `json:"add_time"` + FileCreateTime int64 `json:"file_create_time"` + FileUpdateTime int64 `json:"file_update_time"` + ParentID int64 `json:"parent_id"` + UpdateTime int64 `json:"update_time"` + LastOpenTime int64 `json:"last_open_time"` + IsDir int64 `json:"is_dir"` + Epub int64 `json:"epub"` + IsMusicList int64 `json:"is_music_list"` + OssFid int64 `json:"oss_fid"` + Faststart int64 `json:"faststart"` + HasVideoQuality int64 `json:"has_video_quality"` + TotalDownload int64 `json:"total_download"` + Status int64 `json:"status"` + Remark string `json:"remark"` + OldHash string `json:"old_hash"` + Hash string `json:"hash"` + HashType string `json:"hash_type"` + FromUID int64 `json:"from_uid"` + FidOrg int64 `json:"fid_org"` + ShareID int64 `json:"share_id"` + InvitePermission int64 `json:"invite_permission"` + ThumbSmall string `json:"thumb_small"` + ThumbSmallWidth int64 `json:"thumb_small_width"` + ThumbSmallHeight int64 `json:"thumb_small_height"` + Thumb string `json:"thumb"` + ThumbWidth int64 `json:"thumb_width"` + ThumbHeight int64 `json:"thumb_height"` + ThumbBig string `json:"thumb_big"` + ThumbBigWidth int64 `json:"thumb_big_width"` + ThumbBigHeight int64 `json:"thumb_big_height"` + IsCustomThumb int64 `json:"is_custom_thumb"` + Photos int64 `json:"photos"` + IsAlbum int64 `json:"is_album"` + ReadOnly int64 `json:"read_only"` + Rules Rules `json:"rules"` + IsShared int64 `json:"is_shared"` +} + +func fileToObj(f File) *model.ObjThumb { + return &model.ObjThumb{ + Object: model.Object{ + ID: strconv.FormatInt(f.Fid, 10), + Name: f.FileName, + Size: f.FileSize, + Ctime: time.Unix(f.FileCreateTime, 0), + Modified: time.Unix(f.FileUpdateTime, 0), + IsFolder: f.IsDir == 1, + HashInfo: utils.NewHashInfo(hash_extend.GCID, f.Hash), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: f.Thumb, + }, + } +} + +type FileDownloadResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data []struct { + Error int `json:"error"` + DownloadURL string `json:"download_url"` + Hash string `json:"hash"` + HashType string `json:"hash_type"` + Fid int `json:"fid"` + FileName string `json:"file_name"` + ParentID int `json:"parent_id"` + FileSize int `json:"file_size"` + Ext string `json:"ext"` + Thumb string `json:"thumb"` + VipLink int `json:"vip_link"` + } `json:"data"` +} diff --git a/drivers/febbox/util.go b/drivers/febbox/util.go new file mode 100644 index 00000000000..ad2efe070e1 --- /dev/null +++ b/drivers/febbox/util.go @@ -0,0 +1,228 @@ +package febbox + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/op" + "github.com/go-resty/resty/v2" + "net/http" + "strconv" +) + +func (d *FebBox) refreshTokenByOAuth2() error { + token, err := d.oauth2Token.Token() + if err != nil { + return err + } + d.Status = "work" + d.accessToken = token.AccessToken + d.Addition.RefreshToken = token.RefreshToken + op.MustSaveDriverStorage(d) + return nil +} + +func (d *FebBox) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + // 使用oauth2 获取 access_token + token, err := d.oauth2Token.Token() + if err != nil { + return nil, err + } + req.SetAuthScheme(token.TokenType).SetAuthToken(token.AccessToken) + + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e ErrResp + req.SetError(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + + switch e.ErrorCode { + case 0: + return res.Body(), nil + case 1: + return res.Body(), nil + case -10001: + if e.ServerName != "" { + // access_token 过期 + if err = d.refreshTokenByOAuth2(); err != nil { + return nil, err + } + return d.request(url, method, callback, resp) + } else { + return nil, errors.New(e.Error()) + } + default: + return nil, errors.New(e.Error()) + } +} + +func (d *FebBox) getFilesList(id string) ([]File, error) { + if d.PageSize <= 0 { + d.PageSize = 100 + } + res, err := d.listWithLimit(id, d.PageSize) + if err != nil { + return nil, err + } + return *res, nil +} + +func (d *FebBox) listWithLimit(dirID string, pageLimit int64) (*[]File, error) { + var files []File + page := int64(1) + for { + result, err := d.getFiles(dirID, page, pageLimit) + if err != nil { + return nil, err + } + files = append(files, *result...) + if int64(len(*result)) < pageLimit { + break + } else { + page++ + } + } + return &files, nil +} + +func (d *FebBox) getFiles(dirID string, page, pageLimit int64) (*[]File, error) { + var fileList FileListResp + queryParams := map[string]string{ + "module": "file_list", + "parent_id": dirID, + "page": strconv.FormatInt(page, 10), + "pagelimit": strconv.FormatInt(pageLimit, 10), + "order": d.Addition.SortRule, + } + + res, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, &fileList) + if err != nil { + return nil, err + } + + if err = json.Unmarshal(res, &fileList); err != nil { + return nil, err + } + + return &fileList.Data.FileList, nil +} + +func (d *FebBox) getDownloadLink(id string, ip string) (string, error) { + var fileDownloadResp FileDownloadResp + queryParams := map[string]string{ + "module": "file_get_download_url", + "fids[]": id, + "ip": ip, + } + + res, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, &fileDownloadResp) + if err != nil { + return "", err + } + + if err = json.Unmarshal(res, &fileDownloadResp); err != nil { + return "", err + } + if len(fileDownloadResp.Data) == 0 { + return "", fmt.Errorf("can not get download link, code:%d, msg:%s", fileDownloadResp.Code, fileDownloadResp.Msg) + } + + return fileDownloadResp.Data[0].DownloadURL, nil +} + +func (d *FebBox) makeDir(id string, name string) error { + queryParams := map[string]string{ + "module": "create_dir", + "parent_id": id, + "name": name, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) move(id string, id2 string) error { + queryParams := map[string]string{ + "module": "file_move", + "fids[]": id, + "to": id2, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) rename(id string, name string) error { + queryParams := map[string]string{ + "module": "file_rename", + "fid": id, + "name": name, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) copy(id string, id2 string) error { + queryParams := map[string]string{ + "module": "file_copy", + "fids[]": id, + "to": id2, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *FebBox) remove(id string) error { + queryParams := map[string]string{ + "module": "file_delete", + "fids[]": id, + } + + _, err := d.request("https://api.febbox.com/oauth", http.MethodPost, func(req *resty.Request) { + req.SetMultipartFormData(queryParams) + }, nil) + if err != nil { + return err + } + + return nil +} diff --git a/drivers/ftp/driver.go b/drivers/ftp/driver.go index 70fbabdcdcd..8f30b780e75 100644 --- a/drivers/ftp/driver.go +++ b/drivers/ftp/driver.go @@ -39,7 +39,7 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m if err := d.login(); err != nil { return nil, err } - entries, err := d.conn.List(dir.GetPath()) + entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) if err != nil { return nil, err } @@ -49,7 +49,7 @@ func (d *FTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]m continue } f := model.Object{ - Name: entry.Name, + Name: decode(entry.Name, d.Encoding), Size: int64(entry.Size), Modified: entry.Time, IsFolder: entry.Type == ftp.EntryTypeFolder, @@ -64,7 +64,7 @@ func (d *FTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*m return nil, err } - r := NewFileReader(d.conn, file.GetPath(), file.GetSize()) + r := NewFileReader(d.conn, encode(file.GetPath(), d.Encoding), file.GetSize()) link := &model.Link{ MFile: r, } @@ -75,21 +75,27 @@ func (d *FTP) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) if err := d.login(); err != nil { return err } - return d.conn.MakeDir(stdpath.Join(parentDir.GetPath(), dirName)) + return d.conn.MakeDir(encode(stdpath.Join(parentDir.GetPath(), dirName), d.Encoding)) } func (d *FTP) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if err := d.login(); err != nil { return err } - return d.conn.Rename(srcObj.GetPath(), stdpath.Join(dstDir.GetPath(), srcObj.GetName())) + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(dstDir.GetPath(), srcObj.GetName()), d.Encoding), + ) } func (d *FTP) Rename(ctx context.Context, srcObj model.Obj, newName string) error { if err := d.login(); err != nil { return err } - return d.conn.Rename(srcObj.GetPath(), stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName)) + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), d.Encoding), + ) } func (d *FTP) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -100,19 +106,23 @@ func (d *FTP) Remove(ctx context.Context, obj model.Obj) error { if err := d.login(); err != nil { return err } + path := encode(obj.GetPath(), d.Encoding) if obj.IsDir() { - return d.conn.RemoveDirRecur(obj.GetPath()) + return d.conn.RemoveDirRecur(path) } else { - return d.conn.Delete(obj.GetPath()) + return d.conn.Delete(path) } } -func (d *FTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *FTP) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { if err := d.login(); err != nil { return err } - // TODO: support cancel - return d.conn.Stor(stdpath.Join(dstDir.GetPath(), stream.GetName()), stream) + path := stdpath.Join(dstDir.GetPath(), s.GetName()) + return d.conn.Stor(encode(path, d.Encoding), driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + })) } var _ driver.Driver = (*FTP)(nil) diff --git a/drivers/ftp/meta.go b/drivers/ftp/meta.go index 61d9d4a824e..5652c12e184 100644 --- a/drivers/ftp/meta.go +++ b/drivers/ftp/meta.go @@ -3,10 +3,28 @@ package ftp import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/op" + "github.com/axgle/mahonia" ) +func encode(str string, encoding string) string { + if encoding == "" { + return str + } + encoder := mahonia.NewEncoder(encoding) + return encoder.ConvertString(str) +} + +func decode(str string, encoding string) string { + if encoding == "" { + return str + } + decoder := mahonia.NewDecoder(encoding) + return decoder.ConvertString(str) +} + type Addition struct { Address string `json:"address" required:"true"` + Encoding string `json:"encoding" required:"true"` Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` driver.RootPath diff --git a/drivers/ftps/driver.go b/drivers/ftps/driver.go new file mode 100644 index 00000000000..4467380f09c --- /dev/null +++ b/drivers/ftps/driver.go @@ -0,0 +1,127 @@ +package ftps + +import ( + "context" + stdpath "path" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/jlaffaye/ftp" +) + +type FTPS struct { + model.Storage + Addition + conn *ftp.ServerConn +} + +func (d *FTPS) Config() driver.Config { + return config +} + +func (d *FTPS) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *FTPS) Init(ctx context.Context) error { + return d.login() +} + +func (d *FTPS) Drop(ctx context.Context) error { + if d.conn != nil { + _ = d.conn.Logout() + } + return nil +} + +func (d *FTPS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.login(); err != nil { + return nil, err + } + entries, err := d.conn.List(encode(dir.GetPath(), d.Encoding)) + if err != nil { + return nil, err + } + res := make([]model.Obj, 0) + for _, entry := range entries { + if entry.Name == "." || entry.Name == ".." { + continue + } + f := model.Object{ + Name: decode(entry.Name, d.Encoding), + Size: int64(entry.Size), + Modified: entry.Time, + IsFolder: entry.Type == ftp.EntryTypeFolder, + } + res = append(res, &f) + } + return res, nil +} + +func (d *FTPS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.login(); err != nil { + return nil, err + } + r := NewFileReader(d.conn, encode(file.GetPath(), d.Encoding), file.GetSize()) + link := &model.Link{ + MFile: r, + } + return link, nil +} + +func (d *FTPS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.login(); err != nil { + return err + } + return d.conn.MakeDir(encode(stdpath.Join(parentDir.GetPath(), dirName), d.Encoding)) +} + +func (d *FTPS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.login(); err != nil { + return err + } + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(dstDir.GetPath(), srcObj.GetName()), d.Encoding), + ) +} + +func (d *FTPS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.login(); err != nil { + return err + } + return d.conn.Rename( + encode(srcObj.GetPath(), d.Encoding), + encode(stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName), d.Encoding), + ) +} + +func (d *FTPS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *FTPS) Remove(ctx context.Context, obj model.Obj) error { + if err := d.login(); err != nil { + return err + } + path := encode(obj.GetPath(), d.Encoding) + if obj.IsDir() { + return d.conn.RemoveDirRecur(path) + } else { + return d.conn.Delete(path) + } +} + +func (d *FTPS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + if err := d.login(); err != nil { + return err + } + path := stdpath.Join(dstDir.GetPath(), s.GetName()) + return d.conn.Stor(encode(path, d.Encoding), driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + })) +} + +var _ driver.Driver = (*FTPS)(nil) diff --git a/drivers/ftps/meta.go b/drivers/ftps/meta.go new file mode 100644 index 00000000000..a752ec01aa6 --- /dev/null +++ b/drivers/ftps/meta.go @@ -0,0 +1,46 @@ +package ftps + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" + "github.com/axgle/mahonia" +) + +func encode(str string, encoding string) string { + if encoding == "" { + return str + } + encoder := mahonia.NewEncoder(encoding) + return encoder.ConvertString(str) +} + +func decode(str string, encoding string) string { + if encoding == "" { + return str + } + decoder := mahonia.NewDecoder(encoding) + return decoder.ConvertString(str) +} + +type Addition struct { + Address string `json:"address" required:"true"` + Encoding string `json:"encoding" required:"false"` + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + TLSMode string `json:"tls_mode" type:"select" options:"Explicit,Implicit" default:"Explicit" required:"true" help:"Explicit: STARTTLS on port 21; Implicit: direct TLS on port 990"` + TLSInsecureSkipVerify bool `json:"tls_insecure_skip_verify" default:"false" help:"Allow insecure TLS connections (e.g. self-signed certificates)"` + driver.RootPath +} + +var config = driver.Config{ + Name: "FTPS", + LocalSort: true, + OnlyLocal: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &FTPS{} + }) +} diff --git a/drivers/ftps/types.go b/drivers/ftps/types.go new file mode 100644 index 00000000000..d01dc417a16 --- /dev/null +++ b/drivers/ftps/types.go @@ -0,0 +1 @@ +package ftps diff --git a/drivers/ftps/util.go b/drivers/ftps/util.go new file mode 100644 index 00000000000..10680fa8b80 --- /dev/null +++ b/drivers/ftps/util.go @@ -0,0 +1,139 @@ +package ftps + +import ( + "crypto/tls" + "io" + "net" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/jlaffaye/ftp" +) + +func (d *FTPS) login() error { + if d.conn != nil { + _, err := d.conn.CurrentDir() + if err == nil { + return nil + } + } + + host, _, err := net.SplitHostPort(d.Address) + if err != nil { + host = d.Address + } + + tlsConfig := &tls.Config{ + ServerName: host, + InsecureSkipVerify: d.TLSInsecureSkipVerify, + } + + opts := []ftp.DialOption{ + ftp.DialWithShutTimeout(10 * time.Second), + } + if d.TLSMode == "Implicit" { + opts = append(opts, ftp.DialWithTLS(tlsConfig)) + } else { + opts = append(opts, ftp.DialWithExplicitTLS(tlsConfig)) + } + + conn, err := ftp.Dial(d.Address, opts...) + if err != nil { + return err + } + err = conn.Login(d.Username, d.Password) + if err != nil { + _ = conn.Quit() + return err + } + d.conn = conn + return nil +} + +type FileReader struct { + conn *ftp.ServerConn + resp *ftp.Response + offset atomic.Int64 + readAtOffset int64 + mu sync.Mutex + path string + size int64 +} + +func NewFileReader(conn *ftp.ServerConn, path string, size int64) *FileReader { + return &FileReader{ + conn: conn, + path: path, + size: size, + } +} + +func (r *FileReader) Read(buf []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + off := r.offset.Load() + n, err = r.readAtLocked(buf, off) + r.offset.Add(int64(n)) + return +} + +func (r *FileReader) ReadAt(buf []byte, off int64) (n int, err error) { + if off < 0 { + return 0, os.ErrInvalid + } + r.mu.Lock() + defer r.mu.Unlock() + return r.readAtLocked(buf, off) +} + +func (r *FileReader) readAtLocked(buf []byte, off int64) (n int, err error) { + if r.resp != nil && off != r.readAtOffset { + _ = r.resp.Close() + r.resp = nil + } + + if r.resp == nil { + r.resp, err = r.conn.RetrFrom(r.path, uint64(off)) + r.readAtOffset = off + if err != nil { + return 0, err + } + } + + n, err = r.resp.Read(buf) + r.readAtOffset += int64(n) + return +} + +func (r *FileReader) Seek(offset int64, whence int) (int64, error) { + oldOffset := r.offset.Load() + var newOffset int64 + switch whence { + case io.SeekStart: + newOffset = offset + case io.SeekCurrent: + newOffset = oldOffset + offset + case io.SeekEnd: + newOffset = r.size + offset + default: + return -1, os.ErrInvalid + } + + if newOffset < 0 { + return oldOffset, os.ErrInvalid + } + if newOffset == oldOffset { + return oldOffset, nil + } + r.offset.Store(newOffset) + return newOffset, nil +} + +func (r *FileReader) Close() error { + if r.resp != nil { + return r.resp.Close() + } + return nil +} diff --git a/drivers/gitee/driver.go b/drivers/gitee/driver.go new file mode 100644 index 00000000000..78a400941b9 --- /dev/null +++ b/drivers/gitee/driver.go @@ -0,0 +1,224 @@ +package gitee + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type Gitee struct { + model.Storage + Addition + client *resty.Client +} + +func (d *Gitee) Config() driver.Config { + return config +} + +func (d *Gitee) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Gitee) Init(ctx context.Context) error { + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + d.Endpoint = strings.TrimSpace(d.Endpoint) + if d.Endpoint == "" { + d.Endpoint = "https://gitee.com/api/v5" + } + d.Endpoint = strings.TrimSuffix(d.Endpoint, "/") + d.Owner = strings.TrimSpace(d.Owner) + d.Repo = strings.TrimSpace(d.Repo) + d.Token = strings.TrimSpace(d.Token) + d.DownloadProxy = strings.TrimSpace(d.DownloadProxy) + if d.Owner == "" || d.Repo == "" { + return errors.New("owner and repo are required") + } + d.client = base.NewRestyClient(). + SetBaseURL(d.Endpoint). + SetHeader("Accept", "application/json") + repo, err := d.getRepo() + if err != nil { + return err + } + d.Ref = strings.TrimSpace(d.Ref) + if d.Ref == "" { + d.Ref = repo.DefaultBranch + } + return nil +} + +func (d *Gitee) Drop(ctx context.Context) error { + return nil +} + +func (d *Gitee) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + relPath := d.relativePath(dir.GetPath()) + contents, err := d.listContents(relPath) + if err != nil { + return nil, err + } + objs := make([]model.Obj, 0, len(contents)) + for i := range contents { + objs = append(objs, contents[i].toModelObj()) + } + return objs, nil +} + +func (d *Gitee) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var downloadURL string + if obj, ok := file.(*Object); ok { + downloadURL = obj.DownloadURL + if downloadURL == "" { + relPath := d.relativePath(file.GetPath()) + content, err := d.getContent(relPath) + if err != nil { + return nil, err + } + if content.DownloadURL == "" { + return nil, errors.New("empty download url") + } + obj.DownloadURL = content.DownloadURL + downloadURL = content.DownloadURL + } + } else { + relPath := d.relativePath(file.GetPath()) + content, err := d.getContent(relPath) + if err != nil { + return nil, err + } + if content.DownloadURL == "" { + return nil, errors.New("empty download url") + } + downloadURL = content.DownloadURL + } + url := d.applyProxy(downloadURL) + return &model.Link{ + URL: url, + Header: http.Header{ + "Cookie": {d.Cookie}, + }, + }, nil +} + +func (d *Gitee) newRequest() *resty.Request { + req := d.client.R() + if d.Token != "" { + req.SetQueryParam("access_token", d.Token) + } + if d.Ref != "" { + req.SetQueryParam("ref", d.Ref) + } + return req +} + +func (d *Gitee) apiPath(path string) string { + escapedOwner := url.PathEscape(d.Owner) + escapedRepo := url.PathEscape(d.Repo) + if path == "" { + return fmt.Sprintf("/repos/%s/%s/contents", escapedOwner, escapedRepo) + } + return fmt.Sprintf("/repos/%s/%s/contents/%s", escapedOwner, escapedRepo, encodePath(path)) +} + +func (d *Gitee) listContents(path string) ([]Content, error) { + res, err := d.newRequest().Get(d.apiPath(path)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var contents []Content + if err := utils.Json.Unmarshal(res.Body(), &contents); err != nil { + var single Content + if err2 := utils.Json.Unmarshal(res.Body(), &single); err2 == nil && single.Type != "" { + if single.Type != "dir" { + return nil, errs.NotFolder + } + return []Content{}, nil + } + return nil, err + } + for i := range contents { + contents[i].Path = joinPath(path, contents[i].Name) + } + return contents, nil +} + +func (d *Gitee) getContent(path string) (*Content, error) { + res, err := d.newRequest().Get(d.apiPath(path)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var content Content + if err := utils.Json.Unmarshal(res.Body(), &content); err != nil { + return nil, err + } + if content.Type == "" { + return nil, errors.New("invalid response") + } + if content.Path == "" { + content.Path = path + } + return &content, nil +} + +func (d *Gitee) relativePath(full string) string { + full = utils.FixAndCleanPath(full) + root := utils.FixAndCleanPath(d.RootFolderPath) + if root == "/" { + return strings.TrimPrefix(full, "/") + } + if utils.PathEqual(full, root) { + return "" + } + prefix := utils.PathAddSeparatorSuffix(root) + if strings.HasPrefix(full, prefix) { + return strings.TrimPrefix(full, prefix) + } + return strings.TrimPrefix(full, "/") +} + +func (d *Gitee) applyProxy(raw string) string { + if raw == "" || d.DownloadProxy == "" { + return raw + } + proxy := d.DownloadProxy + if !strings.HasSuffix(proxy, "/") { + proxy += "/" + } + return proxy + strings.TrimLeft(raw, "/") +} + +func encodePath(p string) string { + if p == "" { + return "" + } + parts := strings.Split(p, "/") + for i, part := range parts { + parts[i] = url.PathEscape(part) + } + return strings.Join(parts, "/") +} + +func joinPath(base, name string) string { + if base == "" { + return name + } + return strings.TrimPrefix(stdpath.Join(base, name), "./") +} diff --git a/drivers/gitee/meta.go b/drivers/gitee/meta.go new file mode 100644 index 00000000000..2f926d635f3 --- /dev/null +++ b/drivers/gitee/meta.go @@ -0,0 +1,29 @@ +package gitee + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Endpoint string `json:"endpoint" type:"string" help:"Gitee API endpoint, default https://gitee.com/api/v5"` + Token string `json:"token" type:"string"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"Branch, tag or commit SHA, defaults to repository default branch"` + DownloadProxy string `json:"download_proxy" type:"string" help:"Prefix added before download URLs, e.g. https://mirror.example.com/"` + Cookie string `json:"cookie" type:"string" help:"Cookie returned from user info request"` +} + +var config = driver.Config{ + Name: "Gitee", + LocalSort: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Gitee{} + }) +} diff --git a/drivers/gitee/types.go b/drivers/gitee/types.go new file mode 100644 index 00000000000..c10536a5d10 --- /dev/null +++ b/drivers/gitee/types.go @@ -0,0 +1,60 @@ +package gitee + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type Links struct { + Self string `json:"self"` + Html string `json:"html"` +} + +type Content struct { + Type string `json:"type"` + Size *int64 `json:"size"` + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + URL string `json:"url"` + HtmlURL string `json:"html_url"` + DownloadURL string `json:"download_url"` + Links Links `json:"_links"` +} + +func (c Content) toModelObj() model.Obj { + size := int64(0) + if c.Size != nil { + size = *c.Size + } + return &Object{ + Object: model.Object{ + ID: c.Path, + Name: c.Name, + Size: size, + Modified: time.Unix(0, 0), + IsFolder: c.Type == "dir", + }, + DownloadURL: c.DownloadURL, + HtmlURL: c.HtmlURL, + } +} + +type Object struct { + model.Object + DownloadURL string + HtmlURL string +} + +func (o *Object) URL() string { + return o.DownloadURL +} + +type Repo struct { + DefaultBranch string `json:"default_branch"` +} + +type ErrResp struct { + Message string `json:"message"` +} diff --git a/drivers/gitee/util.go b/drivers/gitee/util.go new file mode 100644 index 00000000000..fbef972ad3d --- /dev/null +++ b/drivers/gitee/util.go @@ -0,0 +1,44 @@ +package gitee + +import ( + "fmt" + "net/url" + + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +func (d *Gitee) getRepo() (*Repo, error) { + req := d.client.R() + if d.Token != "" { + req.SetQueryParam("access_token", d.Token) + } + if d.Cookie != "" { + req.SetHeader("Cookie", d.Cookie) + } + escapedOwner := url.PathEscape(d.Owner) + escapedRepo := url.PathEscape(d.Repo) + res, err := req.Get(fmt.Sprintf("/repos/%s/%s", escapedOwner, escapedRepo)) + if err != nil { + return nil, err + } + if res.IsError() { + return nil, toErr(res) + } + var repo Repo + if err := utils.Json.Unmarshal(res.Body(), &repo); err != nil { + return nil, err + } + if repo.DefaultBranch == "" { + return nil, fmt.Errorf("failed to fetch default branch") + } + return &repo, nil +} + +func toErr(res *resty.Response) error { + var errMsg ErrResp + if err := utils.Json.Unmarshal(res.Body(), &errMsg); err == nil && errMsg.Message != "" { + return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) + } + return fmt.Errorf(res.Status()) +} diff --git a/drivers/github/driver.go b/drivers/github/driver.go new file mode 100644 index 00000000000..dedd4945bdc --- /dev/null +++ b/drivers/github/driver.go @@ -0,0 +1,975 @@ +package github + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + stdpath "path" + "strings" + "sync" + "text/template" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type Github struct { + model.Storage + Addition + client *resty.Client + mkdirMsgTmpl *template.Template + deleteMsgTmpl *template.Template + putMsgTmpl *template.Template + renameMsgTmpl *template.Template + copyMsgTmpl *template.Template + moveMsgTmpl *template.Template + isOnBranch bool + commitMutex sync.Mutex + pgpEntity *openpgp.Entity +} + +func (d *Github) Config() driver.Config { + return config +} + +func (d *Github) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Github) Init(ctx context.Context) error { + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + if d.CommitterName != "" && d.CommitterEmail == "" { + return errors.New("committer email is required") + } + if d.CommitterName == "" && d.CommitterEmail != "" { + return errors.New("committer name is required") + } + if d.AuthorName != "" && d.AuthorEmail == "" { + return errors.New("author email is required") + } + if d.AuthorName == "" && d.AuthorEmail != "" { + return errors.New("author name is required") + } + var err error + d.mkdirMsgTmpl, err = template.New("mkdirCommitMsgTemplate").Parse(d.MkdirCommitMsg) + if err != nil { + return err + } + d.deleteMsgTmpl, err = template.New("deleteCommitMsgTemplate").Parse(d.DeleteCommitMsg) + if err != nil { + return err + } + d.putMsgTmpl, err = template.New("putCommitMsgTemplate").Parse(d.PutCommitMsg) + if err != nil { + return err + } + d.renameMsgTmpl, err = template.New("renameCommitMsgTemplate").Parse(d.RenameCommitMsg) + if err != nil { + return err + } + d.copyMsgTmpl, err = template.New("copyCommitMsgTemplate").Parse(d.CopyCommitMsg) + if err != nil { + return err + } + d.moveMsgTmpl, err = template.New("moveCommitMsgTemplate").Parse(d.MoveCommitMsg) + if err != nil { + return err + } + d.client = base.NewRestyClient(). + SetHeader("Accept", "application/vnd.github.object+json"). + SetHeader("X-GitHub-Api-Version", "2022-11-28"). + SetLogger(log.StandardLogger()). + SetDebug(false) + token := strings.TrimSpace(d.Token) + if token != "" { + d.client = d.client.SetHeader("Authorization", "Bearer "+token) + } + if d.Ref == "" { + repo, err := d.getRepo() + if err != nil { + return err + } + d.Ref = repo.DefaultBranch + d.isOnBranch = true + } else { + _, err = d.getBranchHead() + d.isOnBranch = err == nil + } + if d.GPGPrivateKey != "" { + if d.CommitterName == "" || d.AuthorName == "" { + user, e := d.getAuthenticatedUser() + if e != nil { + return e + } + if d.CommitterName == "" { + d.CommitterName = user.Name + d.CommitterEmail = user.Email + } + if d.AuthorName == "" { + d.AuthorName = user.Name + d.AuthorEmail = user.Email + } + } + d.pgpEntity, err = loadPrivateKey(d.GPGPrivateKey, d.GPGKeyPassphrase) + if err != nil { + return err + } + } + return nil +} + +func (d *Github) Drop(ctx context.Context) error { + return nil +} + +func (d *Github) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + obj, err := d.get(dir.GetPath()) + if err != nil { + return nil, err + } + if obj.Entries == nil { + return nil, errs.NotFolder + } + if len(obj.Entries) >= 1000 { + tree, err := d.getTree(obj.Sha) + if err != nil { + return nil, err + } + if tree.Truncated { + return nil, fmt.Errorf("tree %s is truncated", dir.GetPath()) + } + ret := make([]model.Obj, 0, len(tree.Trees)) + for _, t := range tree.Trees { + if t.Path != ".gitkeep" { + ret = append(ret, t.toModelObj()) + } + } + return ret, nil + } else { + ret := make([]model.Obj, 0, len(obj.Entries)) + for _, entry := range obj.Entries { + if entry.Name != ".gitkeep" { + ret = append(ret, entry.toModelObj()) + } + } + return ret, nil + } +} + +func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, err := d.get(file.GetPath()) + if err != nil { + return nil, err + } + if obj.Type == "submodule" { + return nil, errors.New("cannot download a submodule") + } + url := obj.DownloadURL + ghProxy := strings.TrimSpace(d.Addition.GitHubProxy) + if ghProxy != "" { + url = strings.Replace(url, "https://raw.githubusercontent.com", ghProxy, 1) + } + return &model.Link{ + URL: url, + }, nil +} + +func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + parent, err := d.get(parentDir.GetPath()) + if err != nil { + return err + } + if parent.Entries == nil { + return errs.NotFolder + } + subDirSha, err := d.newTree("", []interface{}{ + map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }, + }) + if err != nil { + return err + } + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, TreeObjReq{ + Path: dirName, + Mode: "040000", + Type: "tree", + Sha: subDirSha, + }) + if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { + newTree = append(newTree, TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + }) + } + newSha, err := d.newTree(parent.Sha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(parentDir.GetPath(), parent.Sha, newSha, "/") + if err != nil { + return err + } + + commitMessage, err := getMessage(d.mkdirMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: dirName, + ObjPath: stdpath.Join(parentDir.GetPath(), dirName), + ParentName: parentDir.GetName(), + ParentPath: parentDir.GetPath(), + }, "mkdir") + if err != nil { + return err + } + return d.commit(commitMessage, rootSha) +} + +func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return errors.New("cannot move parent dir to child") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + + var rootSha string + if strings.HasPrefix(dstDir.GetPath(), stdpath.Dir(srcObj.GetPath())) { // /aa/1 -> /aa/bb/ + dstOldSha, dstNewSha, ancestorOldSha, srcParentTree, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + + srcParentPath := stdpath.Dir(srcObj.GetPath()) + dstRest := dstDir.GetPath()[len(srcParentPath):] + if dstRest[0] == '/' { + dstRest = dstRest[1:] + } + dstNextName, _, _ := strings.Cut(dstRest, "/") + dstNextPath := stdpath.Join(srcParentPath, dstNextName) + dstNextTreeSha, err := d.renewParentTrees(dstDir.GetPath(), dstOldSha, dstNewSha, dstNextPath) + if err != nil { + return err + } + var delSrc, dstNextTree *TreeObjReq = nil, nil + for _, t := range srcParentTree.Trees { + if t.Path == dstNextName { + dstNextTree = &t.TreeObjReq + dstNextTree.Sha = dstNextTreeSha + } + if t.Path == srcObj.GetName() { + delSrc = &t.TreeObjReq + delSrc.Sha = nil + } + if delSrc != nil && dstNextTree != nil { + break + } + } + if delSrc == nil || dstNextTree == nil { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, []interface{}{*delSrc, *dstNextTree}) + if err != nil { + return err + } + rootSha, err = d.renewParentTrees(srcParentPath, ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } else if strings.HasPrefix(srcObj.GetPath(), dstDir.GetPath()) { // /aa/bb/1 -> /aa/ + srcParentPath := stdpath.Dir(srcObj.GetPath()) + srcParentTree, srcParentOldSha, err := d.getTreeDirectly(srcParentPath) + if err != nil { + return err + } + var src *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return errors.New("cannot move a submodule") + } + src = &t.TreeObjReq + break + } + } + if src == nil { + return errs.ObjectNotFound + } + + delSrc := *src + delSrc.Sha = nil + delSrcTree := make([]interface{}, 0, 2) + delSrcTree = append(delSrcTree, delSrc) + if len(srcParentTree.Trees) == 1 { + delSrcTree = append(delSrcTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + srcParentNewSha, err := d.newTree(srcParentOldSha, delSrcTree) + if err != nil { + return err + } + srcRest := srcObj.GetPath()[len(dstDir.GetPath()):] + if srcRest[0] == '/' { + srcRest = srcRest[1:] + } + srcNextName, _, ok := strings.Cut(srcRest, "/") + if !ok { // /aa/1 -> /aa/ + return errors.New("cannot move in place") + } + srcNextPath := stdpath.Join(dstDir.GetPath(), srcNextName) + srcNextTreeSha, err := d.renewParentTrees(srcParentPath, srcParentOldSha, srcParentNewSha, srcNextPath) + if err != nil { + return err + } + + ancestorTree, ancestorOldSha, err := d.getTreeDirectly(dstDir.GetPath()) + if err != nil { + return err + } + var srcNextTree *TreeObjReq = nil + for _, t := range ancestorTree.Trees { + if t.Path == srcNextName { + srcNextTree = &t.TreeObjReq + srcNextTree.Sha = srcNextTreeSha + break + } + } + if srcNextTree == nil { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, []interface{}{*srcNextTree, *src}) + if err != nil { + return err + } + rootSha, err = d.renewParentTrees(dstDir.GetPath(), ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } else { // /aa/1 -> /bb/ + // do copy + dstOldSha, dstNewSha, srcParentOldSha, srcParentTree, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + + // delete src object and create new tree + var srcNewTree *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + srcNewTree = &t.TreeObjReq + srcNewTree.Sha = nil + break + } + } + if srcNewTree == nil { + return errs.ObjectNotFound + } + delSrcTree := make([]interface{}, 0, 2) + delSrcTree = append(delSrcTree, *srcNewTree) + if len(srcParentTree.Trees) == 1 { + delSrcTree = append(delSrcTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + srcParentNewSha, err := d.newTree(srcParentOldSha, delSrcTree) + if err != nil { + return err + } + + // renew but the common ancestor of srcPath and dstPath + ancestor, srcChildName, dstChildName, _, _ := getPathCommonAncestor(srcObj.GetPath(), dstDir.GetPath()) + dstNextTreeSha, err := d.renewParentTrees(dstDir.GetPath(), dstOldSha, dstNewSha, stdpath.Join(ancestor, dstChildName)) + if err != nil { + return err + } + srcNextTreeSha, err := d.renewParentTrees(stdpath.Dir(srcObj.GetPath()), srcParentOldSha, srcParentNewSha, stdpath.Join(ancestor, srcChildName)) + if err != nil { + return err + } + + // renew the tree of the last common ancestor + ancestorTree, ancestorOldSha, err := d.getTreeDirectly(ancestor) + if err != nil { + return err + } + newTree := make([]interface{}, 2) + srcBind := false + dstBind := false + for _, t := range ancestorTree.Trees { + if t.Path == srcChildName { + t.Sha = srcNextTreeSha + newTree[0] = t.TreeObjReq + srcBind = true + } + if t.Path == dstChildName { + t.Sha = dstNextTreeSha + newTree[1] = t.TreeObjReq + dstBind = true + } + if srcBind && dstBind { + break + } + } + if !srcBind || !dstBind { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, newTree) + if err != nil { + return err + } + // renew until root + rootSha, err = d.renewParentTrees(ancestor, ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } + + // commit + message, err := getMessage(d.moveMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), + ParentPath: stdpath.Dir(srcObj.GetPath()), + TargetName: stdpath.Base(dstDir.GetPath()), + TargetPath: dstDir.GetPath(), + }, "move") + if err != nil { + return err + } + return d.commit(message, rootSha) +} + +func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + parentDir := stdpath.Dir(srcObj.GetPath()) + tree, _, err := d.getTreeDirectly(parentDir) + if err != nil { + return err + } + newTree := make([]interface{}, 2) + operated := false + for _, t := range tree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return errors.New("cannot rename a submodule") + } + delCopy := t.TreeObjReq + delCopy.Sha = nil + newTree[0] = delCopy + t.Path = newName + newTree[1] = t.TreeObjReq + operated = true + break + } + } + if !operated { + return errs.ObjectNotFound + } + newSha, err := d.newTree(tree.Sha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(parentDir, tree.Sha, newSha, "/") + if err != nil { + return err + } + message, err := getMessage(d.renameMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(parentDir), + ParentPath: parentDir, + TargetName: newName, + TargetPath: stdpath.Join(parentDir, newName), + }, "rename") + if err != nil { + return err + } + return d.commit(message, rootSha) +} + +func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return errors.New("cannot copy parent dir to child") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + + dstSha, newSha, _, _, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(dstDir.GetPath(), dstSha, newSha, "/") + if err != nil { + return err + } + message, err := getMessage(d.copyMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), + ParentPath: stdpath.Dir(srcObj.GetPath()), + TargetName: stdpath.Base(dstDir.GetPath()), + TargetPath: dstDir.GetPath(), + }, "copy") + if err != nil { + return err + } + return d.commit(message, rootSha) +} + +func (d *Github) Remove(ctx context.Context, obj model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + parentDir := stdpath.Dir(obj.GetPath()) + tree, treeSha, err := d.getTreeDirectly(parentDir) + if err != nil { + return err + } + var del *TreeObjReq = nil + for _, t := range tree.Trees { + if t.Path == obj.GetName() { + if t.Type == "commit" { + return errors.New("cannot remove a submodule") + } + del = &t.TreeObjReq + del.Sha = nil + break + } + } + if del == nil { + return errs.ObjectNotFound + } + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, *del) + if len(tree.Trees) == 1 { // completely emptying the repository will get a 404 + newTree = append(newTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + newSha, err := d.newTree(treeSha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(parentDir, treeSha, newSha, "/") + if err != nil { + return err + } + commitMessage, err := getMessage(d.deleteMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: obj.GetName(), + ObjPath: obj.GetPath(), + ParentName: stdpath.Base(parentDir), + ParentPath: parentDir, + }, "remove") + if err != nil { + return err + } + return d.commit(commitMessage, rootSha) +} + +func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + blob, err := d.putBlob(ctx, stream, up) + if err != nil { + return err + } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + parent, err := d.get(dstDir.GetPath()) + if err != nil { + return err + } + if parent.Entries == nil { + return errs.NotFolder + } + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, TreeObjReq{ + Path: stream.GetName(), + Mode: "100644", + Type: "blob", + Sha: blob, + }) + if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { + newTree = append(newTree, TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + }) + } + newSha, err := d.newTree(parent.Sha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(dstDir.GetPath(), parent.Sha, newSha, "/") + if err != nil { + return err + } + + commitMessage, err := getMessage(d.putMsgTmpl, &MessageTemplateVars{ + UserName: getUsername(ctx), + ObjName: stream.GetName(), + ObjPath: stdpath.Join(dstDir.GetPath(), stream.GetName()), + ParentName: dstDir.GetName(), + ParentPath: dstDir.GetPath(), + }, "upload") + if err != nil { + return err + } + return d.commit(commitMessage, rootSha) +} + +var _ driver.Driver = (*Github)(nil) + +func (d *Github) getContentApiUrl(path string) string { + path = utils.FixAndCleanPath(path) + return fmt.Sprintf("https://api.github.com/repos/%s/%s/contents%s", d.Owner, d.Repo, path) +} + +func (d *Github) get(path string) (*Object, error) { + res, err := d.client.R().SetQueryParam("ref", d.Ref).Get(d.getContentApiUrl(path)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp Object + err = utils.Json.Unmarshal(res.Body(), &resp) + return &resp, err +} + +func (d *Github) putBlob(ctx context.Context, s model.FileStreamer, up driver.UpdateProgress) (string, error) { + beforeContent := "{\"encoding\":\"base64\",\"content\":\"" + afterContent := "\"}" + length := int64(len(beforeContent)) + calculateBase64Length(s.GetSize()) + int64(len(afterContent)) + beforeContentReader := strings.NewReader(beforeContent) + contentReader, contentWriter := io.Pipe() + go func() { + encoder := base64.NewEncoder(base64.StdEncoding, contentWriter) + if _, err := utils.CopyWithBuffer(encoder, s); err != nil { + _ = contentWriter.CloseWithError(err) + return + } + _ = encoder.Close() + _ = contentWriter.Close() + }() + afterContentReader := strings.NewReader(afterContent) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("https://api.github.com/repos/%s/%s/git/blobs", d.Owner, d.Repo), + driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: io.MultiReader(beforeContentReader, contentReader, afterContentReader), + Size: length, + }, + UpdateProgress: up, + })) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + token := strings.TrimSpace(d.Token) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + req.ContentLength = length + + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + resBody, err := io.ReadAll(res.Body) + if err != nil { + return "", err + } + if res.StatusCode != 201 { + var errMsg ErrResp + if err = utils.Json.Unmarshal(resBody, &errMsg); err != nil { + return "", errors.New(res.Status) + } else { + return "", fmt.Errorf("%s: %s", res.Status, errMsg.Message) + } + } + var resp PutBlobResp + if err = utils.Json.Unmarshal(resBody, &resp); err != nil { + return "", err + } + return resp.Sha, nil +} + +func (d *Github) renewParentTrees(path, prevSha, curSha, until string) (string, error) { + for path != until { + path = stdpath.Dir(path) + tree, sha, err := d.getTreeDirectly(path) + if err != nil { + return "", err + } + var newTree *TreeObjReq = nil + for _, t := range tree.Trees { + if t.Sha == prevSha { + newTree = &t.TreeObjReq + newTree.Sha = curSha + break + } + } + if newTree == nil { + return "", errs.ObjectNotFound + } + curSha, err = d.newTree(sha, []interface{}{*newTree}) + if err != nil { + return "", err + } + prevSha = sha + } + return curSha, nil +} + +func (d *Github) getTree(sha string) (*TreeResp, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees/%s", d.Owner, d.Repo, sha)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp TreeResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Github) getTreeDirectly(path string) (*TreeResp, string, error) { + p, err := d.get(path) + if err != nil { + return nil, "", err + } + if p.Entries == nil { + return nil, "", fmt.Errorf("%s is not a folder", path) + } + tree, err := d.getTree(p.Sha) + if err != nil { + return nil, "", err + } + if tree.Truncated { + return nil, "", fmt.Errorf("tree %s is truncated", path) + } + return tree, p.Sha, nil +} + +func (d *Github) newTree(baseSha string, tree []interface{}) (string, error) { + body := &TreeReq{Trees: tree} + if baseSha != "" { + body.BaseTree = baseSha + } + res, err := d.client.R().SetBody(body). + Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees", d.Owner, d.Repo)) + if err != nil { + return "", err + } + if res.StatusCode() != 201 { + return "", toErr(res) + } + var resp TreeResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + return resp.Sha, nil +} + +func (d *Github) commit(message, treeSha string) error { + oldCommit, err := d.getBranchHead() + body := map[string]interface{}{ + "message": message, + "tree": treeSha, + "parents": []string{oldCommit}, + } + d.addCommitterAndAuthor(&body) + if d.pgpEntity != nil { + signature, e := signCommit(&body, d.pgpEntity) + if e != nil { + return e + } + body["signature"] = signature + } + res, err := d.client.R().SetBody(body).Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/commits", d.Owner, d.Repo)) + if err != nil { + return err + } + if res.StatusCode() != 201 { + return toErr(res) + } + var resp CommitResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return err + } + + // update branch head + res, err = d.client.R(). + SetBody(&UpdateRefReq{ + Sha: resp.Sha, + Force: false, + }). + Patch(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/refs/heads/%s", d.Owner, d.Repo, d.Ref)) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toErr(res) + } + return nil +} + +func (d *Github) getBranchHead() (string, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/branches/%s", d.Owner, d.Repo, d.Ref)) + if err != nil { + return "", err + } + if res.StatusCode() != 200 { + return "", toErr(res) + } + var resp BranchResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + return resp.Commit.Sha, nil +} + +func (d *Github) copyWithoutRenewTree(srcObj, dstDir model.Obj) (dstSha, newSha, srcParentSha string, srcParentTree *TreeResp, err error) { + dst, err := d.get(dstDir.GetPath()) + if err != nil { + return "", "", "", nil, err + } + if dst.Entries == nil { + return "", "", "", nil, errs.NotFolder + } + dstSha = dst.Sha + srcParentPath := stdpath.Dir(srcObj.GetPath()) + srcParentTree, srcParentSha, err = d.getTreeDirectly(srcParentPath) + if err != nil { + return "", "", "", nil, err + } + var src *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return "", "", "", nil, errors.New("cannot copy a submodule") + } + src = &t.TreeObjReq + break + } + } + if src == nil { + return "", "", "", nil, errs.ObjectNotFound + } + + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, *src) + if len(dst.Entries) == 1 && dst.Entries[0].Name == ".gitkeep" { + newTree = append(newTree, TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + }) + } + newSha, err = d.newTree(dstSha, newTree) + if err != nil { + return "", "", "", nil, err + } + return dstSha, newSha, srcParentSha, srcParentTree, nil +} + +func (d *Github) getRepo() (*RepoResp, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s", d.Owner, d.Repo)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp RepoResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Github) getAuthenticatedUser() (*UserResp, error) { + res, err := d.client.R().Get("https://api.github.com/user") + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + resp := &UserResp{} + if err = utils.Json.Unmarshal(res.Body(), resp); err != nil { + return nil, err + } + return resp, nil +} + +func (d *Github) addCommitterAndAuthor(m *map[string]interface{}) { + if d.CommitterName != "" { + committer := map[string]string{ + "name": d.CommitterName, + "email": d.CommitterEmail, + } + (*m)["committer"] = committer + } + if d.AuthorName != "" { + author := map[string]string{ + "name": d.AuthorName, + "email": d.AuthorEmail, + } + (*m)["author"] = author + } +} diff --git a/drivers/github/meta.go b/drivers/github/meta.go new file mode 100644 index 00000000000..7de8d73c391 --- /dev/null +++ b/drivers/github/meta.go @@ -0,0 +1,39 @@ +package github + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Token string `json:"token" type:"string" required:"true"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string" help:"A branch, a tag or a commit SHA, main branch by default."` + GitHubProxy string `json:"gh_proxy" type:"string" help:"GitHub proxy, e.g. https://ghproxy.net/raw.githubusercontent.com or https://gh-proxy.com/raw.githubusercontent.com"` + GPGPrivateKey string `json:"gpg_private_key" type:"text"` + GPGKeyPassphrase string `json:"gpg_key_passphrase" type:"string"` + CommitterName string `json:"committer_name" type:"string"` + CommitterEmail string `json:"committer_email" type:"string"` + AuthorName string `json:"author_name" type:"string"` + AuthorEmail string `json:"author_email" type:"string"` + MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` + DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` + PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` + RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` + CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` + MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` +} + +var config = driver.Config{ + Name: "GitHub API", + LocalSort: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Github{} + }) +} diff --git a/drivers/github/types.go b/drivers/github/types.go new file mode 100644 index 00000000000..b057385cdf5 --- /dev/null +++ b/drivers/github/types.go @@ -0,0 +1,107 @@ +package github + +import ( + "github.com/alist-org/alist/v3/internal/model" + "time" +) + +type Links struct { + Git string `json:"git"` + Html string `json:"html"` + Self string `json:"self"` +} + +type Object struct { + Type string `json:"type"` + Encoding string `json:"encoding" required:"false"` + Size int64 `json:"size"` + Name string `json:"name"` + Path string `json:"path"` + Content string `json:"Content" required:"false"` + Sha string `json:"sha"` + URL string `json:"url"` + GitURL string `json:"git_url"` + HtmlURL string `json:"html_url"` + DownloadURL string `json:"download_url"` + Entries []Object `json:"entries" required:"false"` + Links Links `json:"_links"` + SubmoduleGitURL string `json:"submodule_git_url" required:"false"` + Target string `json:"target" required:"false"` +} + +func (o *Object) toModelObj() *model.Object { + return &model.Object{ + Name: o.Name, + Size: o.Size, + Modified: time.Unix(0, 0), + IsFolder: o.Type == "dir", + } +} + +type PutBlobResp struct { + URL string `json:"url"` + Sha string `json:"sha"` +} + +type ErrResp struct { + Message string `json:"message"` + DocumentationURL string `json:"documentation_url"` + Status string `json:"status"` +} + +type TreeObjReq struct { + Path string `json:"path"` + Mode string `json:"mode"` + Type string `json:"type"` + Sha interface{} `json:"sha"` +} + +type TreeObjResp struct { + TreeObjReq + Size int64 `json:"size" required:"false"` + URL string `json:"url"` +} + +func (o *TreeObjResp) toModelObj() *model.Object { + return &model.Object{ + Name: o.Path, + Size: o.Size, + Modified: time.Unix(0, 0), + IsFolder: o.Type == "tree", + } +} + +type TreeResp struct { + Sha string `json:"sha"` + URL string `json:"url"` + Trees []TreeObjResp `json:"tree"` + Truncated bool `json:"truncated"` +} + +type TreeReq struct { + BaseTree interface{} `json:"base_tree,omitempty"` + Trees []interface{} `json:"tree"` +} + +type CommitResp struct { + Sha string `json:"sha"` +} + +type BranchResp struct { + Name string `json:"name"` + Commit CommitResp `json:"commit"` +} + +type UpdateRefReq struct { + Sha string `json:"sha"` + Force bool `json:"force"` +} + +type RepoResp struct { + DefaultBranch string `json:"default_branch"` +} + +type UserResp struct { + Name string `json:"name"` + Email string `json:"email"` +} diff --git a/drivers/github/util.go b/drivers/github/util.go new file mode 100644 index 00000000000..7ddf8746c8f --- /dev/null +++ b/drivers/github/util.go @@ -0,0 +1,166 @@ +package github + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "text/template" + "time" + + "github.com/ProtonMail/go-crypto/openpgp" + "github.com/ProtonMail/go-crypto/openpgp/armor" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type MessageTemplateVars struct { + UserName string + ObjName string + ObjPath string + ParentName string + ParentPath string + TargetName string + TargetPath string +} + +func getMessage(tmpl *template.Template, vars *MessageTemplateVars, defaultOpStr string) (string, error) { + sb := strings.Builder{} + if err := tmpl.Execute(&sb, vars); err != nil { + return fmt.Sprintf("%s %s %s", vars.UserName, defaultOpStr, vars.ObjPath), err + } + return sb.String(), nil +} + +func calculateBase64Length(inputLength int64) int64 { + return 4 * ((inputLength + 2) / 3) +} + +func toErr(res *resty.Response) error { + var errMsg ErrResp + if err := utils.Json.Unmarshal(res.Body(), &errMsg); err != nil { + return errors.New(res.Status()) + } else { + return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) + } +} + +// Example input: +// a = /aaa/bbb/ccc +// b = /aaa/b11/ddd/ccc +// +// Output: +// ancestor = /aaa +// aChildName = bbb +// bChildName = b11 +// aRest = bbb/ccc +// bRest = b11/ddd/ccc +func getPathCommonAncestor(a, b string) (ancestor, aChildName, bChildName, aRest, bRest string) { + a = utils.FixAndCleanPath(a) + b = utils.FixAndCleanPath(b) + idx := 1 + for idx < len(a) && idx < len(b) { + if a[idx] != b[idx] { + break + } + idx++ + } + aNextIdx := idx + for aNextIdx < len(a) { + if a[aNextIdx] == '/' { + break + } + aNextIdx++ + } + bNextIdx := idx + for bNextIdx < len(b) { + if b[bNextIdx] == '/' { + break + } + bNextIdx++ + } + for idx > 0 { + if a[idx] == '/' { + break + } + idx-- + } + ancestor = utils.FixAndCleanPath(a[:idx]) + aChildName = a[idx+1 : aNextIdx] + bChildName = b[idx+1 : bNextIdx] + aRest = a[idx+1:] + bRest = b[idx+1:] + return ancestor, aChildName, bChildName, aRest, bRest +} + +func getUsername(ctx context.Context) string { + user, ok := ctx.Value("user").(*model.User) + if !ok { + return "" + } + return user.Username +} + +func loadPrivateKey(key, passphrase string) (*openpgp.Entity, error) { + entityList, err := openpgp.ReadArmoredKeyRing(strings.NewReader(key)) + if err != nil { + return nil, err + } + if len(entityList) < 1 { + return nil, fmt.Errorf("no keys found in key ring") + } + entity := entityList[0] + + pass := []byte(passphrase) + if entity.PrivateKey != nil && entity.PrivateKey.Encrypted { + if err = entity.PrivateKey.Decrypt(pass); err != nil { + return nil, fmt.Errorf("password incorrect: %+v", err) + } + } + for _, subKey := range entity.Subkeys { + if subKey.PrivateKey != nil && subKey.PrivateKey.Encrypted { + if err = subKey.PrivateKey.Decrypt(pass); err != nil { + return nil, fmt.Errorf("password incorrect: %+v", err) + } + } + } + return entity, nil +} + +func signCommit(m *map[string]interface{}, entity *openpgp.Entity) (string, error) { + var commit strings.Builder + commit.WriteString(fmt.Sprintf("tree %s\n", (*m)["tree"].(string))) + parents := (*m)["parents"].([]string) + for _, p := range parents { + commit.WriteString(fmt.Sprintf("parent %s\n", p)) + } + now := time.Now() + _, offset := now.Zone() + hour := offset / 3600 + author := (*m)["author"].(map[string]string) + commit.WriteString(fmt.Sprintf("author %s <%s> %d %+03d00\n", author["name"], author["email"], now.Unix(), hour)) + author["date"] = now.Format(time.RFC3339) + committer := (*m)["committer"].(map[string]string) + commit.WriteString(fmt.Sprintf("committer %s <%s> %d %+03d00\n", committer["name"], committer["email"], now.Unix(), hour)) + committer["date"] = now.Format(time.RFC3339) + commit.WriteString(fmt.Sprintf("\n%s", (*m)["message"].(string))) + data := commit.String() + + var sigBuffer bytes.Buffer + err := openpgp.DetachSign(&sigBuffer, entity, strings.NewReader(data), nil) + if err != nil { + return "", fmt.Errorf("signing failed: %v", err) + } + var armoredSig bytes.Buffer + armorWriter, err := armor.Encode(&armoredSig, "PGP SIGNATURE", nil) + if err != nil { + return "", err + } + if _, err = utils.CopyWithBuffer(armorWriter, &sigBuffer); err != nil { + return "", err + } + _ = armorWriter.Close() + return armoredSig.String(), nil +} diff --git a/drivers/github_releases/driver.go b/drivers/github_releases/driver.go new file mode 100644 index 00000000000..3268dc2fd88 --- /dev/null +++ b/drivers/github_releases/driver.go @@ -0,0 +1,210 @@ +package github_releases + +import ( + "context" + "fmt" + "net/http" + "strings" + "sync" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type GithubReleases struct { + model.Storage + Addition + + points []MountPoint +} + +func (d *GithubReleases) Config() driver.Config { + return config +} + +func (d *GithubReleases) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GithubReleases) Init(ctx context.Context) error { + d.ParseRepos(d.Addition.RepoStructure) + return nil +} + +func (d *GithubReleases) Drop(ctx context.Context) error { + return nil +} + +// processPoint 处理单个挂载点的文件列表 +func (d *GithubReleases) processPoint(point *MountPoint, path string, args model.ListArgs) []File { + var pointFiles []File + + if !d.Addition.ShowAllVersion { // latest + point.RequestLatestRelease(d.GetRequest, args.Refresh) + pointFiles = d.processLatestVersion(point, path) + } else { // all version + point.RequestReleases(d.GetRequest, args.Refresh) + pointFiles = d.processAllVersions(point, path) + } + + return pointFiles +} + +// processLatestVersion 处理最新版本的逻辑 +func (d *GithubReleases) processLatestVersion(point *MountPoint, path string) []File { + var pointFiles []File + + if point.Point == path { // 与仓库路径相同 + pointFiles = append(pointFiles, point.GetLatestRelease()...) + if d.Addition.ShowReadme { + files := point.GetOtherFile(d.GetRequest, false) + pointFiles = append(pointFiles, files...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir != "" { + dirFile := File{ + Path: path + "/" + nextDir, + FileName: nextDir, + Size: point.GetLatestSize(), + UpdateAt: point.Release.PublishedAt, + CreateAt: point.Release.CreatedAt, + Type: "dir", + Url: "", + } + pointFiles = append(pointFiles, dirFile) + } + } + + return pointFiles +} + +// processAllVersions 处理所有版本的逻辑 +func (d *GithubReleases) processAllVersions(point *MountPoint, path string) []File { + var pointFiles []File + + if point.Point == path { // 与仓库路径相同 + pointFiles = append(pointFiles, point.GetAllVersion()...) + if d.Addition.ShowReadme { + files := point.GetOtherFile(d.GetRequest, false) + pointFiles = append(pointFiles, files...) + } + } else if strings.HasPrefix(point.Point, path) { // 仓库目录的父目录 + nextDir := GetNextDir(point.Point, path) + if nextDir != "" { + dirFile := File{ + FileName: nextDir, + Path: path + "/" + nextDir, + Size: point.GetAllVersionSize(), + UpdateAt: (*point.Releases)[0].PublishedAt, + CreateAt: (*point.Releases)[0].CreatedAt, + Type: "dir", + Url: "", + } + pointFiles = append(pointFiles, dirFile) + } + } else if strings.HasPrefix(path, point.Point) { // 仓库目录的子目录 + tagName := GetNextDir(path, point.Point) + if tagName != "" { + pointFiles = append(pointFiles, point.GetReleaseByTagName(tagName)...) + } + } + + return pointFiles +} + +// mergeFiles 合并文件列表,处理重复目录 +func (d *GithubReleases) mergeFiles(files *[]File, newFiles []File) { + for _, newFile := range newFiles { + if newFile.Type == "dir" { + hasSameDir := false + for index := range *files { + if (*files)[index].GetName() == newFile.GetName() && (*files)[index].Type == "dir" { + hasSameDir = true + (*files)[index].Size += newFile.Size + break + } + } + if !hasSameDir { + *files = append(*files, newFile) + } + } else { + *files = append(*files, newFile) + } + } +} + +func (d *GithubReleases) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]File, 0) + path := fmt.Sprintf("/%s", strings.Trim(dir.GetPath(), "/")) + + if d.Addition.ConcurrentRequests && d.Addition.Token != "" { // 并发处理 + var mu sync.Mutex + var wg sync.WaitGroup + + for i := range d.points { + wg.Add(1) + go func(point *MountPoint) { + defer wg.Done() + pointFiles := d.processPoint(point, path, args) + + mu.Lock() + d.mergeFiles(&files, pointFiles) + mu.Unlock() + }(&d.points[i]) + } + wg.Wait() + } else { // 串行处理 + for i := range d.points { + point := &d.points[i] + pointFiles := d.processPoint(point, path, args) + d.mergeFiles(&files, pointFiles) + } + } + + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return src, nil + }) +} + +func (d *GithubReleases) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + url := file.GetID() + gh_proxy := strings.TrimSpace(d.Addition.GitHubProxy) + + if gh_proxy != "" { + url = strings.Replace(url, "https://github.com", gh_proxy, 1) + } + + link := model.Link{ + URL: url, + Header: http.Header{}, + } + return &link, nil +} + +func (d *GithubReleases) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + // TODO create folder, optional + return nil, errs.NotImplement +} + +func (d *GithubReleases) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO move obj, optional + return nil, errs.NotImplement +} + +func (d *GithubReleases) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + // TODO rename obj, optional + return nil, errs.NotImplement +} + +func (d *GithubReleases) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *GithubReleases) Remove(ctx context.Context, obj model.Obj) error { + // TODO remove obj, optional + return errs.NotImplement +} diff --git a/drivers/github_releases/meta.go b/drivers/github_releases/meta.go new file mode 100644 index 00000000000..b54cb3cc608 --- /dev/null +++ b/drivers/github_releases/meta.go @@ -0,0 +1,36 @@ +package github_releases + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + RepoStructure string `json:"repo_structure" type:"text" required:"true" default:"alistGo/alist" help:"structure:[path:]org/repo"` + ShowReadme bool `json:"show_readme" type:"bool" default:"true" help:"show README、LICENSE file"` + Token string `json:"token" type:"string" required:"false" help:"GitHub token, if you want to access private repositories or increase the rate limit"` + ShowAllVersion bool `json:"show_all_version" type:"bool" default:"false" help:"show all versions"` + ConcurrentRequests bool `json:"concurrent_requests" type:"bool" default:"false" help:"To concurrently request the GitHub API, you must enter a GitHub token"` + GitHubProxy string `json:"gh_proxy" type:"string" default:"" help:"GitHub proxy, e.g. https://ghproxy.net/github.com or https://gh-proxy.com/github.com "` +} + +var config = driver.Config{ + Name: "GitHub Releases", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GithubReleases{} + }) +} diff --git a/drivers/github_releases/models.go b/drivers/github_releases/models.go new file mode 100644 index 00000000000..a9a0e493c44 --- /dev/null +++ b/drivers/github_releases/models.go @@ -0,0 +1,86 @@ +package github_releases + +type Release struct { + Url string `json:"url"` + AssetsUrl string `json:"assets_url"` + UploadUrl string `json:"upload_url"` + HtmlUrl string `json:"html_url"` + Id int `json:"id"` + Author User `json:"author"` + NodeId string `json:"node_id"` + TagName string `json:"tag_name"` + TargetCommitish string `json:"target_commitish"` + Name string `json:"name"` + Draft bool `json:"draft"` + Prerelease bool `json:"prerelease"` + CreatedAt string `json:"created_at"` + PublishedAt string `json:"published_at"` + Assets []Asset `json:"assets"` + TarballUrl string `json:"tarball_url"` + ZipballUrl string `json:"zipball_url"` + Body string `json:"body"` + Reactions Reactions `json:"reactions"` +} + +type User struct { + Login string `json:"login"` + Id int `json:"id"` + NodeId string `json:"node_id"` + AvatarUrl string `json:"avatar_url"` + GravatarId string `json:"gravatar_id"` + Url string `json:"url"` + HtmlUrl string `json:"html_url"` + FollowersUrl string `json:"followers_url"` + FollowingUrl string `json:"following_url"` + GistsUrl string `json:"gists_url"` + StarredUrl string `json:"starred_url"` + SubscriptionsUrl string `json:"subscriptions_url"` + OrganizationsUrl string `json:"organizations_url"` + ReposUrl string `json:"repos_url"` + EventsUrl string `json:"events_url"` + ReceivedEventsUrl string `json:"received_events_url"` + Type string `json:"type"` + UserViewType string `json:"user_view_type"` + SiteAdmin bool `json:"site_admin"` +} + +type Asset struct { + Url string `json:"url"` + Id int `json:"id"` + NodeId string `json:"node_id"` + Name string `json:"name"` + Label string `json:"label"` + Uploader User `json:"uploader"` + ContentType string `json:"content_type"` + State string `json:"state"` + Size int64 `json:"size"` + DownloadCount int `json:"download_count"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + BrowserDownloadUrl string `json:"browser_download_url"` +} + +type Reactions struct { + Url string `json:"url"` + TotalCount int `json:"total_count"` + PlusOne int `json:"+1"` + MinusOne int `json:"-1"` + Laugh int `json:"laugh"` + Hooray int `json:"hooray"` + Confused int `json:"confused"` + Heart int `json:"heart"` + Rocket int `json:"rocket"` + Eyes int `json:"eyes"` +} + +type FileInfo struct { + Name string `json:"name"` + Path string `json:"path"` + Sha string `json:"sha"` + Size int64 `json:"size"` + Url string `json:"url"` + HtmlUrl string `json:"html_url"` + GitUrl string `json:"git_url"` + DownloadUrl string `json:"download_url"` + Type string `json:"type"` +} diff --git a/drivers/github_releases/types.go b/drivers/github_releases/types.go new file mode 100644 index 00000000000..b4562056185 --- /dev/null +++ b/drivers/github_releases/types.go @@ -0,0 +1,213 @@ +package github_releases + +import ( + "encoding/json" + "strings" + "time" + + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type MountPoint struct { + Point string // 挂载点 + Repo string // 仓库名 owner/repo + Release *Release // Release 指针 latest + Releases *[]Release // []Release 指针 + OtherFile *[]FileInfo // 仓库根目录下的其他文件 +} + +// 请求最新版本 +func (m *MountPoint) RequestLatestRelease(get func(url string) (*resty.Response, error), refresh bool) { + if m.Repo == "" { + return + } + + if m.Release == nil || refresh { + resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases/latest") + m.Release = new(Release) + json.Unmarshal(resp.Body(), m.Release) + } +} + +// 请求所有版本 +func (m *MountPoint) RequestReleases(get func(url string) (*resty.Response, error), refresh bool) { + if m.Repo == "" { + return + } + + if m.Releases == nil || refresh { + resp, _ := get("https://api.github.com/repos/" + m.Repo + "/releases") + m.Releases = new([]Release) + json.Unmarshal(resp.Body(), m.Releases) + } +} + +// 获取最新版本 +func (m *MountPoint) GetLatestRelease() []File { + files := make([]File, 0) + for _, asset := range m.Release.Assets { + files = append(files, File{ + Path: m.Point + "/" + asset.Name, + FileName: asset.Name, + Size: asset.Size, + Type: "file", + UpdateAt: asset.UpdatedAt, + CreateAt: asset.CreatedAt, + Url: asset.BrowserDownloadUrl, + }) + } + return files +} + +// 获取最新版本大小 +func (m *MountPoint) GetLatestSize() int64 { + size := int64(0) + for _, asset := range m.Release.Assets { + size += asset.Size + } + return size +} + +// 获取所有版本 +func (m *MountPoint) GetAllVersion() []File { + files := make([]File, 0) + for _, release := range *m.Releases { + file := File{ + Path: m.Point + "/" + release.TagName, + FileName: release.TagName, + Size: m.GetSizeByTagName(release.TagName), + Type: "dir", + UpdateAt: release.PublishedAt, + CreateAt: release.CreatedAt, + Url: release.HtmlUrl, + } + for _, asset := range release.Assets { + file.Size += asset.Size + } + files = append(files, file) + } + return files +} + +// 根据版本号获取版本 +func (m *MountPoint) GetReleaseByTagName(tagName string) []File { + for _, item := range *m.Releases { + if item.TagName == tagName { + files := make([]File, 0) + for _, asset := range item.Assets { + files = append(files, File{ + Path: m.Point + "/" + tagName + "/" + asset.Name, + FileName: asset.Name, + Size: asset.Size, + Type: "file", + UpdateAt: asset.UpdatedAt, + CreateAt: asset.CreatedAt, + Url: asset.BrowserDownloadUrl, + }) + } + return files + } + } + return nil +} + +// 根据版本号获取版本大小 +func (m *MountPoint) GetSizeByTagName(tagName string) int64 { + if m.Releases == nil { + return 0 + } + for _, item := range *m.Releases { + if item.TagName == tagName { + size := int64(0) + for _, asset := range item.Assets { + size += asset.Size + } + return size + } + } + return 0 +} + +// 获取所有版本大小 +func (m *MountPoint) GetAllVersionSize() int64 { + if m.Releases == nil { + return 0 + } + size := int64(0) + for _, release := range *m.Releases { + for _, asset := range release.Assets { + size += asset.Size + } + } + return size +} + +func (m *MountPoint) GetOtherFile(get func(url string) (*resty.Response, error), refresh bool) []File { + if m.OtherFile == nil || refresh { + resp, _ := get("https://api.github.com/repos/" + m.Repo + "/contents") + m.OtherFile = new([]FileInfo) + json.Unmarshal(resp.Body(), m.OtherFile) + } + + files := make([]File, 0) + defaultTime := "1970-01-01T00:00:00Z" + for _, file := range *m.OtherFile { + if strings.HasSuffix(file.Name, ".md") || strings.HasPrefix(file.Name, "LICENSE") { + files = append(files, File{ + Path: m.Point + "/" + file.Name, + FileName: file.Name, + Size: file.Size, + Type: "file", + UpdateAt: defaultTime, + CreateAt: defaultTime, + Url: file.DownloadUrl, + }) + } + } + return files +} + +type File struct { + Path string // 文件路径 + FileName string // 文件名 + Size int64 // 文件大小 + Type string // 文件类型 + UpdateAt string // 更新时间 eg:"2025-01-27T16:10:16Z" + CreateAt string // 创建时间 + Url string // 下载链接 +} + +func (f File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f File) GetPath() string { + return f.Path +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) ModTime() time.Time { + t, _ := time.Parse(time.RFC3339, f.CreateAt) + return t +} + +func (f File) CreateTime() time.Time { + t, _ := time.Parse(time.RFC3339, f.CreateAt) + return t +} + +func (f File) IsDir() bool { + return f.Type == "dir" +} + +func (f File) GetID() string { + return f.Url +} diff --git a/drivers/github_releases/util.go b/drivers/github_releases/util.go new file mode 100644 index 00000000000..097295bf408 --- /dev/null +++ b/drivers/github_releases/util.go @@ -0,0 +1,85 @@ +package github_releases + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +// 发送 GET 请求 +func (d *GithubReleases) GetRequest(url string) (*resty.Response, error) { + req := base.RestyClient.R() + req.SetHeader("Accept", "application/vnd.github+json") + req.SetHeader("X-GitHub-Api-Version", "2022-11-28") + if d.Addition.Token != "" { + req.SetHeader("Authorization", fmt.Sprintf("Bearer %s", d.Addition.Token)) + } + res, err := req.Get(url) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + utils.Log.Warnf("failed to get request: %s %d %s", url, res.StatusCode(), res.String()) + } + return res, nil +} + +// 解析挂载结构 +func (d *GithubReleases) ParseRepos(text string) ([]MountPoint, error) { + lines := strings.Split(text, "\n") + points := make([]MountPoint, 0) + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + parts := strings.Split(line, ":") + path, repo := "", "" + if len(parts) == 1 { + path = "/" + repo = parts[0] + } else if len(parts) == 2 { + path = fmt.Sprintf("/%s", strings.Trim(parts[0], "/")) + repo = parts[1] + } else { + return nil, fmt.Errorf("invalid format: %s", line) + } + + points = append(points, MountPoint{ + Point: path, + Repo: repo, + Release: nil, + Releases: nil, + }) + } + d.points = points + return points, nil +} + +// 获取下一级目录 +func GetNextDir(wholePath string, basePath string) string { + basePath = fmt.Sprintf("%s/", strings.TrimRight(basePath, "/")) + if !strings.HasPrefix(wholePath, basePath) { + return "" + } + remainingPath := strings.TrimLeft(strings.TrimPrefix(wholePath, basePath), "/") + if remainingPath != "" { + parts := strings.Split(remainingPath, "/") + nextDir := parts[0] + if strings.HasPrefix(wholePath, strings.TrimRight(basePath, "/")+"/"+nextDir) { + return nextDir + } + } + return "" +} + +// 判断当前目录是否是目标目录的祖先目录 +func IsAncestorDir(parentDir string, targetDir string) bool { + absTargetDir, _ := filepath.Abs(targetDir) + absParentDir, _ := filepath.Abs(parentDir) + return strings.HasPrefix(absTargetDir, absParentDir) +} diff --git a/drivers/gofile/driver.go b/drivers/gofile/driver.go new file mode 100644 index 00000000000..8046bd163fb --- /dev/null +++ b/drivers/gofile/driver.go @@ -0,0 +1,271 @@ +package gofile + +import ( + "context" + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" +) + +type Gofile struct { + model.Storage + Addition + + accountId string +} + +func (d *Gofile) Config() driver.Config { + return config +} + +func (d *Gofile) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Gofile) Init(ctx context.Context) error { + if d.APIToken == "" { + return fmt.Errorf("API token is required") + } + + // Get account ID + accountId, err := d.getAccountId(ctx) + if err != nil { + return fmt.Errorf("failed to get account ID: %w", err) + } + d.accountId = accountId + + // Get account info to set root folder if not specified + if d.RootFolderID == "" { + accountInfo, err := d.getAccountInfo(ctx, accountId) + if err != nil { + return fmt.Errorf("failed to get account info: %w", err) + } + d.RootFolderID = accountInfo.Data.RootFolder + } + + // Save driver storage + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Gofile) Drop(ctx context.Context) error { + return nil +} + +func (d *Gofile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var folderId string + if dir.GetID() == "" { + folderId = d.GetRootId() + } else { + folderId = dir.GetID() + } + + endpoint := fmt.Sprintf("/contents/%s", folderId) + + var response ContentsResponse + err := d.getJSON(ctx, endpoint, &response) + if err != nil { + return nil, err + } + + var objects []model.Obj + + // Process children or contents + contents := response.Data.Children + if contents == nil { + contents = response.Data.Contents + } + + for _, content := range contents { + objects = append(objects, d.convertContentToObj(content)) + } + + return objects, nil +} + +func (d *Gofile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + + // Create a direct link for the file + directLink, err := d.createDirectLink(ctx, file.GetID()) + if err != nil { + return nil, fmt.Errorf("failed to create direct link: %w", err) + } + + // Configure cache expiration based on user setting + link := &model.Link{ + URL: directLink, + } + + // Only set expiration if LinkExpiry > 0 (0 means no caching) + if d.LinkExpiry > 0 { + expiration := time.Duration(d.LinkExpiry) * 24 * time.Hour + link.Expiration = &expiration + } + + return link, nil +} + +func (d *Gofile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var parentId string + if parentDir.GetID() == "" { + parentId = d.GetRootId() + } else { + parentId = parentDir.GetID() + } + + data := map[string]interface{}{ + "parentFolderId": parentId, + "folderName": dirName, + } + + var response CreateFolderResponse + err := d.postJSON(ctx, "/contents/createFolder", data, &response) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: response.Data.ID, + Name: response.Data.Name, + IsFolder: true, + }, nil +} + +func (d *Gofile) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var dstId string + if dstDir.GetID() == "" { + dstId = d.GetRootId() + } else { + dstId = dstDir.GetID() + } + + data := map[string]interface{}{ + "contentsId": srcObj.GetID(), + "folderId": dstId, + } + + err := d.putJSON(ctx, "/contents/move", data, nil) + if err != nil { + return nil, err + } + + // Return updated object + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + data := map[string]interface{}{ + "attribute": "name", + "attributeValue": newName, + } + + var response UpdateResponse + err := d.putJSON(ctx, fmt.Sprintf("/contents/%s/update", srcObj.GetID()), data, &response) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var dstId string + if dstDir.GetID() == "" { + dstId = d.GetRootId() + } else { + dstId = dstDir.GetID() + } + + data := map[string]interface{}{ + "contentsId": srcObj.GetID(), + "folderId": dstId, + } + + var response CopyResponse + err := d.postJSON(ctx, "/contents/copy", data, &response) + if err != nil { + return nil, err + } + + // Get the new ID from the response + newId := srcObj.GetID() + if response.Data.CopiedContents != nil { + if id, ok := response.Data.CopiedContents[srcObj.GetID()]; ok { + newId = id + } + } + + return &model.Object{ + ID: newId, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Gofile) Remove(ctx context.Context, obj model.Obj) error { + data := map[string]interface{}{ + "contentsId": obj.GetID(), + } + + return d.deleteJSON(ctx, "/contents", data) +} + +func (d *Gofile) Put(ctx context.Context, dstDir model.Obj, fileStreamer model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var folderId string + if dstDir.GetID() == "" { + folderId = d.GetRootId() + } else { + folderId = dstDir.GetID() + } + + response, err := d.uploadFile(ctx, folderId, fileStreamer, up) + if err != nil { + return nil, err + } + + return &model.Object{ + ID: response.Data.FileId, + Name: response.Data.FileName, + Size: fileStreamer.GetSize(), + IsFolder: false, + }, nil +} + +func (d *Gofile) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Gofile) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*Gofile)(nil) diff --git a/drivers/gofile/meta.go b/drivers/gofile/meta.go new file mode 100644 index 00000000000..00656025770 --- /dev/null +++ b/drivers/gofile/meta.go @@ -0,0 +1,28 @@ +package gofile + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APIToken string `json:"api_token" required:"true" help:"Get your API token from your Gofile profile page"` + LinkExpiry int `json:"link_expiry" type:"number" default:"30" help:"Direct link cache duration in days. Set to 0 to disable caching"` + DirectLinkExpiry int `json:"direct_link_expiry" type:"number" default:"0" help:"Direct link expiration time in hours on Gofile server. Set to 0 for no expiration"` +} + +var config = driver.Config{ + Name: "Gofile", + DefaultRoot: "", + LocalSort: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Gofile{} + }) +} diff --git a/drivers/gofile/types.go b/drivers/gofile/types.go new file mode 100644 index 00000000000..be307347081 --- /dev/null +++ b/drivers/gofile/types.go @@ -0,0 +1,124 @@ +package gofile + +import "time" + +type APIResponse struct { + Status string `json:"status"` + Data interface{} `json:"data"` +} + +type AccountResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + } `json:"data"` +} + +type AccountInfoResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Email string `json:"email"` + RootFolder string `json:"rootFolder"` + } `json:"data"` +} + +type Content struct { + ID string `json:"id"` + Type string `json:"type"` // "file" or "folder" + Name string `json:"name"` + Size int64 `json:"size,omitempty"` + CreateTime int64 `json:"createTime"` + ModTime int64 `json:"modTime,omitempty"` + DirectLink string `json:"directLink,omitempty"` + Children map[string]Content `json:"children,omitempty"` + ParentFolder string `json:"parentFolder,omitempty"` + MD5 string `json:"md5,omitempty"` + MimeType string `json:"mimeType,omitempty"` + Link string `json:"link,omitempty"` +} + +type ContentsResponse struct { + Status string `json:"status"` + Data struct { + IsOwner bool `json:"isOwner"` + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ParentFolder string `json:"parentFolder"` + CreateTime int64 `json:"createTime"` + ChildrenList []string `json:"childrenList,omitempty"` + Children map[string]Content `json:"children,omitempty"` + Contents map[string]Content `json:"contents,omitempty"` + Public bool `json:"public,omitempty"` + Description string `json:"description,omitempty"` + Tags string `json:"tags,omitempty"` + Expiry int64 `json:"expiry,omitempty"` + } `json:"data"` +} + +type UploadResponse struct { + Status string `json:"status"` + Data struct { + DownloadPage string `json:"downloadPage"` + Code string `json:"code"` + ParentFolder string `json:"parentFolder"` + FileId string `json:"fileId"` + FileName string `json:"fileName"` + GuestToken string `json:"guestToken,omitempty"` + } `json:"data"` +} + +type DirectLinkResponse struct { + Status string `json:"status"` + Data struct { + DirectLink string `json:"directLink"` + ID string `json:"id"` + } `json:"data"` +} + +type CreateFolderResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Type string `json:"type"` + Name string `json:"name"` + ParentFolder string `json:"parentFolder"` + CreateTime int64 `json:"createTime"` + } `json:"data"` +} + +type CopyResponse struct { + Status string `json:"status"` + Data struct { + CopiedContents map[string]string `json:"copiedContents"` // oldId -> newId mapping + } `json:"data"` +} + +type UpdateResponse struct { + Status string `json:"status"` + Data struct { + ID string `json:"id"` + Name string `json:"name"` + } `json:"data"` +} + +type ErrorResponse struct { + Status string `json:"status"` + Error struct { + Message string `json:"message"` + Code string `json:"code"` + } `json:"error"` +} + +func (c *Content) ModifiedTime() time.Time { + if c.ModTime > 0 { + return time.Unix(c.ModTime, 0) + } + return time.Unix(c.CreateTime, 0) +} + +func (c *Content) IsDir() bool { + return c.Type == "folder" +} diff --git a/drivers/gofile/util.go b/drivers/gofile/util.go new file mode 100644 index 00000000000..1dd6229a773 --- /dev/null +++ b/drivers/gofile/util.go @@ -0,0 +1,265 @@ +package gofile + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + log "github.com/sirupsen/logrus" +) + +const ( + baseAPI = "https://api.gofile.io" + uploadAPI = "https://upload.gofile.io" +) + +func (d *Gofile) request(ctx context.Context, method, endpoint string, body io.Reader, headers map[string]string) (*http.Response, error) { + var url string + if strings.HasPrefix(endpoint, "http") { + url = endpoint + } else { + url = baseAPI + endpoint + } + + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", "Bearer "+d.APIToken) + req.Header.Set("User-Agent", "AList/3.0") + + for k, v := range headers { + req.Header.Set(k, v) + } + + return base.HttpClient.Do(req) +} + +func (d *Gofile) getJSON(ctx context.Context, endpoint string, result interface{}) error { + resp, err := d.request(ctx, "GET", endpoint, nil, nil) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + return json.NewDecoder(resp.Body).Decode(result) +} + +func (d *Gofile) postJSON(ctx context.Context, endpoint string, data interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "POST", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + if result != nil { + return json.NewDecoder(resp.Body).Decode(result) + } + + return nil +} + +func (d *Gofile) putJSON(ctx context.Context, endpoint string, data interface{}, result interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "PUT", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + if result != nil { + return json.NewDecoder(resp.Body).Decode(result) + } + + return nil +} + +func (d *Gofile) deleteJSON(ctx context.Context, endpoint string, data interface{}) error { + jsonData, err := json.Marshal(data) + if err != nil { + return err + } + + headers := map[string]string{ + "Content-Type": "application/json", + } + + resp, err := d.request(ctx, "DELETE", endpoint, bytes.NewBuffer(jsonData), headers) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return d.handleError(resp) + } + + return nil +} + +func (d *Gofile) handleError(resp *http.Response) error { + body, _ := io.ReadAll(resp.Body) + log.Debugf("Gofile API error (HTTP %d): %s", resp.StatusCode, string(body)) + + var errorResp ErrorResponse + if err := json.Unmarshal(body, &errorResp); err == nil && errorResp.Status == "error" { + return fmt.Errorf("gofile API error: %s (code: %s)", errorResp.Error.Message, errorResp.Error.Code) + } + + return fmt.Errorf("gofile API error: HTTP %d - %s", resp.StatusCode, string(body)) +} + +func (d *Gofile) uploadFile(ctx context.Context, folderId string, file model.FileStreamer, up driver.UpdateProgress) (*UploadResponse, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + if folderId != "" { + writer.WriteField("folderId", folderId) + } + + part, err := writer.CreateFormFile("file", filepath.Base(file.GetName())) + if err != nil { + return nil, err + } + + // Copy with progress tracking if available + if up != nil { + reader := &progressReader{ + reader: file, + total: file.GetSize(), + up: up, + } + _, err = io.Copy(part, reader) + } else { + _, err = io.Copy(part, file) + } + + if err != nil { + return nil, err + } + + writer.Close() + + headers := map[string]string{ + "Content-Type": writer.FormDataContentType(), + } + + resp, err := d.request(ctx, "POST", uploadAPI+"/uploadfile", &body, headers) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, d.handleError(resp) + } + + var result UploadResponse + err = json.NewDecoder(resp.Body).Decode(&result) + return &result, err +} + +func (d *Gofile) createDirectLink(ctx context.Context, contentId string) (string, error) { + data := map[string]interface{}{} + + if d.DirectLinkExpiry > 0 { + expireTime := time.Now().Add(time.Duration(d.DirectLinkExpiry) * time.Hour).Unix() + data["expireTime"] = expireTime + } + + var result DirectLinkResponse + err := d.postJSON(ctx, fmt.Sprintf("/contents/%s/directlinks", contentId), data, &result) + if err != nil { + return "", err + } + + return result.Data.DirectLink, nil +} + +func (d *Gofile) convertContentToObj(content Content) model.Obj { + return &model.ObjThumb{ + Object: model.Object{ + ID: content.ID, + Name: content.Name, + Size: content.Size, + Modified: content.ModifiedTime(), + IsFolder: content.IsDir(), + }, + } +} + +func (d *Gofile) getAccountId(ctx context.Context) (string, error) { + var result AccountResponse + err := d.getJSON(ctx, "/accounts/getid", &result) + if err != nil { + return "", err + } + return result.Data.ID, nil +} + +func (d *Gofile) getAccountInfo(ctx context.Context, accountId string) (*AccountInfoResponse, error) { + var result AccountInfoResponse + err := d.getJSON(ctx, fmt.Sprintf("/accounts/%s", accountId), &result) + if err != nil { + return nil, err + } + return &result, nil +} + +// progressReader wraps an io.Reader to track upload progress +type progressReader struct { + reader io.Reader + total int64 + read int64 + up driver.UpdateProgress +} + +func (pr *progressReader) Read(p []byte) (n int, err error) { + n, err = pr.reader.Read(p) + pr.read += int64(n) + if pr.up != nil && pr.total > 0 { + progress := float64(pr.read) * 100 / float64(pr.total) + pr.up(progress) + } + return n, err +} diff --git a/drivers/google_drive/driver.go b/drivers/google_drive/driver.go index dccdcea902f..c8afb08499b 100644 --- a/drivers/google_drive/driver.go +++ b/drivers/google_drive/driver.go @@ -158,7 +158,8 @@ func (d *GoogleDrive) Put(ctx context.Context, dstDir model.Obj, stream model.Fi putUrl := res.Header().Get("location") if stream.GetSize() < d.ChunkSize*1024*1024 { _, err = d.request(putUrl, http.MethodPut, func(req *resty.Request) { - req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)).SetBody(stream) + req.SetHeader("Content-Length", strconv.FormatInt(stream.GetSize(), 10)). + SetBody(driver.NewLimitedUploadStream(ctx, stream)) }, nil) } else { err = d.chunkUpload(ctx, stream, putUrl) diff --git a/drivers/google_drive/util.go b/drivers/google_drive/util.go index 2c1f13eb8a5..0fe543468b8 100644 --- a/drivers/google_drive/util.go +++ b/drivers/google_drive/util.go @@ -5,17 +5,16 @@ import ( "crypto/x509" "encoding/pem" "fmt" - "io/ioutil" "net/http" "os" "regexp" "strconv" "time" - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" "github.com/golang-jwt/jwt/v4" @@ -44,7 +43,7 @@ func (d *GoogleDrive) refreshToken() error { gdsaFileThis := d.RefreshToken if gdsaFile.IsDir() { if len(d.ServiceAccountFileList) <= 0 { - gdsaReadDir, gdsaDirErr := ioutil.ReadDir(d.RefreshToken) + gdsaReadDir, gdsaDirErr := os.ReadDir(d.RefreshToken) if gdsaDirErr != nil { log.Error("read dir fail") return gdsaDirErr @@ -76,7 +75,7 @@ func (d *GoogleDrive) refreshToken() error { } } - gdsaFileThisContent, err := ioutil.ReadFile(gdsaFileThis) + gdsaFileThisContent, err := os.ReadFile(gdsaFileThis) if err != nil { return err } @@ -127,8 +126,7 @@ func (d *GoogleDrive) refreshToken() error { } d.AccessToken = resp.AccessToken return nil - } - if gdsaFileErr != nil && os.IsExist(gdsaFileErr) { + } else if os.IsExist(gdsaFileErr) { return gdsaFileErr } url := "https://www.googleapis.com/oauth2/v4/token" @@ -230,6 +228,7 @@ func (d *GoogleDrive) chunkUpload(ctx context.Context, stream model.FileStreamer if err != nil { return err } + reader = driver.NewLimitedUploadStream(ctx, reader) _, err = d.request(url, http.MethodPut, func(req *resty.Request) { req.SetHeaders(map[string]string{ "Content-Length": strconv.FormatInt(chunkSize, 10), diff --git a/drivers/google_photo/driver.go b/drivers/google_photo/driver.go index b54132ef9ed..e6f0abc6416 100644 --- a/drivers/google_photo/driver.go +++ b/drivers/google_photo/driver.go @@ -124,7 +124,7 @@ func (d *GooglePhoto) Put(ctx context.Context, dstDir model.Obj, stream model.Fi } resp, err := d.request(postUrl, http.MethodPost, func(req *resty.Request) { - req.SetBody(stream).SetContext(ctx) + req.SetBody(driver.NewLimitedUploadStream(ctx, stream)).SetContext(ctx) }, nil, postHeaders) if err != nil { diff --git a/drivers/google_photo/util.go b/drivers/google_photo/util.go index fbbff9ab1cb..0fd271b9bb4 100644 --- a/drivers/google_photo/util.go +++ b/drivers/google_photo/util.go @@ -151,7 +151,7 @@ func (d *GooglePhoto) getMedia(id string) (MediaItem, error) { var resp MediaItem query := map[string]string{ - "fields": "baseUrl,mimeType", + "fields": "mediaMetadata,baseUrl,mimeType", } _, err := d.request(fmt.Sprintf("https://photoslibrary.googleapis.com/v1/mediaItems/%s", id), http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) diff --git a/drivers/guangyapan/driver.go b/drivers/guangyapan/driver.go new file mode 100644 index 00000000000..e18d1b795ca --- /dev/null +++ b/drivers/guangyapan/driver.go @@ -0,0 +1,1051 @@ +package guangyapan + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +const ( + accountBaseURL = "https://account.guangyapan.com" + apiBaseURL = "https://api.guangyapan.com" + defaultClient = "aMe-8VSlkrbQXpUR" +) + +type GuangYaPan struct { + model.Storage + Addition + + accountClient *resty.Client + apiClient *resty.Client + + resolvedRootFolderID string + rootFolderResolved bool +} + +func (d *GuangYaPan) Config() driver.Config { + return config +} + +func (d *GuangYaPan) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *GuangYaPan) Init(ctx context.Context) error { + d.ClientID = strings.TrimSpace(d.ClientID) + if d.ClientID == "" { + d.ClientID = defaultClient + } + d.DeviceID = normalizeDeviceID(d.DeviceID) + if d.DeviceID == "" { + d.DeviceID = randomDeviceID() + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + if d.OrderBy < 0 { + d.OrderBy = 3 + } + if d.SortType != 0 && d.SortType != 1 { + d.SortType = 1 + } + + d.RootPath = strings.TrimSpace(d.RootPath) + d.AccessToken = strings.TrimSpace(d.AccessToken) + d.RefreshToken = strings.TrimSpace(d.RefreshToken) + d.PhoneNumber = strings.TrimSpace(d.PhoneNumber) + d.VerifyCode = strings.TrimSpace(d.VerifyCode) + d.CaptchaToken = strings.TrimSpace(d.CaptchaToken) + d.VerificationID = strings.TrimSpace(d.VerificationID) + d.resolvedRootFolderID = "" + d.rootFolderResolved = false + + d.accountClient = base.NewRestyClient(). + SetBaseURL(accountBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("X-Device-Model", "chrome%2F147.0.0.0"). + SetHeader("X-Device-Name", "PC-Chrome"). + SetHeader("X-Device-Sign", "wdi10."+d.DeviceID+"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"). + SetHeader("X-Net-Work-Type", "NONE"). + SetHeader("X-OS-Version", "MacIntel"). + SetHeader("X-Platform-Version", "1"). + SetHeader("X-Protocol-Version", "301"). + SetHeader("X-Provider-Name", "NONE"). + SetHeader("X-SDK-Version", "9.0.2"). + SetHeader("X-Client-Id", d.ClientID). + SetHeader("X-Client-Version", "0.0.1"). + SetHeader("X-Device-Id", d.DeviceID) + if d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + d.apiClient = base.NewRestyClient(). + SetBaseURL(apiBaseURL). + SetHeader("Accept", "application/json, text/plain, */*"). + SetHeader("Content-Type", "application/json"). + SetHeader("Did", d.DeviceID). + SetHeader("Dt", "4") + + // Priority: access_token -> refresh_token -> sms login. + if d.AccessToken != "" { + if err := d.validateToken(ctx); err == nil { + return d.prepareRootFolder(ctx) + } + d.AccessToken = "" + } + if d.RefreshToken != "" { + if err := d.refreshToken(ctx); err == nil { + if err2 := d.validateToken(ctx); err2 == nil { + return d.prepareRootFolder(ctx) + } + } + } + // Two-stage SMS flow: + // 1) phone only + send_code=true: send code and cache verification_id (do not fail init). + // 2) phone + verify_code: complete login and save tokens. + if d.PhoneNumber != "" { + if d.canSMSLogin() { + if err := d.loginBySMSCode(ctx); err != nil { + return err + } + if err := d.validateToken(ctx); err != nil { + return err + } + return d.prepareRootFolder(ctx) + } + if d.SendCode { + d.setTempStatus("SMS sending in progress...") + if err := d.prepareSMSCode(ctx); err != nil { + d.setTempStatus(fmt.Sprintf("SMS send failed: %v. Please check captcha/meta and set send_code=true to retry.", err)) + log.Warnf("guangyapan: prepare sms code failed: %v", err) + } else { + d.setTempStatus("SMS sent successfully. Please fill verify_code and save to complete login.") + } + } + return nil + } + return errors.New("login failed: provide a valid access_token, or refresh_token, or phone_number + verify_code + captcha_token") +} + +func (d *GuangYaPan) Drop(ctx context.Context) error { + return nil +} + +func (d *GuangYaPan) GetRoot(ctx context.Context) (model.Obj, error) { + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + return &model.Object{ + ID: rootID, + Path: "/", + Name: "root", + Size: 0, + Modified: d.Modified, + IsFolder: true, + }, nil +} + +func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + parentID := dir.GetID() + + res := make([]model.Obj, 0, d.PageSize) + for page := 0; ; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": d.PageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + "fileTypes": []int{}, + } + if err := d.postAPI(ctx, "/userres/v1/file/get_file_list", body, &resp); err != nil { + return nil, err + } + for _, item := range resp.Data.List { + res = append(res, &model.Object{ + ID: item.FileID, + Path: parentID, + Name: item.FileName, + Size: item.FileSize, + Modified: unixOrZero(item.UTime), + Ctime: unixOrZero(item.CTime), + IsFolder: item.ResType == 2, + }) + } + if len(resp.Data.List) < d.PageSize { + break + } + if resp.Data.Total > 0 && len(res) >= resp.Data.Total { + break + } + } + return res, nil +} + +func (d *GuangYaPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + + var resp downloadResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_download_url", map[string]any{ + "fileId": file.GetID(), + }, &resp); err != nil { + return nil, err + } + + url := strings.TrimSpace(resp.Data.SignedURL) + if url == "" { + url = strings.TrimSpace(resp.Data.DownloadURL) + } + if url == "" { + return nil, errors.New("empty download url") + } + return &model.Link{URL: url}, nil +} + +func (d *GuangYaPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + name := strings.TrimSpace(dirName) + if name == "" { + return errors.New("dir name is empty") + } + + parentID := parentDir.GetID() + + var out createDirResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/create_dir", map[string]any{ + "parentId": parentID, + "dirName": name, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("make dir failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + name := strings.TrimSpace(newName) + if name == "" { + return errors.New("new name is empty") + } + + var out commonResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/rename", map[string]any{ + "fileId": fileID, + "newName": name, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("rename failed: %s", strings.TrimSpace(out.Msg)) + } + return nil +} + +func (d *GuangYaPan) Remove(ctx context.Context, obj model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(obj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + + var del deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/delete_file", map[string]any{ + "fileIds": []string{fileID}, + }, &del); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(del.Msg), "success") { + return fmt.Errorf("delete failed: %s", strings.TrimSpace(del.Msg)) + } + + taskID := strings.TrimSpace(del.Data.TaskID) + if taskID == "" { + // Some backends may apply deletion synchronously. + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + + var out deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/move_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("move failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + + fileID := strings.TrimSpace(srcObj.GetID()) + if fileID == "" { + return errors.New("file id is empty") + } + parentID := dstDir.GetID() + + var out deleteResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/copy_file", map[string]any{ + "fileIds": []string{fileID}, + "parentId": parentID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("copy failed: %s", strings.TrimSpace(out.Msg)) + } + taskID := strings.TrimSpace(out.Data.TaskID) + if taskID == "" { + return nil + } + return d.waitTaskDone(ctx, taskID) +} + +func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if file == nil { + return errors.New("file is nil") + } + if file.GetSize() < 0 { + return errors.New("invalid file size") + } + name := strings.TrimSpace(file.GetName()) + if name == "" { + return errors.New("file name is empty") + } + + parentID := dstDir.GetID() + + token, code, err := d.getUploadToken(ctx, parentID, name, file.GetSize()) + if err != nil { + return err + } + taskID := strings.TrimSpace(token.TaskID) + if code == 156 { + if taskID == "" { + return errors.New("instant upload returns empty task id") + } + return d.waitUploadTaskInfo(ctx, taskID) + } + + if token.ObjectPath == "" || token.BucketName == "" || token.EndPoint == "" || token.AccessKeyID == "" || token.SecretAccessKey == "" { + return errors.New("upload token is incomplete") + } + + ossEndpoint := normalizeOSSEndpoint(token.EndPoint, token.BucketName) + client, err := oss.New(ossEndpoint, token.AccessKeyID, token.SecretAccessKey, oss.SecurityToken(token.SessionToken)) + if err != nil { + return fmt.Errorf("create oss client failed: %w", err) + } + bucket, err := client.Bucket(token.BucketName) + if err != nil { + return fmt.Errorf("create oss bucket failed: %w", err) + } + + if file.GetSize() == 0 { + if err := bucket.PutObject(token.ObjectPath, strings.NewReader("")); err != nil { + return err + } + } else { + if err := d.multipartUploadToOSS(ctx, bucket, token.ObjectPath, file, up); err != nil { + return err + } + } + + if taskID == "" { + return nil + } + return d.waitUploadTaskInfo(ctx, taskID) +} + +func (d *GuangYaPan) getRootFolderID(ctx context.Context) (string, error) { + if d.rootFolderResolved { + return d.resolvedRootFolderID, nil + } + if err := d.ensureAccessToken(ctx); err != nil { + return "", err + } + if err := d.prepareRootFolder(ctx); err != nil { + return "", err + } + return d.resolvedRootFolderID, nil +} + +func (d *GuangYaPan) prepareRootFolder(ctx context.Context) error { + rootID, err := d.resolveConfiguredRootFolderID(ctx) + if err != nil { + return err + } + d.resolvedRootFolderID = rootID + d.rootFolderResolved = true + return nil +} + +func (d *GuangYaPan) resolveConfiguredRootFolderID(ctx context.Context) (string, error) { + root := strings.TrimSpace(d.RootPath) + if root == "" { + return "", nil + } + return d.resolveFolderPath(ctx, root) +} + +func (d *GuangYaPan) resolveFolderPath(ctx context.Context, rootPath string) (string, error) { + cleanPath := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rootPath), "\\", "/"), "/") + if cleanPath == "" { + return "", nil + } + + parentID := "" + for _, name := range strings.Split(cleanPath, "/") { + if name == "" { + continue + } + childID, err := d.findChildFolderID(ctx, parentID, name) + if err != nil { + return "", err + } + parentID = childID + } + return parentID, nil +} + +func (d *GuangYaPan) findChildFolderID(ctx context.Context, parentID, name string) (string, error) { + pageSize := d.PageSize + if pageSize <= 0 { + pageSize = 100 + } + + seen := 0 + for page := 0; ; page++ { + var resp listResp + body := map[string]any{ + "parentId": parentID, + "page": page, + "pageSize": pageSize, + "orderBy": d.OrderBy, + "sortType": d.SortType, + "fileTypes": []int{}, + } + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil { + return "", err + } + for _, item := range resp.Data.List { + seen++ + if item.ResType == 2 && item.FileName == name { + return item.FileID, nil + } + } + if len(resp.Data.List) < pageSize { + break + } + if resp.Data.Total > 0 && seen >= resp.Data.Total { + break + } + } + + if parentID == "" { + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under /", name) + } + return "", fmt.Errorf("resolve root folder path failed: folder %q not found under parent %s", name, parentID) +} + +func (d *GuangYaPan) ensureAccessToken(ctx context.Context) error { + if strings.TrimSpace(d.AccessToken) != "" { + return nil + } + if strings.TrimSpace(d.RefreshToken) == "" { + if d.canSMSLogin() { + return d.loginBySMSCode(ctx) + } + if d.PhoneNumber != "" { + return errors.New("not logged in yet: please fill verify_code and save storage to finish SMS login") + } + return errors.New("access token is empty") + } + return d.refreshToken(ctx) +} + +func (d *GuangYaPan) validateToken(ctx context.Context) error { + var me userMeResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetResult(&me). + Get("/v1/user/me") + if err != nil { + return err + } + if resp.IsError() { + return fmt.Errorf("validate token failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if strings.TrimSpace(me.Sub) == "" { + return errors.New("validate token failed: empty user sub") + } + return nil +} + +func (d *GuangYaPan) refreshToken(ctx context.Context) error { + if strings.TrimSpace(d.RefreshToken) == "" { + return errors.New("refresh_token is empty") + } + + var out tokenResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&out). + Post("/v1/auth/token") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + errMsg := strings.TrimSpace(out.ErrorDesc) + if errMsg == "" { + errMsg = strings.TrimSpace(out.Error) + } + if errMsg == "" { + errMsg = strings.TrimSpace(resp.String()) + } + if errMsg == "" { + errMsg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + return fmt.Errorf("refresh token failed: %s", errMsg) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + if strings.TrimSpace(out.RefreshToken) != "" { + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + } + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) canSMSLogin() bool { + return d.PhoneNumber != "" && d.VerifyCode != "" +} + +func (d *GuangYaPan) loginBySMSCode(ctx context.Context) error { + verificationID := strings.TrimSpace(d.VerificationID) + if verificationID == "" { + var err error + verificationID, err = d.requestVerificationID(ctx) + if err != nil { + return err + } + } + + var step2 verifyResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_id": verificationID, + "verification_code": d.VerifyCode, + "client_id": d.ClientID, + }). + SetResult(&step2). + Post("/v1/auth/verification/verify") + if err != nil { + return err + } + if resp.IsError() || step2.Error != "" || strings.TrimSpace(step2.VerificationToken) == "" { + return fmt.Errorf("verify code failed: %s", d.accountErr(step2.ErrorDesc, step2.Error, resp)) + } + + var out tokenResp + resp, err = d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "verification_code": d.VerifyCode, + "verification_token": step2.VerificationToken, + "username": normalizePhoneE164(d.PhoneNumber), + "client_id": d.ClientID, + }). + SetResult(&out). + Post("/v1/auth/signin") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" { + return fmt.Errorf("signin failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + + d.AccessToken = strings.TrimSpace(out.AccessToken) + d.RefreshToken = strings.TrimSpace(out.RefreshToken) + d.VerificationID = "" + // One-time SMS code should not be reused after successful login. + d.VerifyCode = "" + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) prepareSMSCode(ctx context.Context) error { + // Explicit send action should always refresh verification_id. + d.VerificationID = "" + if err := d.ensureCaptchaToken(ctx, false); err != nil { + return err + } + verificationID, err := d.requestVerificationID(ctx) + if err != nil { + return err + } + d.VerificationID = verificationID + d.SendCode = false + op.MustSaveDriverStorage(d) + return nil +} + +func (d *GuangYaPan) requestVerificationID(ctx context.Context) (string, error) { + if d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + } + + var step1 verificationResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "phone_number": normalizePhoneE164(d.PhoneNumber), + "target": "ANY", + "client_id": d.ClientID, + }). + SetResult(&step1). + Post("/v1/auth/verification") + if err != nil { + return "", err + } + if resp.IsError() || step1.Error != "" || strings.TrimSpace(step1.VerificationID) == "" { + // If captcha token is expired/invalid, refresh it once and retry. + if strings.Contains(step1.Error, "captcha_invalid") || strings.Contains(step1.ErrorDesc, "captcha_token expired") { + if err := d.ensureCaptchaToken(ctx, true); err == nil { + return d.requestVerificationID(ctx) + } + } + return "", fmt.Errorf("request verification failed: %s", d.accountErr(step1.ErrorDesc, step1.Error, resp)) + } + return strings.TrimSpace(step1.VerificationID), nil +} + +func (d *GuangYaPan) ensureCaptchaToken(ctx context.Context, force bool) error { + if !force && d.CaptchaToken != "" { + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + return nil + } + + var out captchaInitResp + resp, err := d.accountClient.R(). + SetContext(ctx). + SetBody(map[string]any{ + "client_id": d.ClientID, + "action": "POST:/v1/auth/verification", + "device_id": d.DeviceID, + "meta": map[string]any{ + "username": normalizePhoneE164(d.PhoneNumber), + "phone_number": normalizePhoneE164(d.PhoneNumber), + "VERIFICATION_PHONE": normalizePhoneE164(d.PhoneNumber), + }, + }). + SetResult(&out). + Post("/v1/shield/captcha/init") + if err != nil { + return err + } + if resp.IsError() || out.Error != "" || strings.TrimSpace(out.CaptchaToken) == "" { + return fmt.Errorf("init captcha token failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp)) + } + d.CaptchaToken = strings.TrimSpace(out.CaptchaToken) + d.accountClient.SetHeader("X-Captcha-Token", d.CaptchaToken) + op.MustSaveDriverStorage(d) + return nil +} + +func normalizeCaptchaUsername(phone string) string { + p := strings.TrimSpace(phone) + p = strings.ReplaceAll(p, " ", "") + p = strings.TrimPrefix(p, "+") + // Keep only digits. + b := make([]rune, 0, len(p)) + for _, ch := range p { + if ch >= '0' && ch <= '9' { + b = append(b, ch) + } + } + digits := string(b) + // Mainland number normalization: +86xxxxxxxxxxx -> xxxxxxxxxxx + if strings.HasPrefix(digits, "86") && len(digits) > 11 { + digits = digits[2:] + } + return digits +} + +func normalizePhoneE164(phone string) string { + p := strings.TrimSpace(phone) + if p == "" { + return "" + } + p = strings.ReplaceAll(p, " ", "") + if strings.HasPrefix(p, "+") { + // Format as "+86 1xxxxxxxxxx" to match browser payload expectations. + if strings.HasPrefix(p, "+86") && len(p) > 3 { + rest := strings.TrimPrefix(p, "+86") + return "+86 " + rest + } + return p + } + // If raw mainland number is provided, normalize with +86 prefix. + digits := normalizeCaptchaUsername(p) + if len(digits) == 11 { + return "+86 " + digits + } + return p +} + +func (d *GuangYaPan) setTempStatus(status string) { + // initStorage sets status to WORK after Init returns, so we update it shortly after. + time.AfterFunc(200*time.Millisecond, func() { + d.GetStorage().SetStatus(status) + op.MustSaveDriverStorage(d) + }) +} + +func (d *GuangYaPan) accountErr(desc, short string, resp *resty.Response) string { + msg := strings.TrimSpace(desc) + if msg == "" { + msg = strings.TrimSpace(short) + } + if msg == "" && resp != nil { + msg = strings.TrimSpace(resp.String()) + } + if msg == "" && resp != nil { + msg = fmt.Sprintf("status=%d", resp.StatusCode()) + } + if msg == "" { + msg = "unknown error" + } + return msg +} + +func (d *GuangYaPan) postAPI(ctx context.Context, path string, body any, out any) error { + if strings.TrimSpace(d.AccessToken) == "" { + return errors.New("access token is empty") + } + resp, err := d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + if resp.StatusCode() == 401 || resp.StatusCode() == 403 { + if strings.TrimSpace(d.RefreshToken) == "" { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + if err := d.refreshToken(ctx); err != nil { + return err + } + resp, err = d.apiClient.R(). + SetContext(ctx). + SetHeader("Authorization", "Bearer "+d.AccessToken). + SetBody(body). + SetResult(out). + Post(path) + if err != nil { + return err + } + } + if resp.IsError() { + return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String()) + } + return nil +} + +func (d *GuangYaPan) waitTaskDone(ctx context.Context, taskID string) error { + const ( + maxTry = 30 + interval = 300 * time.Millisecond + ) + for i := 0; i < maxTry; i++ { + var out taskStatusResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_task_status", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if !strings.EqualFold(strings.TrimSpace(out.Msg), "success") { + return fmt.Errorf("get task status failed: %s", strings.TrimSpace(out.Msg)) + } + switch out.Data.Status { + case 2: + return nil + case -1, 3: + return fmt.Errorf("task %s failed with status=%d", taskID, out.Data.Status) + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("task %s timeout", taskID) +} + +func (d *GuangYaPan) getUploadToken(ctx context.Context, parentID, name string, size int64) (*uploadTokenData, int, error) { + var out uploadTokenResp + err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_center_token", map[string]any{ + "capacity": 2, + "name": name, + "parentId": parentID, + "res": map[string]any{ + "fileSize": size, + }, + }, &out) + if err != nil { + return nil, 0, err + } + msg := strings.TrimSpace(out.Msg) + if msg != "" && !strings.EqualFold(msg, "success") { + return nil, out.Code, fmt.Errorf("get upload token failed: %s", msg) + } + if out.Data.TaskID == "" { + return nil, out.Code, errors.New("get upload token failed: empty task id") + } + if out.Data.AccessKeyID == "" { + out.Data.AccessKeyID = out.Data.Creds.AccessKeyID + } + if out.Data.SecretAccessKey == "" { + out.Data.SecretAccessKey = out.Data.Creds.SecretAccessKey + } + if out.Data.SessionToken == "" { + out.Data.SessionToken = out.Data.Creds.SessionToken + } + if strings.TrimSpace(out.Data.EndPoint) == "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } + if strings.TrimSpace(out.Data.EndPoint) != "" && !strings.HasPrefix(out.Data.EndPoint, "http://") && !strings.HasPrefix(out.Data.EndPoint, "https://") { + if strings.TrimSpace(out.Data.FullEndPoint) != "" { + out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint) + } else if strings.TrimSpace(out.Data.BucketName) != "" { + host := strings.TrimSpace(out.Data.EndPoint) + prefix := strings.TrimSpace(out.Data.BucketName) + "." + if strings.HasPrefix(host, prefix) { + out.Data.EndPoint = "https://" + host + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.BucketName) + "." + host + } + } else { + out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.EndPoint) + } + } + return &out.Data, out.Code, nil +} + +func (d *GuangYaPan) waitUploadTaskInfo(ctx context.Context, taskID string) error { + const ( + maxTry = 300 + interval = 1 * time.Second + ) + for i := 0; i < maxTry; i++ { + var out taskInfoResp + if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_info_by_task_id", map[string]any{ + "taskId": taskID, + }, &out); err != nil { + return err + } + if out.Data.FileID != "" { + return nil + } + switch out.Code { + case 145, 146, 147, 155, 163, 0: + // uploading/verifying/processing + default: + if strings.TrimSpace(out.Msg) != "" { + return fmt.Errorf("upload task failed: code=%d msg=%s", out.Code, strings.TrimSpace(out.Msg)) + } + } + if i == maxTry-1 { + break + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(interval): + } + } + return fmt.Errorf("upload task %s timeout", taskID) +} + +func (d *GuangYaPan) multipartUploadToOSS(ctx context.Context, bucket *oss.Bucket, objectPath string, file model.FileStreamer, up driver.UpdateProgress) error { + partSize := calcUploadPartSize(file.GetSize()) + imur, err := bucket.InitiateMultipartUpload(objectPath, oss.Sequential()) + if err != nil { + return err + } + + total := file.GetSize() + partCount := int((total + partSize - 1) / partSize) + parts := make([]oss.UploadPart, 0, partCount) + var uploaded int64 + partNumber := 1 + + for uploaded < total { + if err := ctx.Err(); err != nil { + return err + } + curPartSize := partSize + left := total - uploaded + if left < curPartSize { + curPartSize = left + } + + reader := io.LimitReader(file, curPartSize) + part, err := bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, reader), curPartSize, partNumber) + if err != nil { + return err + } + parts = append(parts, part) + uploaded += curPartSize + partNumber++ + if total > 0 { + up(100 * float64(uploaded) / float64(total)) + } + } + + _, err = bucket.CompleteMultipartUpload(imur, parts) + return err +} + +func calcUploadPartSize(size int64) int64 { + const ( + mb = int64(1024 * 1024) + gb = int64(1024 * 1024 * 1024) + ) + switch { + case size <= 100*mb: + return 1 * mb + case size <= 16*gb: + return 2 * mb + case size <= 160*gb: + return 4 * mb + default: + return 8 * mb + } +} + +func normalizeOSSEndpoint(endpoint, bucket string) string { + ep := strings.TrimSpace(endpoint) + if ep == "" { + return ep + } + if !strings.HasPrefix(ep, "http://") && !strings.HasPrefix(ep, "https://") { + ep = "https://" + ep + } + u, err := url.Parse(ep) + if err != nil || u.Host == "" { + return ep + } + host := u.Host + prefix := strings.TrimSpace(bucket) + if prefix != "" && strings.HasPrefix(host, prefix+".") { + host = strings.TrimPrefix(host, prefix+".") + } + u.Host = host + return u.String() +} + +func normalizeDeviceID(v string) string { + v = strings.ToLower(strings.TrimSpace(v)) + v = strings.ReplaceAll(v, "-", "") + if len(v) != 32 { + return "" + } + for _, ch := range v { + if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') { + return "" + } + } + return v +} + +func randomDeviceID() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "0123456789abcdef0123456789abcdef" + } + return hex.EncodeToString(b) +} + +var _ driver.Driver = (*GuangYaPan)(nil) +var _ driver.GetRooter = (*GuangYaPan)(nil) \ No newline at end of file diff --git a/drivers/guangyapan/meta.go b/drivers/guangyapan/meta.go new file mode 100644 index 00000000000..7d5f8fbcb51 --- /dev/null +++ b/drivers/guangyapan/meta.go @@ -0,0 +1,42 @@ +package guangyapan + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + RootPath string `json:"root_path" help:"光鸭云盘中的完整路径"` + PhoneNumber string `json:"phone_number" type:"text" help:"Phone number for SMS login, e.g. +86 13800000000"` + CaptchaToken string `json:"captcha_token" type:"text" help:"Captcha token required by /v1/auth/verification"` + SendCode bool `json:"send_code" type:"bool" help:"Set true and save to send SMS code, it auto-resets to false after sending"` + VerifyCode string `json:"verify_code" type:"text" help:"SMS verification code used with phone_number; fill then save to finish login"` + VerificationID string `json:"verification_id" type:"text" help:"Auto-generated after sending SMS code; do not edit manually"` + AccessToken string `json:"access_token" type:"text" help:"Bearer access token (optional if refresh_token is provided)"` + RefreshToken string `json:"refresh_token" type:"text" help:"Refresh token for auto-login/auto-refresh"` + ClientID string `json:"client_id" default:"aMe-8VSlkrbQXpUR"` + DeviceID string `json:"device_id" help:"Optional custom device id (32 hex chars), auto-generated when empty"` + PageSize int `json:"page_size" type:"number" default:"100"` + OrderBy int `json:"order_by" type:"number" options:"0,1,2,3,4" default:"3" help:"Sort field used by the file list"` + SortType int `json:"sort_type" type:"number" options:"0,1" default:"1" help:"Sort direction used by the file list"` +} + +var config = driver.Config{ + Name: "GuangYaPan", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "", + CheckStatus: true, + Alert: "info|Two-stage SMS login: (1) fill phone_number (+ captcha_token if needed), set send_code=true and save; (2) fill verify_code and save to finish login and auto-save access_token/refresh_token.", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &GuangYaPan{} + }) +} \ No newline at end of file diff --git a/drivers/guangyapan/offline.go b/drivers/guangyapan/offline.go new file mode 100644 index 00000000000..a53b8adc33f --- /dev/null +++ b/drivers/guangyapan/offline.go @@ -0,0 +1,169 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" +) + +func (d *GuangYaPan) ResolveOfflineResource(ctx context.Context, fileURL string) (*OfflineResolveData, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + fileURL = strings.TrimSpace(fileURL) + if fileURL == "" { + return nil, errors.New("offline url is empty") + } + + var resp offlineResolveResp + if err := d.postAPI(ctx, "/cloudcollection/v1/resolve_res", map[string]any{ + "url": fileURL, + }, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("resolve offline resource failed: %s", strings.TrimSpace(resp.Msg)) + } + return &resp.Data, nil +} + +func (d *GuangYaPan) OfflineDownload(ctx context.Context, fileURL string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + resolved, err := d.ResolveOfflineResource(ctx, fileURL) + if err != nil { + return nil, err + } + + parentID := parentDir.GetID() + rootID, err := d.getRootFolderID(ctx) + if err != nil { + return nil, err + } + if parentID == rootID { + parentID = "" + } + + taskURL := strings.TrimSpace(resolved.URL) + if taskURL == "" { + taskURL = strings.TrimSpace(fileURL) + } + name := strings.TrimSpace(fileName) + if name == "" { + name = resolved.defaultName(taskURL) + } + + body := map[string]any{ + "url": taskURL, + "parentId": parentID, + "newName": name, + } + if indexes := resolved.fileIndexes(); len(indexes) > 0 { + body["fileIndexes"] = indexes + } + + var resp offlineCreateResp + if err := d.postAPI(ctx, "/cloudcollection/v1/create_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("create offline task failed: %s", strings.TrimSpace(resp.Msg)) + } + taskID := strings.TrimSpace(resp.Data.TaskID) + if taskID == "" { + return nil, errors.New("create offline task failed: empty task id") + } + return &OfflineTask{ + TaskID: taskID, + FileName: name, + Res: taskURL, + }, nil +} + +func (d *GuangYaPan) OfflineList(ctx context.Context, taskIDs []string, statuses []int, cursor string, pageSize int) ([]OfflineTask, error) { + if err := d.ensureAccessToken(ctx); err != nil { + return nil, err + } + body := map[string]any{} + if len(taskIDs) > 0 { + body["taskIds"] = taskIDs + } + if len(statuses) > 0 { + body["status"] = statuses + } + if cursor = strings.TrimSpace(cursor); cursor != "" { + body["cursor"] = cursor + } + if pageSize > 0 { + body["pageSize"] = pageSize + } + + var resp offlineListResp + if err := d.postAPI(ctx, "/cloudcollection/v1/list_task", body, &resp); err != nil { + return nil, err + } + if !isSuccessMsg(resp.Msg) { + return nil, fmt.Errorf("list offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return resp.Data.List, nil +} + +func (d *GuangYaPan) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + if err := d.ensureAccessToken(ctx); err != nil { + return err + } + if len(taskIDs) == 0 { + return nil + } + + var resp offlineDeleteResp + if err := d.postAPI(ctx, "/cloudcollection/v2/delete_task", map[string]any{ + "taskIds": taskIDs, + }, &resp); err != nil { + return err + } + if !isSuccessMsg(resp.Msg) { + return fmt.Errorf("delete offline tasks failed: %s", strings.TrimSpace(resp.Msg)) + } + return nil +} + +func (d OfflineResolveData) defaultName(fileURL string) string { + if d.BTResInfo != nil && strings.TrimSpace(d.BTResInfo.FileName) != "" { + return strings.TrimSpace(d.BTResInfo.FileName) + } + u, err := url.Parse(fileURL) + if err == nil { + name := strings.TrimSpace(stdpath.Base(u.Path)) + if name != "" && name != "." && name != "/" { + if decoded, err := url.PathUnescape(name); err == nil { + name = decoded + } + return name + } + } + return "offline_download" +} + +func (d OfflineResolveData) fileIndexes() []int { + if d.BTResInfo == nil || len(d.BTResInfo.Subfiles) == 0 { + return nil + } + indexes := make([]int, 0, len(d.BTResInfo.Subfiles)) + for i, file := range d.BTResInfo.Subfiles { + if file.FileIndex != nil { + indexes = append(indexes, *file.FileIndex) + continue + } + indexes = append(indexes, i) + } + return indexes +} + +func isSuccessMsg(msg string) bool { + msg = strings.TrimSpace(msg) + return msg == "" || strings.EqualFold(msg, "success") +} diff --git a/drivers/guangyapan/types.go b/drivers/guangyapan/types.go new file mode 100644 index 00000000000..1a1c129c06a --- /dev/null +++ b/drivers/guangyapan/types.go @@ -0,0 +1,214 @@ +package guangyapan + +import "time" + +type tokenResp struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + TokenType string `json:"token_type"` + ExpiresIn int64 `json:"expires_in"` + Sub string `json:"sub"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verificationResp struct { + VerificationID string `json:"verification_id"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type captchaInitResp struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type verifyResp struct { + VerificationToken string `json:"verification_token"` + Error string `json:"error"` + ErrorCode int `json:"error_code"` + ErrorDesc string `json:"error_description"` +} + +type userMeResp struct { + Sub string `json:"sub"` +} + +type listResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Total int `json:"total"` + List []fileItem `json:"list"` + } `json:"data"` +} + +type fileItem struct { + FileID string `json:"fileId"` + ParentID string `json:"parentId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` +} + +type downloadResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + SignedURL string `json:"signedURL"` + DownloadURL string `json:"downloadUrl"` + } `json:"data"` +} + +type createDirResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + FileName string `json:"fileName"` + ResType int `json:"resType"` + CTime int64 `json:"ctime"` + UTime int64 `json:"utime"` + } `json:"data"` +} + +type commonResp struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +type deleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + } `json:"data"` +} + +type taskStatusResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + Status int `json:"status"` + } `json:"data"` +} + +type uploadTokenResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data uploadTokenData `json:"data"` +} + +type uploadTokenData struct { + TaskID string `json:"taskId"` + ObjectPath string `json:"objectPath"` + Provider any `json:"provider"` + Region string `json:"region"` + BucketName string `json:"bucketName"` + EndPoint string `json:"endPoint"` + FullEndPoint string `json:"fullEndPoint"` + CallbackVar string `json:"callbackVar"` + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + Creds struct { + AccessKeyID string `json:"accessKeyID"` + SecretAccessKey string `json:"secretAccessKey"` + SessionToken string `json:"sessionToken"` + } `json:"creds"` +} + +type taskInfoResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + FileID string `json:"fileId"` + } `json:"data"` +} + +type offlineResolveResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data OfflineResolveData `json:"data"` +} + +type OfflineResolveData struct { + ResType int `json:"resType"` + BTResInfo *OfflineBTResInfo `json:"btResInfo"` + URL string `json:"url"` +} + +type OfflineBTResInfo struct { + InfoHash string `json:"infoHash"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + SubfilesNum int `json:"subfilesNum"` + Subfiles []OfflineSubfile `json:"subfiles"` + CreateTime int64 `json:"createTime"` + ExcludeIndices []int `json:"excludeIndices"` +} + +type OfflineSubfile struct { + FileName string `json:"fileName"` + FileIndex *int `json:"fileIndex"` + FileSize int64 `json:"fileSize"` +} + +type offlineCreateResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskID string `json:"taskId"` + URL string `json:"url"` + } `json:"data"` +} + +type offlineDeleteResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + TaskIDs []string `json:"taskIds"` + } `json:"data"` +} + +type offlineListResp struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data struct { + StatusCounts []struct { + Status int `json:"status"` + Count int `json:"count"` + } `json:"statusCounts"` + Cursor string `json:"cursor"` + List []OfflineTask `json:"list"` + Total int `json:"total"` + } `json:"data"` +} + +type OfflineTask struct { + TaskID string `json:"taskId"` + FileName string `json:"fileName"` + TotalSize int64 `json:"totalSize"` + Status int `json:"status"` + CreateTime int64 `json:"createTime"` + Res string `json:"res"` + ResType int `json:"resType"` + Progress int `json:"progress"` + FileID string `json:"fileId"` + IsDir bool `json:"isDir"` + Exist bool `json:"exist"` +} + +func unixOrZero(v int64) time.Time { + if v <= 0 { + return time.Time{} + } + return time.Unix(v, 0) +} diff --git a/drivers/halalcloud/driver.go b/drivers/halalcloud/driver.go new file mode 100644 index 00000000000..26832760117 --- /dev/null +++ b/drivers/halalcloud/driver.go @@ -0,0 +1,405 @@ +package halalcloud + +import ( + "context" + "crypto/sha1" + "fmt" + "io" + "net/url" + "path" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/city404/v6-public-rpc-proto/go/v6/common" + pbPublicUser "github.com/city404/v6-public-rpc-proto/go/v6/user" + pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" + "github.com/rclone/rclone/lib/readers" + "github.com/zzzhr1990/go-common-entity/userfile" +) + +type HalalCloud struct { + *HalalCommon + model.Storage + Addition + + uploadThread int +} + +func (d *HalalCloud) Config() driver.Config { + return config +} + +func (d *HalalCloud) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *HalalCloud) Init(ctx context.Context) error { + d.uploadThread, _ = strconv.Atoi(d.UploadThread) + if d.uploadThread < 1 || d.uploadThread > 32 { + d.uploadThread, d.UploadThread = 3, "3" + } + + if d.HalalCommon == nil { + d.HalalCommon = &HalalCommon{ + Common: &Common{}, + AuthService: &AuthService{ + appID: func() string { + if d.Addition.AppID != "" { + return d.Addition.AppID + } + return AppID + }(), + appVersion: func() string { + if d.Addition.AppVersion != "" { + return d.Addition.AppVersion + } + return AppVersion + }(), + appSecret: func() string { + if d.Addition.AppSecret != "" { + return d.Addition.AppSecret + } + return AppSecret + }(), + tr: &TokenResp{ + RefreshToken: d.Addition.RefreshToken, + }, + }, + UserInfo: &UserInfo{}, + refreshTokenFunc: func(token string) error { + d.Addition.RefreshToken = token + op.MustSaveDriverStorage(d) + return nil + }, + } + } + + // 防止重复登录 + if d.Addition.RefreshToken == "" || !d.IsLogin() { + as, err := d.NewAuthServiceWithOauth() + if err != nil { + d.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + return err + } + d.HalalCommon.AuthService = as + d.SetTokenResp(as.tr) + op.MustSaveDriverStorage(d) + } + var err error + d.HalalCommon.serv, err = d.NewAuthService(d.Addition.RefreshToken) + if err != nil { + return err + } + + return nil +} + +func (d *HalalCloud) Drop(ctx context.Context) error { + return nil +} + +func (d *HalalCloud) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.getFiles(ctx, dir) +} + +func (d *HalalCloud) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return d.getLink(ctx, file, args) +} + +func (d *HalalCloud) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return d.makeDir(ctx, parentDir, dirName) +} + +func (d *HalalCloud) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.move(ctx, srcObj, dstDir) +} + +func (d *HalalCloud) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return d.rename(ctx, srcObj, newName) +} + +func (d *HalalCloud) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.copy(ctx, srcObj, dstDir) +} + +func (d *HalalCloud) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(ctx, obj) +} + +func (d *HalalCloud) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return d.put(ctx, dstDir, stream, up) +} + +func (d *HalalCloud) IsLogin() bool { + if d.AuthService.tr == nil { + return false + } + serv, err := d.NewAuthService(d.Addition.RefreshToken) + if err != nil { + return false + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + result, err := pbPublicUser.NewPubUserClient(serv.GetGrpcConnection()).Get(ctx, &pbPublicUser.User{ + Identity: "", + }) + if result == nil || err != nil { + return false + } + d.UserInfo.Identity = result.Identity + d.UserInfo.CreateTs = result.CreateTs + d.UserInfo.Name = result.Name + d.UserInfo.UpdateTs = result.UpdateTs + return true +} + +type HalalCommon struct { + *Common + *AuthService // 登录信息 + *UserInfo // 用户信息 + refreshTokenFunc func(token string) error + serv *AuthService +} + +func (d *HalalCloud) SetTokenResp(tr *TokenResp) { + d.Addition.RefreshToken = tr.RefreshToken +} + +func (d *HalalCloud) getFiles(ctx context.Context, dir model.Obj) ([]model.Obj, error) { + + files := make([]model.Obj, 0) + limit := int64(100) + token := "" + client := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()) + + opDir := d.GetCurrentDir(dir) + + for { + result, err := client.List(ctx, &pubUserFile.FileListRequest{ + Parent: &pubUserFile.File{Path: opDir}, + ListInfo: &common.ScanListRequest{ + Limit: limit, + Token: token, + }, + }) + if err != nil { + return nil, err + } + + for i := 0; len(result.Files) > i; i++ { + files = append(files, (*Files)(result.Files[i])) + } + + if result.ListInfo == nil || result.ListInfo.Token == "" { + break + } + token = result.ListInfo.Token + + } + return files, nil +} + +func (d *HalalCloud) getLink(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + + client := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()) + ctx1, cancelFunc := context.WithCancel(context.Background()) + defer cancelFunc() + + result, err := client.ParseFileSlice(ctx1, (*pubUserFile.File)(file.(*Files))) + if err != nil { + return nil, err + } + fileAddrs := []*pubUserFile.SliceDownloadInfo{} + var addressDuration int64 + + nodesNumber := len(result.RawNodes) + nodesIndex := nodesNumber - 1 + startIndex, endIndex := 0, nodesIndex + for nodesIndex >= 0 { + if nodesIndex >= 200 { + endIndex = 200 + } else { + endIndex = nodesNumber + } + for ; endIndex <= nodesNumber; endIndex += 200 { + if endIndex == 0 { + endIndex = 1 + } + sliceAddress, err := client.GetSliceDownloadAddress(ctx, &pubUserFile.SliceDownloadAddressRequest{ + Identity: result.RawNodes[startIndex:endIndex], + Version: 1, + }) + if err != nil { + return nil, err + } + addressDuration = sliceAddress.ExpireAt + fileAddrs = append(fileAddrs, sliceAddress.Addresses...) + startIndex = endIndex + nodesIndex -= 200 + } + + } + + size := result.FileSize + chunks := getChunkSizes(result.Sizes) + resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + length := httpRange.Length + if httpRange.Length >= 0 && httpRange.Start+httpRange.Length >= size { + length = -1 + } + if err != nil { + return nil, fmt.Errorf("open download file failed: %w", err) + } + oo := &openObject{ + ctx: ctx, + d: fileAddrs, + chunk: &[]byte{}, + chunks: &chunks, + skip: httpRange.Start, + sha: result.Sha1, + shaTemp: sha1.New(), + } + + return readers.NewLimitedReadCloser(oo, length), nil + } + + var duration time.Duration + if addressDuration != 0 { + duration = time.Until(time.UnixMilli(addressDuration)) + } else { + duration = time.Until(time.Now().Add(time.Hour)) + } + + resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader} + return &model.Link{ + RangeReadCloser: resultRangeReadCloser, + Expiration: &duration, + }, nil +} + +func (d *HalalCloud) makeDir(ctx context.Context, dir model.Obj, name string) (model.Obj, error) { + newDir := userfile.NewFormattedPath(d.GetCurrentOpDir(dir, []string{name}, 0)).GetPath() + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Create(ctx, &pubUserFile.File{ + Path: newDir, + }) + return nil, err +} + +func (d *HalalCloud) move(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { + oldDir := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() + newDir := userfile.NewFormattedPath(d.GetCurrentDir(dir)).GetPath() + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Move(ctx, &pubUserFile.BatchOperationRequest{ + Source: []*pubUserFile.File{ + { + Identity: obj.GetID(), + Path: oldDir, + }, + }, + Dest: &pubUserFile.File{ + Identity: dir.GetID(), + Path: newDir, + }, + }) + return nil, err +} + +func (d *HalalCloud) rename(ctx context.Context, obj model.Obj, name string) (model.Obj, error) { + id := obj.GetID() + newPath := userfile.NewFormattedPath(d.GetCurrentOpDir(obj, []string{name}, 0)).GetPath() + + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Rename(ctx, &pubUserFile.File{ + Path: newPath, + Identity: id, + Name: name, + }) + return nil, err +} + +func (d *HalalCloud) copy(ctx context.Context, obj model.Obj, dir model.Obj) (model.Obj, error) { + id := obj.GetID() + sourcePath := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() + if len(id) > 0 { + sourcePath = "" + } + dest := &pubUserFile.File{ + Identity: dir.GetID(), + Path: userfile.NewFormattedPath(d.GetCurrentDir(dir)).GetPath(), + } + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Copy(ctx, &pubUserFile.BatchOperationRequest{ + Source: []*pubUserFile.File{ + { + Path: sourcePath, + Identity: id, + }, + }, + Dest: dest, + }) + return nil, err +} + +func (d *HalalCloud) remove(ctx context.Context, obj model.Obj) error { + id := obj.GetID() + newPath := userfile.NewFormattedPath(d.GetCurrentDir(obj)).GetPath() + //if len(id) > 0 { + // newPath = "" + //} + _, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).Delete(ctx, &pubUserFile.BatchOperationRequest{ + Source: []*pubUserFile.File{ + { + Path: newPath, + Identity: id, + }, + }, + }) + return err +} + +func (d *HalalCloud) put(ctx context.Context, dstDir model.Obj, fileStream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + + newDir := path.Join(dstDir.GetPath(), fileStream.GetName()) + + result, err := pubUserFile.NewPubUserFileClient(d.HalalCommon.serv.GetGrpcConnection()).CreateUploadToken(ctx, &pubUserFile.File{ + Path: newDir, + }) + if err != nil { + return nil, err + } + u, _ := url.Parse(result.Endpoint) + u.Host = "s3." + u.Host + result.Endpoint = u.String() + s, err := session.NewSession(&aws.Config{ + HTTPClient: base.HttpClient, + Credentials: credentials.NewStaticCredentials(result.AccessKey, result.SecretKey, result.Token), + Region: aws.String(result.Region), + Endpoint: aws.String(result.Endpoint), + S3ForcePathStyle: aws.Bool(true), + }) + if err != nil { + return nil, err + } + uploader := s3manager.NewUploader(s, func(u *s3manager.Uploader) { + u.Concurrency = d.uploadThread + }) + if fileStream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = fileStream.GetSize() / (s3manager.MaxUploadParts - 1) + } + reader := driver.NewLimitedUploadStream(ctx, fileStream) + _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(result.Bucket), + Key: aws.String(result.Key), + Body: io.TeeReader(reader, driver.NewProgress(fileStream.GetSize(), up)), + }) + return nil, err + +} + +var _ driver.Driver = (*HalalCloud)(nil) diff --git a/drivers/halalcloud/meta.go b/drivers/halalcloud/meta.go new file mode 100644 index 00000000000..d4040323eb0 --- /dev/null +++ b/drivers/halalcloud/meta.go @@ -0,0 +1,38 @@ +package halalcloud + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // define other + RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` + UploadThread string `json:"upload_thread" default:"3" help:"1 <= thread <= 32"` + + AppID string `json:"app_id" required:"true" default:"alist/10001"` + AppVersion string `json:"app_version" required:"true" default:"1.0.0"` + AppSecret string `json:"app_secret" required:"true" default:"bR4SJwOkvnG5WvVJ"` +} + +var config = driver.Config{ + Name: "HalalCloud", + LocalSort: false, + OnlyLocal: true, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &HalalCloud{} + }) +} diff --git a/drivers/halalcloud/options.go b/drivers/halalcloud/options.go new file mode 100644 index 00000000000..56e5fdc5c09 --- /dev/null +++ b/drivers/halalcloud/options.go @@ -0,0 +1,52 @@ +package halalcloud + +import "google.golang.org/grpc" + +func defaultOptions() halalOptions { + return halalOptions{ + // onRefreshTokenRefreshed: func(string) {}, + grpcOptions: []grpc.DialOption{ + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(1024 * 1024 * 32)), + // grpc.WithMaxMsgSize(1024 * 1024 * 1024), + }, + } +} + +type HalalOption interface { + apply(*halalOptions) +} + +// halalOptions configure a RPC call. halalOptions are set by the HalalOption +// values passed to Dial. +type halalOptions struct { + onTokenRefreshed func(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64) + grpcOptions []grpc.DialOption +} + +// funcDialOption wraps a function that modifies halalOptions into an +// implementation of the DialOption interface. +type funcDialOption struct { + f func(*halalOptions) +} + +func (fdo *funcDialOption) apply(do *halalOptions) { + fdo.f(do) +} + +func newFuncDialOption(f func(*halalOptions)) *funcDialOption { + return &funcDialOption{ + f: f, + } +} + +func WithRefreshTokenRefreshedCallback(s func(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64)) HalalOption { + return newFuncDialOption(func(o *halalOptions) { + o.onTokenRefreshed = s + }) +} + +func WithGrpcDialOptions(opts ...grpc.DialOption) HalalOption { + return newFuncDialOption(func(o *halalOptions) { + o.grpcOptions = opts + }) +} diff --git a/drivers/halalcloud/types.go b/drivers/halalcloud/types.go new file mode 100644 index 00000000000..9772421264b --- /dev/null +++ b/drivers/halalcloud/types.go @@ -0,0 +1,101 @@ +package halalcloud + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/city404/v6-public-rpc-proto/go/v6/common" + pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" + "google.golang.org/grpc" + "time" +) + +type AuthService struct { + appID string + appVersion string + appSecret string + grpcConnection *grpc.ClientConn + dopts halalOptions + tr *TokenResp +} + +type TokenResp struct { + AccessToken string `json:"accessToken,omitempty"` + AccessTokenExpiredAt int64 `json:"accessTokenExpiredAt,omitempty"` + RefreshToken string `json:"refreshToken,omitempty"` + RefreshTokenExpiredAt int64 `json:"refreshTokenExpiredAt,omitempty"` +} + +type UserInfo struct { + Identity string `json:"identity,omitempty"` + UpdateTs int64 `json:"updateTs,omitempty"` + Name string `json:"name,omitempty"` + CreateTs int64 `json:"createTs,omitempty"` +} + +type OrderByInfo struct { + Field string `json:"field,omitempty"` + Asc bool `json:"asc,omitempty"` +} + +type ListInfo struct { + Token string `json:"token,omitempty"` + Limit int64 `json:"limit,omitempty"` + OrderBy []*OrderByInfo `json:"order_by,omitempty"` + Version int32 `json:"version,omitempty"` +} + +type FilesList struct { + Files []*Files `json:"files,omitempty"` + ListInfo *common.ScanListRequest `json:"list_info,omitempty"` +} + +var _ model.Obj = (*Files)(nil) + +type Files pubUserFile.File + +func (f *Files) GetSize() int64 { + return f.Size +} + +func (f *Files) GetName() string { + return f.Name +} + +func (f *Files) ModTime() time.Time { + return time.UnixMilli(f.UpdateTs) +} + +func (f *Files) CreateTime() time.Time { + return time.UnixMilli(f.UpdateTs) +} + +func (f *Files) IsDir() bool { + return f.Dir +} + +func (f *Files) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f *Files) GetID() string { + if len(f.Identity) == 0 { + f.Identity = "/" + } + return f.Identity +} + +func (f *Files) GetPath() string { + return f.Path +} + +type SteamFile struct { + file model.File +} + +func (s *SteamFile) Read(p []byte) (n int, err error) { + return s.file.Read(p) +} + +func (s *SteamFile) Close() error { + return s.file.Close() +} diff --git a/drivers/halalcloud/util.go b/drivers/halalcloud/util.go new file mode 100644 index 00000000000..f3012a8c83c --- /dev/null +++ b/drivers/halalcloud/util.go @@ -0,0 +1,385 @@ +package halalcloud + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/tls" + "encoding/hex" + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + pbPublicUser "github.com/city404/v6-public-rpc-proto/go/v6/user" + pubUserFile "github.com/city404/v6-public-rpc-proto/go/v6/userfile" + "github.com/google/uuid" + "github.com/ipfs/go-cid" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "hash" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" +) + +const ( + AppID = "alist/10001" + AppVersion = "1.0.0" + AppSecret = "bR4SJwOkvnG5WvVJ" +) + +const ( + grpcServer = "grpcuserapi.2dland.cn:443" + grpcServerAuth = "grpcuserapi.2dland.cn" +) + +func (d *HalalCloud) NewAuthServiceWithOauth(options ...HalalOption) (*AuthService, error) { + + aService := &AuthService{} + err2 := errors.New("") + + svc := d.HalalCommon.AuthService + for _, opt := range options { + opt.apply(&svc.dopts) + } + + grpcOptions := svc.dopts.grpcOptions + grpcOptions = append(grpcOptions, grpc.WithAuthority(grpcServerAuth), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})), grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctxx := svc.signContext(method, ctx) + err := invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method + return err + })) + + grpcConnection, err := grpc.NewClient(grpcServer, grpcOptions...) + if err != nil { + return nil, err + } + defer grpcConnection.Close() + userClient := pbPublicUser.NewPubUserClient(grpcConnection) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + stateString := uuid.New().String() + // queryValues.Add("callback", oauthToken.Callback) + oauthToken, err := userClient.CreateAuthToken(ctx, &pbPublicUser.LoginRequest{ + ReturnType: 2, + State: stateString, + ReturnUrl: "", + }) + if err != nil { + return nil, err + } + if len(oauthToken.State) < 1 { + oauthToken.State = stateString + } + + if oauthToken.Url != "" { + + return nil, fmt.Errorf(`need verify: Click Here`, oauthToken.Url) + } + + return aService, err2 + +} + +func (d *HalalCloud) NewAuthService(refreshToken string, options ...HalalOption) (*AuthService, error) { + svc := d.HalalCommon.AuthService + + if len(refreshToken) < 1 { + refreshToken = d.Addition.RefreshToken + } + + if len(d.tr.AccessToken) > 0 { + accessTokenExpiredAt := d.tr.AccessTokenExpiredAt + current := time.Now().UnixMilli() + if accessTokenExpiredAt < current { + // access token expired + d.tr.AccessToken = "" + d.tr.AccessTokenExpiredAt = 0 + } else { + svc.tr.AccessTokenExpiredAt = accessTokenExpiredAt + svc.tr.AccessToken = d.tr.AccessToken + } + } + + for _, opt := range options { + opt.apply(&svc.dopts) + } + + grpcOptions := svc.dopts.grpcOptions + grpcOptions = append(grpcOptions, grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(10*1024*1024), grpc.MaxCallRecvMsgSize(10*1024*1024)), grpc.WithAuthority(grpcServerAuth), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{})), grpc.WithUnaryInterceptor(func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + ctxx := svc.signContext(method, ctx) + err := invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method + if err != nil { + grpcStatus, ok := status.FromError(err) + + if ok && grpcStatus.Code() == codes.Unauthenticated && strings.Contains(grpcStatus.Err().Error(), "invalid accesstoken") && len(refreshToken) > 0 { + // refresh token + refreshResponse, err := pbPublicUser.NewPubUserClient(cc).Refresh(ctx, &pbPublicUser.Token{ + RefreshToken: refreshToken, + }) + if err != nil { + return err + } + if len(refreshResponse.AccessToken) > 0 { + svc.tr.AccessToken = refreshResponse.AccessToken + svc.tr.AccessTokenExpiredAt = refreshResponse.AccessTokenExpireTs + svc.OnAccessTokenRefreshed(refreshResponse.AccessToken, refreshResponse.AccessTokenExpireTs, refreshResponse.RefreshToken, refreshResponse.RefreshTokenExpireTs) + } + // retry + ctxx := svc.signContext(method, ctx) + err = invoker(ctxx, method, req, reply, cc, opts...) // invoking RPC method + if err != nil { + return err + } else { + return nil + } + } + } + return err + })) + grpcConnection, err := grpc.NewClient(grpcServer, grpcOptions...) + + if err != nil { + return nil, err + } + + svc.grpcConnection = grpcConnection + return svc, err +} + +func (s *AuthService) OnAccessTokenRefreshed(accessToken string, accessTokenExpiredAt int64, refreshToken string, refreshTokenExpiredAt int64) { + s.tr.AccessToken = accessToken + s.tr.AccessTokenExpiredAt = accessTokenExpiredAt + s.tr.RefreshToken = refreshToken + s.tr.RefreshTokenExpiredAt = refreshTokenExpiredAt + + if s.dopts.onTokenRefreshed != nil { + s.dopts.onTokenRefreshed(accessToken, accessTokenExpiredAt, refreshToken, refreshTokenExpiredAt) + } + +} + +func (s *AuthService) GetGrpcConnection() *grpc.ClientConn { + return s.grpcConnection +} + +func (s *AuthService) Close() { + _ = s.grpcConnection.Close() +} + +func (s *AuthService) signContext(method string, ctx context.Context) context.Context { + var kvString []string + currentTimeStamp := strconv.FormatInt(time.Now().UnixMilli(), 10) + bufferedString := bytes.NewBufferString(method) + kvString = append(kvString, "timestamp", currentTimeStamp) + bufferedString.WriteString(currentTimeStamp) + kvString = append(kvString, "appid", s.appID) + bufferedString.WriteString(s.appID) + kvString = append(kvString, "appversion", s.appVersion) + bufferedString.WriteString(s.appVersion) + if s.tr != nil && len(s.tr.AccessToken) > 0 { + authorization := "Bearer " + s.tr.AccessToken + kvString = append(kvString, "authorization", authorization) + bufferedString.WriteString(authorization) + } + bufferedString.WriteString(s.appSecret) + sign := GetMD5Hash(bufferedString.String()) + kvString = append(kvString, "sign", sign) + return metadata.AppendToOutgoingContext(ctx, kvString...) +} + +func (d *HalalCloud) GetCurrentOpDir(dir model.Obj, args []string, index int) string { + currentDir := dir.GetPath() + if len(currentDir) == 0 { + currentDir = "/" + } + opPath := currentDir + "/" + args[index] + if strings.HasPrefix(args[index], "/") { + opPath = args[index] + } + return opPath +} + +func (d *HalalCloud) GetCurrentDir(dir model.Obj) string { + currentDir := dir.GetPath() + if len(currentDir) == 0 { + currentDir = "/" + } + return currentDir +} + +type Common struct { +} + +func getRawFiles(addr *pubUserFile.SliceDownloadInfo) ([]byte, error) { + + if addr == nil { + return nil, errors.New("addr is nil") + } + + client := http.Client{ + Timeout: time.Duration(60 * time.Second), // Set timeout to 5 seconds + } + resp, err := client.Get(addr.DownloadAddress) + if err != nil { + + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("bad status: %s, body: %s", resp.Status, body) + } + + if addr.Encrypt > 0 { + cd := uint8(addr.Encrypt) + for idx := 0; idx < len(body); idx++ { + body[idx] = body[idx] ^ cd + } + } + + if addr.StoreType != 10 { + + sourceCid, err := cid.Decode(addr.Identity) + if err != nil { + return nil, err + } + checkCid, err := sourceCid.Prefix().Sum(body) + if err != nil { + return nil, err + } + if !checkCid.Equals(sourceCid) { + return nil, fmt.Errorf("bad cid: %s, body: %s", checkCid.String(), body) + } + } + + return body, nil + +} + +type openObject struct { + ctx context.Context + mu sync.Mutex + d []*pubUserFile.SliceDownloadInfo + id int + skip int64 + chunk *[]byte + chunks *[]chunkSize + closed bool + sha string + shaTemp hash.Hash +} + +// get the next chunk +func (oo *openObject) getChunk(ctx context.Context) (err error) { + if oo.id >= len(*oo.chunks) { + return io.EOF + } + var chunk []byte + err = utils.Retry(3, time.Second, func() (err error) { + chunk, err = getRawFiles(oo.d[oo.id]) + return err + }) + if err != nil { + return err + } + oo.id++ + oo.chunk = &chunk + return nil +} + +// Read reads up to len(p) bytes into p. +func (oo *openObject) Read(p []byte) (n int, err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return 0, fmt.Errorf("read on closed file") + } + // Skip data at the start if requested + for oo.skip > 0 { + //size := 1024 * 1024 + _, size, err := oo.ChunkLocation(oo.id) + if err != nil { + return 0, err + } + if oo.skip < int64(size) { + break + } + oo.id++ + oo.skip -= int64(size) + } + if len(*oo.chunk) == 0 { + err = oo.getChunk(oo.ctx) + if err != nil { + return 0, err + } + if oo.skip > 0 { + *oo.chunk = (*oo.chunk)[oo.skip:] + oo.skip = 0 + } + } + n = copy(p, *oo.chunk) + *oo.chunk = (*oo.chunk)[n:] + + oo.shaTemp.Write(*oo.chunk) + + return n, nil +} + +// Close closed the file - MAC errors are reported here +func (oo *openObject) Close() (err error) { + oo.mu.Lock() + defer oo.mu.Unlock() + if oo.closed { + return nil + } + // 校验Sha1 + if string(oo.shaTemp.Sum(nil)) != oo.sha { + return fmt.Errorf("failed to finish download: %w", err) + } + + oo.closed = true + return nil +} + +func GetMD5Hash(text string) string { + tHash := md5.Sum([]byte(text)) + return hex.EncodeToString(tHash[:]) +} + +// chunkSize describes a size and position of chunk +type chunkSize struct { + position int64 + size int +} + +func getChunkSizes(sliceSize []*pubUserFile.SliceSize) (chunks []chunkSize) { + chunks = make([]chunkSize, 0) + for _, s := range sliceSize { + // 对最后一个做特殊处理 + if s.EndIndex == 0 { + s.EndIndex = s.StartIndex + } + for j := s.StartIndex; j <= s.EndIndex; j++ { + chunks = append(chunks, chunkSize{position: j, size: int(s.Size)}) + } + } + return chunks +} + +func (oo *openObject) ChunkLocation(id int) (position int64, size int, err error) { + if id < 0 || id >= len(*oo.chunks) { + return 0, 0, errors.New("invalid arguments") + } + + return (*oo.chunks)[id].position, (*oo.chunks)[id].size, nil +} diff --git a/drivers/ilanzou/driver.go b/drivers/ilanzou/driver.go new file mode 100644 index 00000000000..044193d3584 --- /dev/null +++ b/drivers/ilanzou/driver.go @@ -0,0 +1,392 @@ +package template + +import ( + "context" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/foxxorcat/mopan-sdk-go" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +type ILanZou struct { + model.Storage + Addition + + userID string + account string + upClient *resty.Client + conf Conf + config driver.Config +} + +func (d *ILanZou) Config() driver.Config { + return d.config +} + +func (d *ILanZou) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ILanZou) Init(ctx context.Context) error { + d.upClient = base.NewRestyClient().SetTimeout(time.Minute * 10) + if d.UUID == "" { + res, err := d.unproved("/getUuid", http.MethodGet, nil) + if err != nil { + return err + } + d.UUID = utils.Json.Get(res, "uuid").ToString() + } + res, err := d.proved("/user/account/map", http.MethodGet, nil) + if err != nil { + return err + } + d.userID = utils.Json.Get(res, "map", "userId").ToString() + d.account = utils.Json.Get(res, "map", "account").ToString() + log.Debugf("[ilanzou] init response: %s", res) + return nil +} + +func (d *ILanZou) Drop(ctx context.Context) error { + return nil +} + +func (d *ILanZou) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + offset := 1 + var res []ListItem + for { + var resp ListResp + _, err := d.proved("/record/file/list", http.MethodGet, func(req *resty.Request) { + params := []string{ + "offset=" + strconv.Itoa(offset), + "limit=60", + "folderId=" + dir.GetID(), + "type=0", + } + queryString := strings.Join(params, "&") + req.SetQueryString(queryString).SetResult(&resp) + }) + if err != nil { + return nil, err + } + res = append(res, resp.List...) + if resp.Offset < resp.TotalPage { + offset++ + } else { + break + } + } + return utils.SliceConvert(res, func(f ListItem) (model.Obj, error) { + updTime, err := time.ParseInLocation("2006-01-02 15:04:05", f.UpdTime, time.Local) + if err != nil { + return nil, err + } + obj := model.Object{ + ID: strconv.FormatInt(f.FileId, 10), + //Path: "", + Name: f.FileName, + Size: f.FileSize * 1024, + Modified: updTime, + Ctime: updTime, + IsFolder: false, + //HashInfo: utils.HashInfo{}, + } + if f.FileType == 2 { + obj.IsFolder = true + obj.Size = 0 + obj.ID = strconv.FormatInt(f.FolderId, 10) + obj.Name = f.FolderName + } + return &obj, nil + }) +} + +func (d *ILanZou) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + u, err := url.Parse(d.conf.base + "/" + d.conf.unproved + "/file/redirect") + if err != nil { + return nil, err + } + ts, ts_str, _ := getTimestamp(d.conf.secret) + + params := []string{ + "uuid=" + url.QueryEscape(d.UUID), + "devType=6", + "devCode=" + url.QueryEscape(d.UUID), + "devModel=chrome", + "devVersion=" + url.QueryEscape(d.conf.devVersion), + "appVersion=", + "timestamp=" + ts_str, + "appToken=" + url.QueryEscape(d.Token), + "enable=0", + } + + downloadId, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%s", file.GetID(), d.userID)), d.conf.secret) + if err != nil { + return nil, err + } + params = append(params, "downloadId="+url.QueryEscape(hex.EncodeToString(downloadId))) + + auth, err := mopan.AesEncrypt([]byte(fmt.Sprintf("%s|%d", file.GetID(), ts)), d.conf.secret) + if err != nil { + return nil, err + } + params = append(params, "auth="+url.QueryEscape(hex.EncodeToString(auth))) + + u.RawQuery = strings.Join(params, "&") + realURL := u.String() + // get the url after redirect + req := base.NoRedirectClient.R() + + req.SetHeaders(map[string]string{ + "Referer": d.conf.site + "/", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", + }) + if d.Addition.Ip != "" { + req.SetHeader("X-Forwarded-For", d.Addition.Ip) + } + + res, err := req.Get(realURL) + if err != nil { + return nil, err + } + if res.StatusCode() == 302 { + realURL = res.Header().Get("location") + } else { + return nil, fmt.Errorf("redirect failed, status: %d, msg: %s", res.StatusCode(), utils.Json.Get(res.Body(), "msg").ToString()) + } + link := model.Link{URL: realURL} + return &link, nil +} + +func (d *ILanZou) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + res, err := d.proved("/file/folder/save", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "folderDesc": "", + "folderId": parentDir.GetID(), + "folderName": dirName, + }) + }) + if err != nil { + return nil, err + } + return &model.Object{ + ID: utils.Json.Get(res, "list", 0, "id").ToString(), + //Path: "", + Name: dirName, + Size: 0, + Modified: time.Now(), + Ctime: time.Now(), + IsFolder: true, + //HashInfo: utils.HashInfo{}, + }, nil +} + +func (d *ILanZou) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var fileIds, folderIds []string + if srcObj.IsDir() { + folderIds = []string{srcObj.GetID()} + } else { + fileIds = []string{srcObj.GetID()} + } + _, err := d.proved("/file/folder/move", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "folderIds": strings.Join(folderIds, ","), + "fileIds": strings.Join(fileIds, ","), + "targetId": dstDir.GetID(), + }) + }) + if err != nil { + return nil, err + } + return srcObj, nil +} + +func (d *ILanZou) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var err error + if srcObj.IsDir() { + _, err = d.proved("/file/folder/edit", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "folderDesc": "", + "folderId": srcObj.GetID(), + "folderName": newName, + }) + }) + } else { + _, err = d.proved("/file/edit", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "fileDesc": "", + "fileId": srcObj.GetID(), + "fileName": newName, + }) + }) + } + if err != nil { + return nil, err + } + return &model.Object{ + ID: srcObj.GetID(), + //Path: "", + Name: newName, + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ILanZou) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // TODO copy obj, optional + return nil, errs.NotImplement +} + +func (d *ILanZou) Remove(ctx context.Context, obj model.Obj) error { + var fileIds, folderIds []string + if obj.IsDir() { + folderIds = []string{obj.GetID()} + } else { + fileIds = []string{obj.GetID()} + } + _, err := d.proved("/file/delete", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "folderIds": strings.Join(folderIds, ","), + "fileIds": strings.Join(fileIds, ","), + "status": 0, + }) + }) + return err +} + +const DefaultPartSize = 1024 * 1024 * 8 + +func (d *ILanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + etag := s.GetHash().GetHash(utils.MD5) + var err error + if len(etag) != utils.MD5.Width { + _, etag, err = stream.CacheFullInTempFileAndHash(s, utils.MD5) + if err != nil { + return nil, err + } + } + // get upToken + res, err := d.proved("/7n/getUpToken", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "fileId": "", + "fileName": s.GetName(), + "fileSize": s.GetSize()/1024 + 1, + "folderId": dstDir.GetID(), + "md5": etag, + "type": 1, + }) + }) + if err != nil { + return nil, err + } + upToken := utils.Json.Get(res, "upToken").ToString() + now := time.Now() + key := fmt.Sprintf("disk/%d/%d/%d/%s/%016d", now.Year(), now.Month(), now.Day(), d.account, now.UnixMilli()) + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: s, + Size: s.GetSize(), + }, + UpdateProgress: up, + }) + var token string + if s.GetSize() <= DefaultPartSize { + res, err := d.upClient.R().SetContext(ctx).SetMultipartFormData(map[string]string{ + "token": upToken, + "key": key, + "fname": s.GetName(), + }).SetMultipartField("file", s.GetName(), s.GetMimetype(), reader). + Post("https://upload.qiniup.com/") + if err != nil { + return nil, err + } + token = utils.Json.Get(res.Body(), "token").ToString() + } else { + keyBase64 := base64.URLEncoding.EncodeToString([]byte(key)) + res, err := d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads", d.conf.bucket, keyBase64)) + if err != nil { + return nil, err + } + uploadId := utils.Json.Get(res.Body(), "uploadId").ToString() + parts := make([]Part, 0) + partNum := (s.GetSize() + DefaultPartSize - 1) / DefaultPartSize + for i := 1; i <= int(partNum); i++ { + u := fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s/%d", d.conf.bucket, keyBase64, uploadId, i) + res, err = d.upClient.R().SetContext(ctx).SetHeader("Authorization", "UpToken "+upToken).SetBody(io.LimitReader(reader, DefaultPartSize)).Put(u) + if err != nil { + return nil, err + } + etag := utils.Json.Get(res.Body(), "etag").ToString() + parts = append(parts, Part{ + PartNumber: i, + ETag: etag, + }) + } + res, err = d.upClient.R().SetHeader("Authorization", "UpToken "+upToken).SetBody(base.Json{ + "fnmae": s.GetName(), + "parts": parts, + }).Post(fmt.Sprintf("https://upload.qiniup.com/buckets/%s/objects/%s/uploads/%s", d.conf.bucket, keyBase64, uploadId)) + if err != nil { + return nil, err + } + token = utils.Json.Get(res.Body(), "token").ToString() + } + // commit upload + var resp UploadResultResp + for i := 0; i < 10; i++ { + _, err = d.unproved("/7n/results", http.MethodPost, func(req *resty.Request) { + params := []string{ + "tokenList=" + token, + "tokenTime=" + time.Now().Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)"), + } + queryString := strings.Join(params, "&") + req.SetQueryString(queryString).SetResult(&resp) + }) + if err != nil { + return nil, err + } + if len(resp.List) == 0 { + return nil, fmt.Errorf("upload failed, empty response") + } + if resp.List[0].Status == 1 { + break + } + time.Sleep(time.Second * 1) + } + file := resp.List[0] + if file.Status != 1 { + return nil, fmt.Errorf("upload failed, status: %d", resp.List[0].Status) + } + return &model.Object{ + ID: strconv.FormatInt(file.FileId, 10), + //Path: , + Name: file.FileName, + Size: s.GetSize(), + Modified: s.ModTime(), + Ctime: s.CreateTime(), + IsFolder: false, + HashInfo: utils.NewHashInfo(utils.MD5, etag), + }, nil +} + +//func (d *ILanZou) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*ILanZou)(nil) diff --git a/drivers/ilanzou/meta.go b/drivers/ilanzou/meta.go new file mode 100644 index 00000000000..7a4a00fb655 --- /dev/null +++ b/drivers/ilanzou/meta.go @@ -0,0 +1,81 @@ +package template + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Username string `json:"username" type:"string" required:"true"` + Password string `json:"password" type:"string" required:"true"` + Ip string `json:"ip" type:"string"` + + Token string + UUID string +} + +type Conf struct { + base string + secret []byte + bucket string + unproved string + proved string + devVersion string + site string +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ILanZou{ + config: driver.Config{ + Name: "ILanZou", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, + }, + conf: Conf{ + base: "https://api.ilanzou.com", + secret: []byte("lanZouY-disk-app"), + bucket: "wpanstore-lanzou", + unproved: "unproved", + proved: "proved", + devVersion: "125", + site: "https://www.ilanzou.com", + }, + } + }) + op.RegisterDriver(func() driver.Driver { + return &ILanZou{ + config: driver.Config{ + Name: "FeijiPan", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, + }, + conf: Conf{ + base: "https://api.feijipan.com", + secret: []byte("dingHao-disk-app"), + bucket: "wpanstore", + unproved: "ws", + proved: "app", + devVersion: "125", + site: "https://www.feijipan.com", + }, + } + }) +} diff --git a/drivers/ilanzou/types.go b/drivers/ilanzou/types.go new file mode 100644 index 00000000000..135724c749c --- /dev/null +++ b/drivers/ilanzou/types.go @@ -0,0 +1,57 @@ +package template + +type ListResp struct { + Msg string `json:"msg"` + Total int `json:"total"` + Code int `json:"code"` + Offset int `json:"offset"` + TotalPage int `json:"totalPage"` + Limit int `json:"limit"` + List []ListItem `json:"list"` +} + +type ListItem struct { + IconId int `json:"iconId"` + IsAmt int `json:"isAmt"` + FolderDesc string `json:"folderDesc,omitempty"` + AddTime string `json:"addTime"` + FolderId int64 `json:"folderId"` + ParentId int64 `json:"parentId"` + ParentName string `json:"parentName"` + NoteType int `json:"noteType,omitempty"` + UpdTime string `json:"updTime"` + IsShare int `json:"isShare"` + FolderIcon string `json:"folderIcon,omitempty"` + FolderName string `json:"folderName,omitempty"` + FileType int `json:"fileType"` + Status int `json:"status"` + IsFileShare int `json:"isFileShare,omitempty"` + FileName string `json:"fileName,omitempty"` + FileStars float64 `json:"fileStars,omitempty"` + IsFileDownload int `json:"isFileDownload,omitempty"` + FileComments int `json:"fileComments,omitempty"` + FileSize int64 `json:"fileSize,omitempty"` + FileIcon string `json:"fileIcon,omitempty"` + FileDownloads int `json:"fileDownloads,omitempty"` + FileUrl interface{} `json:"fileUrl"` + FileLikes int `json:"fileLikes,omitempty"` + FileId int64 `json:"fileId,omitempty"` +} + +type Part struct { + PartNumber int `json:"partNumber"` + ETag string `json:"etag"` +} + +type UploadResultResp struct { + Msg string `json:"msg"` + Code int `json:"code"` + List []struct { + FileIconId int `json:"fileIconId"` + FileName string `json:"fileName"` + FileIcon string `json:"fileIcon"` + FileId int64 `json:"fileId"` + Status int `json:"status"` + Token string `json:"token"` + } `json:"list"` +} diff --git a/drivers/ilanzou/util.go b/drivers/ilanzou/util.go new file mode 100644 index 00000000000..81773afbc80 --- /dev/null +++ b/drivers/ilanzou/util.go @@ -0,0 +1,117 @@ +package template + +import ( + "encoding/hex" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/foxxorcat/mopan-sdk-go" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +func (d *ILanZou) login() error { + res, err := d.unproved("/login", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "loginName": d.Username, + "loginPwd": d.Password, + }) + }) + if err != nil { + return err + } + d.Token = utils.Json.Get(res, "data", "appToken").ToString() + if d.Token == "" { + return fmt.Errorf("failed to login: token is empty, resp: %s", res) + } + return nil +} + +func getTimestamp(secret []byte) (int64, string, error) { + ts := time.Now().UnixMilli() + tsStr := strconv.FormatInt(ts, 10) + res, err := mopan.AesEncrypt([]byte(tsStr), secret) + if err != nil { + return 0, "", err + } + return ts, hex.EncodeToString(res), nil +} + +func (d *ILanZou) request(pathname, method string, callback base.ReqCallback, proved bool, retry ...bool) ([]byte, error) { + _, ts_str, err := getTimestamp(d.conf.secret) + if err != nil { + return nil, err + } + + params := []string{ + "uuid=" + url.QueryEscape(d.UUID), + "devType=6", + "devCode=" + url.QueryEscape(d.UUID), + "devModel=chrome", + "devVersion=" + url.QueryEscape(d.conf.devVersion), + "appVersion=", + "timestamp=" + ts_str, + } + + if proved { + params = append(params, "appToken="+url.QueryEscape(d.Token)) + } + + params = append(params, "extra=2") + + queryString := strings.Join(params, "&") + + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "Origin": d.conf.site, + "Referer": d.conf.site + "/", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0", + "Accept-Encoding": "gzip, deflate, br, zstd", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6,mt;q=0.5", + }) + + if d.Addition.Ip != "" { + req.SetHeader("X-Forwarded-For", d.Addition.Ip) + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, d.conf.base+pathname+"?"+queryString) + if err != nil { + if res != nil { + log.Errorf("[iLanZou] request error: %s", res.String()) + } + return nil, err + } + isRetry := len(retry) > 0 && retry[0] + body := res.Body() + code := utils.Json.Get(body, "code").ToInt() + msg := utils.Json.Get(body, "msg").ToString() + if code != 200 { + if !isRetry && proved && (utils.SliceContains([]int{-1, -2}, code) || d.Token == "") { + err = d.login() + if err != nil { + return nil, err + } + return d.request(pathname, method, callback, proved, true) + } + return nil, fmt.Errorf("%d: %s", code, msg) + } + return body, nil +} + +func (d *ILanZou) unproved(pathname, method string, callback base.ReqCallback) ([]byte, error) { + return d.request("/"+d.conf.unproved+pathname, method, callback, false) +} + +func (d *ILanZou) proved(pathname, method string, callback base.ReqCallback) ([]byte, error) { + return d.request("/"+d.conf.proved+pathname, method, callback, true) +} diff --git a/drivers/ipfs_api/driver.go b/drivers/ipfs_api/driver.go index cf21e62da59..264cef28c05 100644 --- a/drivers/ipfs_api/driver.go +++ b/drivers/ipfs_api/driver.go @@ -4,13 +4,12 @@ import ( "context" "fmt" "net/url" - stdpath "path" - "path/filepath" - "strings" + "path" + + shell "github.com/ipfs/go-ipfs-api" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" - shell "github.com/ipfs/go-ipfs-api" ) type IPFS struct { @@ -43,82 +42,143 @@ func (d *IPFS) Drop(ctx context.Context) error { } func (d *IPFS) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - path := dir.GetPath() - if path[len(path):] != "/" { - path += "/" - } - - path_cid, err := d.sh.FilesStat(ctx, path) - if err != nil { - return nil, err + var ipfsPath string + cid := dir.GetID() + if cid != "" { + ipfsPath = path.Join("/ipfs", cid) + } else { + // 可能出现ipns dns解析失败的情况,需要重复获取cid,其他情况应该不会出错 + ipfsPath = dir.GetPath() + switch d.Mode { + case "ipfs": + ipfsPath = path.Join("/ipfs", ipfsPath) + case "ipns": + ipfsPath = path.Join("/ipns", ipfsPath) + case "mfs": + fileStat, err := d.sh.FilesStat(ctx, ipfsPath) + if err != nil { + return nil, err + } + ipfsPath = path.Join("/ipfs", fileStat.Hash) + default: + return nil, fmt.Errorf("mode error") + } } - - dirs, err := d.sh.List(path_cid.Hash) + dirs, err := d.sh.List(ipfsPath) if err != nil { return nil, err } objlist := []model.Obj{} for _, file := range dirs { - gateurl := *d.gateURL - gateurl.Path = "ipfs/" + file.Hash - gateurl.RawQuery = "filename=" + file.Name - objlist = append(objlist, &model.ObjectURL{ - Object: model.Object{ID: file.Hash, Name: file.Name, Size: int64(file.Size), IsFolder: file.Type == 1}, - Url: model.Url{Url: gateurl.String()}, - }) + objlist = append(objlist, &model.Object{ID: file.Hash, Name: file.Name, Size: int64(file.Size), IsFolder: file.Type == 1}) } return objlist, nil } func (d *IPFS) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - link := d.Gateway + "/ipfs/" + file.GetID() + "/?filename=" + file.GetName() - return &model.Link{URL: link}, nil + gateurl := d.gateURL.JoinPath("/ipfs/", file.GetID()) + gateurl.RawQuery = "filename=" + url.QueryEscape(file.GetName()) + return &model.Link{URL: gateurl.String()}, nil } -func (d *IPFS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - path := parentDir.GetPath() - if path[len(path):] != "/" { - path += "/" +func (d *IPFS) Get(ctx context.Context, rawPath string) (model.Obj, error) { + rawPath = path.Join(d.GetRootPath(), rawPath) + var ipfsPath string + switch d.Mode { + case "ipfs": + ipfsPath = path.Join("/ipfs", rawPath) + case "ipns": + ipfsPath = path.Join("/ipns", rawPath) + case "mfs": + fileStat, err := d.sh.FilesStat(ctx, rawPath) + if err != nil { + return nil, err + } + ipfsPath = path.Join("/ipfs", fileStat.Hash) + default: + return nil, fmt.Errorf("mode error") + } + file, err := d.sh.FilesStat(ctx, ipfsPath) + if err != nil { + return nil, err + } + return &model.Object{ID: file.Hash, Name: path.Base(rawPath), Path: rawPath, Size: int64(file.Size), IsFolder: file.Type == "directory"}, nil +} + +func (d *IPFS) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.Mode != "mfs" { + return nil, fmt.Errorf("only write in mfs mode") + } + dirPath := parentDir.GetPath() + err := d.sh.FilesMkdir(ctx, path.Join(dirPath, dirName), shell.FilesMkdir.Parents(true)) + if err != nil { + return nil, err + } + file, err := d.sh.FilesStat(ctx, path.Join(dirPath, dirName)) + if err != nil { + return nil, err } - return d.sh.FilesMkdir(ctx, path+dirName) + return &model.Object{ID: file.Hash, Name: dirName, Path: path.Join(dirPath, dirName), Size: int64(file.Size), IsFolder: true}, nil } -func (d *IPFS) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - return d.sh.FilesMv(ctx, srcObj.GetPath(), dstDir.GetPath()) +func (d *IPFS) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.Mode != "mfs" { + return nil, fmt.Errorf("only write in mfs mode") + } + dstPath := path.Join(dstDir.GetPath(), path.Base(srcObj.GetPath())) + d.sh.FilesRm(ctx, dstPath, true) + return &model.Object{ID: srcObj.GetID(), Name: srcObj.GetName(), Path: dstPath, Size: int64(srcObj.GetSize()), IsFolder: srcObj.IsDir()}, + d.sh.FilesMv(ctx, srcObj.GetPath(), dstDir.GetPath()) } -func (d *IPFS) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - newFileName := filepath.Dir(srcObj.GetPath()) + "/" + newName - return d.sh.FilesMv(ctx, srcObj.GetPath(), strings.ReplaceAll(newFileName, "\\", "/")) +func (d *IPFS) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.Mode != "mfs" { + return nil, fmt.Errorf("only write in mfs mode") + } + dstPath := path.Join(path.Dir(srcObj.GetPath()), newName) + d.sh.FilesRm(ctx, dstPath, true) + return &model.Object{ID: srcObj.GetID(), Name: newName, Path: dstPath, Size: int64(srcObj.GetSize()), + IsFolder: srcObj.IsDir()}, d.sh.FilesMv(ctx, srcObj.GetPath(), dstPath) } -func (d *IPFS) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - // TODO copy obj, optional - fmt.Println(srcObj.GetPath()) - fmt.Println(dstDir.GetPath()) - newFileName := dstDir.GetPath() + "/" + filepath.Base(srcObj.GetPath()) - fmt.Println(newFileName) - return d.sh.FilesCp(ctx, srcObj.GetPath(), strings.ReplaceAll(newFileName, "\\", "/")) +func (d *IPFS) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.Mode != "mfs" { + return nil, fmt.Errorf("only write in mfs mode") + } + dstPath := path.Join(dstDir.GetPath(), path.Base(srcObj.GetPath())) + d.sh.FilesRm(ctx, dstPath, true) + return &model.Object{ID: srcObj.GetID(), Name: srcObj.GetName(), Path: dstPath, Size: int64(srcObj.GetSize()), IsFolder: srcObj.IsDir()}, + d.sh.FilesCp(ctx, path.Join("/ipfs/", srcObj.GetID()), dstPath, shell.FilesCp.Parents(true)) } func (d *IPFS) Remove(ctx context.Context, obj model.Obj) error { - // TODO remove obj, optional + if d.Mode != "mfs" { + return fmt.Errorf("only write in mfs mode") + } return d.sh.FilesRm(ctx, obj.GetPath(), true) } -func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - // TODO upload file, optional - _, err := d.sh.Add(stream, ToFiles(stdpath.Join(dstDir.GetPath(), stream.GetName()))) - return err -} - -func ToFiles(dstDir string) shell.AddOpts { - return func(rb *shell.RequestBuilder) error { - rb.Option("to-files", dstDir) - return nil +func (d *IPFS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.Mode != "mfs" { + return nil, fmt.Errorf("only write in mfs mode") + } + outHash, err := d.sh.Add(driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + })) + if err != nil { + return nil, err + } + dstPath := path.Join(dstDir.GetPath(), s.GetName()) + if s.GetExist() != nil { + d.sh.FilesRm(ctx, dstPath, true) } + err = d.sh.FilesCp(ctx, path.Join("/ipfs/", outHash), dstPath, shell.FilesCp.Parents(true)) + gateurl := d.gateURL.JoinPath("/ipfs/", outHash) + gateurl.RawQuery = "filename=" + url.QueryEscape(s.GetName()) + return &model.Object{ID: outHash, Name: s.GetName(), Path: dstPath, Size: int64(s.GetSize()), IsFolder: s.IsDir()}, err } //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/ipfs_api/meta.go b/drivers/ipfs_api/meta.go index cdc3042434b..3837bec2cbf 100644 --- a/drivers/ipfs_api/meta.go +++ b/drivers/ipfs_api/meta.go @@ -8,14 +8,16 @@ import ( type Addition struct { // Usually one of two driver.RootPath - Endpoint string `json:"endpoint" default:"http://127.0.0.1:5001"` - Gateway string `json:"gateway" default:"https://ipfs.io"` + Mode string `json:"mode" options:"ipfs,ipns,mfs" type:"select" required:"true"` + Endpoint string `json:"endpoint" default:"http://127.0.0.1:5001" required:"true"` + Gateway string `json:"gateway" default:"http://127.0.0.1:8080" required:"true"` } var config = driver.Config{ Name: "IPFS API", DefaultRoot: "/", LocalSort: true, + OnlyProxy: false, } func init() { diff --git a/drivers/kodbox/driver.go b/drivers/kodbox/driver.go new file mode 100644 index 00000000000..c536c916d7a --- /dev/null +++ b/drivers/kodbox/driver.go @@ -0,0 +1,278 @@ +package kodbox + +import ( + "context" + "fmt" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type KodBox struct { + model.Storage + Addition + authorization string +} + +func (d *KodBox) Config() driver.Config { + return config +} + +func (d *KodBox) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *KodBox) Init(ctx context.Context) error { + d.Address = strings.TrimSuffix(d.Address, "/") + d.RootFolderPath = strings.TrimPrefix(utils.FixAndCleanPath(d.RootFolderPath), "/") + return d.getToken() +} + +func (d *KodBox) Drop(ctx context.Context) error { + return nil +} + +func (d *KodBox) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var ( + resp *CommonResp + listPathData *ListPathData + ) + + _, err := d.request(http.MethodPost, "/?explorer/list/path", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "path": dir.GetPath(), + }) + }, true) + if err != nil { + return nil, err + } + + dataBytes, err := utils.Json.Marshal(resp.Data) + if err != nil { + return nil, err + } + + err = utils.Json.Unmarshal(dataBytes, &listPathData) + if err != nil { + return nil, err + } + FolderAndFiles := append(listPathData.FolderList, listPathData.FileList...) + + return utils.SliceConvert(FolderAndFiles, func(f FolderOrFile) (model.Obj, error) { + return &model.ObjThumb{ + Object: model.Object{ + Path: f.Path, + Name: f.Name, + Ctime: time.Unix(f.CreateTime, 0), + Modified: time.Unix(f.ModifyTime, 0), + Size: f.Size, + IsFolder: f.Type == "folder", + }, + //Thumbnail: model.Thumbnail{}, + }, nil + }) +} + +func (d *KodBox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + path := file.GetPath() + return &model.Link{ + URL: fmt.Sprintf("%s/?explorer/index/fileOut&path=%s&download=1&accessToken=%s", + d.Address, + path, + d.authorization)}, nil +} + +func (d *KodBox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var resp *CommonResp + newDirPath := filepath.Join(parentDir.GetPath(), dirName) + + _, err := d.request(http.MethodPost, "/?explorer/index/mkdir", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "path": newDirPath, + }) + }) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + + return &model.ObjThumb{ + Object: model.Object{ + Path: resp.Info.(string), + Name: dirName, + IsFolder: true, + Modified: time.Now(), + Ctime: time.Now(), + }, + }, nil +} + +func (d *KodBox) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathCuteTo", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", + srcObj.GetPath(), + srcObj.GetName()), + "path": dstDir.GetPath(), + }) + }, true) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + + return &model.ObjThumb{ + Object: model.Object{ + Path: srcObj.GetPath(), + Name: srcObj.GetName(), + IsFolder: srcObj.IsDir(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + }, + }, nil +} + +func (d *KodBox) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathRename", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "path": srcObj.GetPath(), + "newName": newName, + }) + }, true) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + return &model.ObjThumb{ + Object: model.Object{ + Path: srcObj.GetPath(), + Name: newName, + IsFolder: srcObj.IsDir(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + }, + }, nil +} + +func (d *KodBox) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathCopyTo", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", + srcObj.GetPath(), + srcObj.GetName()), + "path": dstDir.GetPath(), + }) + }) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + + path := resp.Info.([]interface{})[0].(string) + objectName, err := d.getFileOrFolderName(ctx, path) + if err != nil { + return nil, err + } + return &model.ObjThumb{ + Object: model.Object{ + Path: path, + Name: *objectName, + IsFolder: srcObj.IsDir(), + Modified: time.Now(), + Ctime: time.Now(), + }, + }, nil +} + +func (d *KodBox) Remove(ctx context.Context, obj model.Obj) error { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathDelete", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "dataArr": fmt.Sprintf("[{\"path\": \"%s\", \"name\": \"%s\"}]", + obj.GetPath(), + obj.GetName()), + "shiftDelete": "1", + }) + }) + if err != nil { + return err + } + code := resp.Code.(bool) + if !code { + return fmt.Errorf("%s", resp.Data) + } + return nil +} + +func (d *KodBox) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/upload/fileUpload", func(req *resty.Request) { + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + req.SetFileReader("file", s.GetName(), r). + SetResult(&resp). + SetFormData(map[string]string{ + "path": dstDir.GetPath(), + }). + SetContext(ctx) + }) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + return &model.ObjThumb{ + Object: model.Object{ + Path: resp.Info.(string), + Name: s.GetName(), + Size: s.GetSize(), + IsFolder: false, + Modified: time.Now(), + Ctime: time.Now(), + }, + }, nil +} + +func (d *KodBox) getFileOrFolderName(ctx context.Context, path string) (*string, error) { + var resp *CommonResp + _, err := d.request(http.MethodPost, "/?explorer/index/pathInfo", func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "dataArr": fmt.Sprintf("[{\"path\": \"%s\"}]", path)}) + }) + if err != nil { + return nil, err + } + code := resp.Code.(bool) + if !code { + return nil, fmt.Errorf("%s", resp.Data) + } + folderOrFileName := resp.Data.(map[string]any)["name"].(string) + return &folderOrFileName, nil +} + +var _ driver.Driver = (*KodBox)(nil) diff --git a/drivers/kodbox/meta.go b/drivers/kodbox/meta.go new file mode 100644 index 00000000000..318fb9ec56f --- /dev/null +++ b/drivers/kodbox/meta.go @@ -0,0 +1,25 @@ +package kodbox + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + + Address string `json:"address" required:"true"` + UserName string `json:"username" required:"false"` + Password string `json:"password" required:"false"` +} + +var config = driver.Config{ + Name: "KodBox", + DefaultRoot: "", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &KodBox{} + }) +} diff --git a/drivers/kodbox/types.go b/drivers/kodbox/types.go new file mode 100644 index 00000000000..9bd45d9b366 --- /dev/null +++ b/drivers/kodbox/types.go @@ -0,0 +1,24 @@ +package kodbox + +type CommonResp struct { + Code any `json:"code"` + TimeUse string `json:"timeUse"` + TimeNow string `json:"timeNow"` + Data any `json:"data"` + Info any `json:"info"` +} + +type ListPathData struct { + FolderList []FolderOrFile `json:"folderList"` + FileList []FolderOrFile `json:"fileList"` +} + +type FolderOrFile struct { + Name string `json:"name"` + Path string `json:"path"` + Type string `json:"type"` + Ext string `json:"ext,omitempty"` // 文件特有字段 + Size int64 `json:"size"` + CreateTime int64 `json:"createTime"` + ModifyTime int64 `json:"modifyTime"` +} diff --git a/drivers/kodbox/util.go b/drivers/kodbox/util.go new file mode 100644 index 00000000000..2c04cd73f29 --- /dev/null +++ b/drivers/kodbox/util.go @@ -0,0 +1,86 @@ +package kodbox + +import ( + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "strings" +) + +func (d *KodBox) getToken() error { + var authResp CommonResp + res, err := base.RestyClient.R(). + SetResult(&authResp). + SetQueryParams(map[string]string{ + "name": d.UserName, + "password": d.Password, + }). + Post(d.Address + "/?user/index/loginSubmit") + if err != nil { + return err + } + if res.StatusCode() >= 400 { + return fmt.Errorf("get token failed: %s", res.String()) + } + + if res.StatusCode() == 200 && authResp.Code.(bool) == false { + return fmt.Errorf("get token failed: %s", res.String()) + } + + d.authorization = fmt.Sprintf("%s", authResp.Info) + return nil +} + +func (d *KodBox) request(method string, pathname string, callback base.ReqCallback, noRedirect ...bool) ([]byte, error) { + full := pathname + if !strings.HasPrefix(pathname, "http") { + full = d.Address + pathname + } + req := base.RestyClient.R() + if len(noRedirect) > 0 && noRedirect[0] { + req = base.NoRedirectClient.R() + } + req.SetFormData(map[string]string{ + "accessToken": d.authorization, + }) + callback(req) + + var ( + res *resty.Response + commonResp *CommonResp + err error + skip bool + ) + for i := 0; i < 2; i++ { + if skip { + break + } + res, err = req.Execute(method, full) + if err != nil { + return nil, err + } + + err := utils.Json.Unmarshal(res.Body(), &commonResp) + if err != nil { + return nil, err + } + + switch commonResp.Code.(type) { + case bool: + skip = true + case string: + if commonResp.Code.(string) == "10001" { + err = d.getToken() + if err != nil { + return nil, err + } + req.SetFormData(map[string]string{"accessToken": d.authorization}) + } + } + } + if commonResp.Code.(bool) == false { + return nil, fmt.Errorf("request failed: %s", commonResp.Data) + } + return res.Body(), nil +} diff --git a/drivers/lanzou/driver.go b/drivers/lanzou/driver.go index cdb56f79658..877e72bb3d1 100644 --- a/drivers/lanzou/driver.go +++ b/drivers/lanzou/driver.go @@ -30,6 +30,9 @@ func (d *LanZou) GetAddition() driver.Additional { } func (d *LanZou) Init(ctx context.Context) (err error) { + if d.UserAgent == "" { + d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/89.0.4389.111 Safari/537.39" + } switch d.Type { case "account": _, err := d.Login() @@ -205,18 +208,22 @@ func (d *LanZou) Remove(ctx context.Context, obj model.Obj) error { return errs.NotSupport } -func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (d *LanZou) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { if d.IsCookie() || d.IsAccount() { var resp RespText[[]FileOrFolder] _, err := d._post(d.BaseUrl+"/html5up.php", func(req *resty.Request) { + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) req.SetFormData(map[string]string{ "task": "1", "vie": "2", "ve": "2", "id": "WU_FILE_0", - "name": stream.GetName(), + "name": s.GetName(), "folder_id_bb_n": dstDir.GetID(), - }).SetFileReader("upload_file", stream.GetName(), stream).SetContext(ctx) + }).SetFileReader("upload_file", s.GetName(), reader).SetContext(ctx) }, &resp, true) if err != nil { return nil, err diff --git a/drivers/lanzou/help.go b/drivers/lanzou/help.go index 31a558e9c75..b9e7e5cdbd4 100644 --- a/drivers/lanzou/help.go +++ b/drivers/lanzou/help.go @@ -1,7 +1,6 @@ package lanzou import ( - "bytes" "fmt" "net/http" "regexp" @@ -78,6 +77,46 @@ func RemoveNotes(html string) string { }) } +// 清理JS注释 +func RemoveJSComment(data string) string { + var result strings.Builder + inComment := false + inSingleLineComment := false + + for i := 0; i < len(data); i++ { + v := data[i] + + if inSingleLineComment && (v == '\n' || v == '\r') { + inSingleLineComment = false + result.WriteByte(v) + continue + } + if inComment && v == '*' && i+1 < len(data) && data[i+1] == '/' { + inComment = false + i++ + continue + } + if v == '/' && i+1 < len(data) { + nextChar := data[i+1] + if nextChar == '*' { + inComment = true + i++ + continue + } else if nextChar == '/' { + inSingleLineComment = true + i++ + continue + } + } + if inComment || inSingleLineComment { + continue + } + result.WriteByte(v) + } + + return result.String() +} + var findAcwScV2Reg = regexp.MustCompile(`arg1='([0-9A-Z]+)'`) // 在页面被过多访问或其他情况下,有时候会先返回一个加密的页面,其执行计算出一个acw_sc__v2后放入页面后再重新访问页面才能获得正常页面 @@ -104,11 +143,13 @@ func Unbox(hex string) string { } func HexXor(hex1, hex2 string) string { - out := bytes.NewBuffer(make([]byte, len(hex1))) - for i := 0; i < len(hex1) && i < len(hex2); i += 2 { + var out strings.Builder + out.Grow(len(hex1)) + for i := 0; i+2 <= len(hex1) && i+2 <= len(hex2); i += 2 { v1, _ := strconv.ParseInt(hex1[i:i+2], 16, 64) v2, _ := strconv.ParseInt(hex2[i:i+2], 16, 64) - out.WriteString(strconv.FormatInt(v1^v2, 16)) + // 必须补足前导 0,否则异或结果 < 0x10 时只输出一个字符,导致整个 hex 串错位 + fmt.Fprintf(&out, "%02x", v1^v2) } return out.String() } @@ -120,9 +161,9 @@ var findKVReg = regexp.MustCompile(`'(.+?)':('?([^' },]*)'?)`) // 拆分kv func findJSVarFunc(key, data string) string { var values []string if key != "sasign" { - values = regexp.MustCompile(`var ` + key + ` = '(.+?)';`).FindStringSubmatch(data) + values = regexp.MustCompile(`var ` + key + `\s*=\s*['"]?(.+?)['"]?;`).FindStringSubmatch(data) } else { - matches := regexp.MustCompile(`var `+key+` = '(.+?)';`).FindAllStringSubmatch(data, -1) + matches := regexp.MustCompile(`var `+key+`\s*=\s*['"]?(.+?)['"]?;`).FindAllStringSubmatch(data, -1) if len(matches) == 3 { values = matches[1] } else { diff --git a/drivers/lanzou/meta.go b/drivers/lanzou/meta.go index c8db6448476..1e8826cadeb 100644 --- a/drivers/lanzou/meta.go +++ b/drivers/lanzou/meta.go @@ -16,7 +16,8 @@ type Addition struct { driver.RootID SharePassword string `json:"share_password"` BaseUrl string `json:"baseUrl" required:"true" default:"https://pc.woozooo.com" help:"basic URL for file operation"` - ShareUrl string `json:"shareUrl" required:"true" default:"https://pan.lanzouo.com" help:"used to get the sharing page"` + ShareUrl string `json:"shareUrl" required:"true" default:"https://pan.lanzoui.com" help:"used to get the sharing page"` + UserAgent string `json:"user_agent" required:"true" default:"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.39 (KHTML, like Gecko) Chrome/89.0.4389.111 Safari/537.39"` RepairFileInfo bool `json:"repair_file_info" help:"To use webdav, you need to enable it"` } diff --git a/drivers/lanzou/util.go b/drivers/lanzou/util.go index 8aeba8113e1..2fe584864e1 100644 --- a/drivers/lanzou/util.go +++ b/drivers/lanzou/util.go @@ -95,34 +95,55 @@ func (d *LanZou) _post(url string, callback base.ReqCallback, resp interface{}, } func (d *LanZou) request(url string, method string, callback base.ReqCallback, up bool) ([]byte, error) { - var req *resty.Request + var client *resty.Client if up { once.Do(func() { upClient = base.NewRestyClient().SetTimeout(120 * time.Second) }) - req = upClient.R() + client = upClient } else { - req = base.RestyClient.R() + client = base.RestyClient } - req.SetHeaders(map[string]string{ - "Referer": "https://pc.woozooo.com", - }) - - if d.Cookie != "" { - req.SetHeader("cookie", d.Cookie) - } - - if callback != nil { - callback(req) - } + // acw_sc__v2 反爬挑战可能出现在任意页面/接口(分享页、iframe 页、ajaxm.php 等)。 + // 挑战页内嵌 arg1,需据此算出 cookie 后用同一请求重试,故在最底层统一处理。 + var acwScV2 string + var body []byte + for i := 0; i < 3; i++ { + req := client.R() + req.SetHeaders(map[string]string{ + "Referer": "https://pc.woozooo.com", + "User-Agent": d.UserAgent, + }) + if d.Cookie != "" { + req.SetHeader("cookie", d.Cookie) + } + if acwScV2 != "" { + req.SetCookie(&http.Cookie{Name: "acw_sc__v2", Value: acwScV2}) + } + if callback != nil { + callback(req) + } - res, err := req.Execute(method, url) - if err != nil { - return nil, err + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + body = res.Body() + log.Debugf("lanzou request: url=>%s ,stats=>%d ,body => %s\n", res.Request.URL, res.StatusCode(), res.String()) + + if findAcwScV2Reg.Match(body) { + vs, e := CalcAcwScV2(string(body)) + if e != nil { + log.Errorf("lanzou: err => acw_sc__v2 validation error ,data => %s\n", body) + return body, e + } + acwScV2 = vs + continue + } + return body, nil } - log.Debugf("lanzou request: url=>%s ,stats=>%d ,body => %s\n", res.Request.URL, res.StatusCode(), res.String()) - return res.Body(), err + return body, errors.New("acw_sc__v2 validation error") } func (d *LanZou) Login() ([]*http.Cookie, error) { @@ -263,42 +284,31 @@ var findSubFolderReg = regexp.MustCompile(`(?i)(?:folderlink|mbxfolder).+href="/ // 获取下载页面链接 var findDownPageParamReg = regexp.MustCompile(` acw_sc__v2 validation error ,data => %s\n", firstPageDataStr) - return "", err - } - continue - } - return firstPageDataStr, nil +// 获取分享链接主界面 +func (d *LanZou) getShareUrlHtml(shareID string) (string, error) { + htmlStr, err := d.getHtml(fmt.Sprint(d.ShareUrl, "/", shareID), nil) + if err != nil { + return "", err + } + if strings.Contains(htmlStr, "取消分享") { + return "", ErrFileShareCancel } - return "", errors.New("acw_sc__v2 validation error") + if strings.Contains(htmlStr, "文件不存在") { + return "", ErrFileNotExist + } + return htmlStr, nil } // 通过分享链接获取文件或文件夹 @@ -344,6 +354,10 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( file FileOrFolderByShareUrl ) + // 删除注释 + sharePageData = RemoveNotes(sharePageData) + sharePageData = RemoveJSComment(sharePageData) + // 需要密码 if strings.Contains(sharePageData, "pwdload") || strings.Contains(sharePageData, "passwddiv") { sharePageData, err := getJSFunctionByName(sharePageData, "down_p") @@ -355,8 +369,16 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( return nil, err } param["p"] = pwd + + fileIDs := findFileIDReg.FindStringSubmatch(sharePageData) + var fileID string + if len(fileIDs) > 1 { + fileID = fileIDs[1] + } else { + return nil, fmt.Errorf("not find file id") + } var resp FileShareInfoAndUrlResp[string] - _, err = d.post(d.ShareUrl+"/ajaxm.php", func(req *resty.Request) { req.SetFormData(param) }, &resp) + _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } @@ -370,18 +392,24 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( log.Errorf("lanzou: err => not find file page param ,data => %s\n", sharePageData) return nil, fmt.Errorf("not find file page param") } - data, err := d.get(fmt.Sprint(d.ShareUrl, urlpaths[1]), nil) + nextPageData, err := d.getHtml(fmt.Sprint(d.ShareUrl, urlpaths[1]), nil) if err != nil { return nil, err } - nextPageData := RemoveNotes(string(data)) param, err = htmlJsonToMap(nextPageData) if err != nil { return nil, err } + fileIDs := findFileIDReg.FindStringSubmatch(nextPageData) + var fileID string + if len(fileIDs) > 1 { + fileID = fileIDs[1] + } else { + return nil, fmt.Errorf("not find file id") + } var resp FileShareInfoAndUrlResp[int] - _, err = d.post(d.ShareUrl+"/ajaxm.php", func(req *resty.Request) { req.SetFormData(param) }, &resp) + _, err = d.post(d.ShareUrl+"/ajaxm.php?file="+fileID, func(req *resty.Request) { req.SetFormData(param) }, &resp) if err != nil { return nil, err } @@ -407,17 +435,35 @@ func (d *LanZou) getFilesByShareUrl(shareID, pwd string, sharePageData string) ( file.Time = timeFindReg.FindString(sharePageData) // 重定向获取真实链接 - res, err := base.NoRedirectClient.R().SetHeaders(map[string]string{ + headers := map[string]string{ "accept-language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6", - }).Get(downloadUrl) + } + res, err := base.NoRedirectClient.R().SetHeaders(headers).Get(downloadUrl) if err != nil { return nil, err } + rPageData := res.String() + if findAcwScV2Reg.MatchString(rPageData) { + log.Debug("lanzou: detected acw_sc__v2 challenge, recalculating cookie") + acwScV2, err := CalcAcwScV2(rPageData) + if err != nil { + return nil, err + } + // retry with calculated cookie to bypass anti-crawler validation + res, err = base.NoRedirectClient.R(). + SetHeaders(headers). + SetCookie(&http.Cookie{Name: "acw_sc__v2", Value: acwScV2}). + Get(downloadUrl) + if err != nil { + return nil, err + } + rPageData = res.String() + } + file.Url = res.Header().Get("location") // 触发验证 - rPageData := res.String() if res.StatusCode() != 302 { param, err = htmlJsonToMap(rPageData) if err != nil { @@ -501,8 +547,8 @@ func (d *LanZou) getFileRealInfo(downURL string) (*int64, *time.Time) { } func (d *LanZou) getVeiAndUid() (vei string, uid string, err error) { - var resp []byte - resp, err = d.get("https://pc.woozooo.com/mydisk.php", func(req *resty.Request) { + // mydisk.php 同样可能返回 acw_sc__v2 反爬挑战页,需经 getHtml 处理后再解析 + html, err := d.getHtml("https://pc.woozooo.com/mydisk.php", func(req *resty.Request) { req.SetQueryParams(map[string]string{ "item": "files", "action": "index", @@ -512,7 +558,7 @@ func (d *LanZou) getVeiAndUid() (vei string, uid string, err error) { return } // uid - uids := regexp.MustCompile(`uid=([^'"&;]+)`).FindStringSubmatch(string(resp)) + uids := regexp.MustCompile(`uid=([^'"&;]+)`).FindStringSubmatch(html) if len(uids) < 2 { err = fmt.Errorf("uid variable not find") return @@ -520,7 +566,6 @@ func (d *LanZou) getVeiAndUid() (vei string, uid string, err error) { uid = uids[1] // vei - html := RemoveNotes(string(resp)) data, err := htmlJsonToMap(html) if err != nil { return diff --git a/drivers/lark.go b/drivers/lark.go new file mode 100644 index 00000000000..d5070078651 --- /dev/null +++ b/drivers/lark.go @@ -0,0 +1,8 @@ +// +build linux darwin windows +// +build amd64 arm64 + +package drivers + +import ( + _ "github.com/alist-org/alist/v3/drivers/lark" +) diff --git a/drivers/lark/driver.go b/drivers/lark/driver.go new file mode 100644 index 00000000000..ca704a2dbf3 --- /dev/null +++ b/drivers/lark/driver.go @@ -0,0 +1,693 @@ +package lark + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + lark "github.com/larksuite/oapi-sdk-go/v3" + larkcore "github.com/larksuite/oapi-sdk-go/v3/core" + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + larkext "github.com/larksuite/oapi-sdk-go/v3/service/ext" + log "github.com/sirupsen/logrus" + "golang.org/x/time/rate" +) + +type Lark struct { + model.Storage + Addition + + client *lark.Client + rootFolderToken string + tokenMu sync.Mutex +} + +const larkListPageSize = 200 +const larkTokenRefreshSkew = 5 * time.Minute + +func (c *Lark) Config() driver.Config { + return config +} + +func (c *Lark) GetAddition() driver.Additional { + return &c.Addition +} + +func (c *Lark) Init(ctx context.Context) error { + c.client = lark.NewClient(c.AppId, c.AppSecret, lark.WithTokenCache(newTokenCache())) + + paths := strings.Split(c.RootFolderPath, "/") + token := "" + + for _, p := range paths { + if p == "" { + token = "" + continue + } + + files, err := c.listFiles(ctx, token) + if err != nil { + return err + } + + found := false + for _, file := range files { + if *file.Type == "folder" && *file.Name == p { + token = *file.Token + found = true + break + } + } + if !found { + return errs.ObjectNotFound + } + } + + c.rootFolderToken = token + + return nil +} + +func (c *Lark) Drop(ctx context.Context) error { + return nil +} + +func (c *Lark) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + token, ok := c.getObjToken(ctx, dir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + if token == emptyFolderToken { + return nil, nil + } + + files, err := c.listFiles(ctx, token) + if err != nil { + return nil, err + } + + var res []model.Obj + + for _, file := range files { + res = append(res, larkFileToObj(c.RootFolderPath, dir.GetPath(), file)) + } + + return res, nil +} + +func larkFileToObj(rootFolderPath, dirPath string, file *larkdrive.File) model.Obj { + name := larkString(file.Name) + fileType := larkString(file.Type) + modifiedUnix, _ := strconv.ParseInt(larkString(file.ModifiedTime), 10, 64) + createdUnix, _ := strconv.ParseInt(larkString(file.CreatedTime), 10, 64) + obj := model.Object{ + ID: larkString(file.Token), + Path: strings.Join([]string{rootFolderPath, dirPath, name}, "/"), + Name: larkDisplayName(name, fileType), + Size: 0, + Modified: time.Unix(modifiedUnix, 0), + Ctime: time.Unix(createdUnix, 0), + IsFolder: fileType == "folder", + } + if file.Url == nil || *file.Url == "" || obj.IsFolder || !isLarkNativeDocType(fileType) { + return &obj + } + return &model.ObjectURL{ + Object: obj, + Url: model.Url{Url: *file.Url}, + } +} + +func larkDisplayName(name, fileType string) string { + if isLarkCloudDocName(name) { + return name + } + switch fileType { + case "doc": + return name + ".lark-doc" + case "docx": + return name + ".lark-docx" + case "sheet": + return name + ".lark-sheet" + case "bitable": + return name + ".lark-bitable" + case "mindnote": + return name + ".lark-mindnote" + case "slides": + return name + ".lark-slides" + default: + return name + } +} + +func isLarkNativeDocType(fileType string) bool { + switch fileType { + case "doc", "docx", "sheet", "bitable", "mindnote", "slides": + return true + default: + return false + } +} + +func larkString(s *string) string { + if s == nil { + return "" + } + return *s +} + +func (c *Lark) requestOpts(ctx context.Context) ([]larkcore.RequestOptionFunc, error) { + userAccessToken, err := c.ensureUserAccessToken(ctx, false) + if err != nil { + return nil, err + } + if userAccessToken == "" { + return nil, nil + } + return []larkcore.RequestOptionFunc{larkcore.WithUserAccessToken(userAccessToken)}, nil +} + +func (c *Lark) ensureUserAccessToken(ctx context.Context, forceRefresh bool) (string, error) { + if strings.TrimSpace(c.RefreshToken) == "" { + return strings.TrimSpace(c.UserAccessToken), nil + } + if token := strings.TrimSpace(c.UserAccessToken); !forceRefresh && token != "" && !c.userAccessTokenExpired() { + return token, nil + } + + c.tokenMu.Lock() + defer c.tokenMu.Unlock() + + if token := strings.TrimSpace(c.UserAccessToken); !forceRefresh && token != "" && !c.userAccessTokenExpired() { + return token, nil + } + if c.RefreshTokenExpiresAt > 0 && time.Now().After(time.Unix(c.RefreshTokenExpiresAt, 0)) { + return "", errors.New("lark refresh token expired") + } + + resp, err := c.client.Ext.Authen.RefreshAuthenAccessToken(ctx, + larkext.NewRefreshAuthenAccessTokenReqBuilder(). + Body(larkext.NewRefreshAuthenAccessTokenReqBodyBuilder(). + GrantType(larkext.GrantTypeRefreshCode). + RefreshToken(strings.TrimSpace(c.RefreshToken)). + Build()). + Build()) + if err != nil { + return "", err + } + if !resp.Success() { + return "", errors.New(resp.Error()) + } + if resp.Data == nil || resp.Data.AccessToken == "" { + return "", errors.New("lark refresh token response missing access token") + } + + now := time.Now() + c.UserAccessToken = resp.Data.AccessToken + c.UserAccessTokenExpiresAt = now.Add(time.Duration(resp.Data.ExpiresIn) * time.Second).Unix() + if resp.Data.RefreshToken != "" { + c.RefreshToken = resp.Data.RefreshToken + } + if resp.Data.RefreshExpiresIn > 0 { + c.RefreshTokenExpiresAt = now.Add(time.Duration(resp.Data.RefreshExpiresIn) * time.Second).Unix() + } + op.MustSaveDriverStorage(c) + + return c.UserAccessToken, nil +} + +func (c *Lark) forceRefreshUserAccessToken(ctx context.Context) error { + if strings.TrimSpace(c.RefreshToken) == "" { + return nil + } + _, err := c.ensureUserAccessToken(ctx, true) + return err +} + +func (c *Lark) userAccessTokenExpired() bool { + if c.UserAccessTokenExpiresAt <= 0 { + return true + } + return time.Now().Add(larkTokenRefreshSkew).After(time.Unix(c.UserAccessTokenExpiresAt, 0)) +} + +func (c *Lark) listFiles(ctx context.Context, folderToken string) ([]*larkdrive.File, error) { + var files []*larkdrive.File + pageToken := "" + + for { + builder := larkdrive.NewListFileReqBuilder(). + FolderToken(folderToken). + OrderBy("EditedTime"). + Direction("DESC") + if folderToken != "" { + builder.PageSize(larkListPageSize) + if pageToken != "" { + builder.PageToken(pageToken) + } + } + + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.ListFileResp, error) { + return c.client.Drive.V1.File.List(ctx, builder.Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + if resp.Data == nil { + return files, nil + } + + files = append(files, resp.Data.Files...) + if folderToken == "" || resp.Data.HasMore == nil || !*resp.Data.HasMore || + resp.Data.NextPageToken == nil || *resp.Data.NextPageToken == "" { + break + } + pageToken = *resp.Data.NextPageToken + } + + return files, nil +} + +type larkResp interface { + Success() bool + Error() string +} + +func doDrive[T larkResp](ctx context.Context, c *Lark, call func(...larkcore.RequestOptionFunc) (T, error)) (T, error) { + opts, err := c.requestOpts(ctx) + if err != nil { + var zero T + return zero, err + } + + resp, err := call(opts...) + if err != nil { + var zero T + return zero, err + } + if !isLarkAuthFailed(resp) || strings.TrimSpace(c.RefreshToken) == "" { + return resp, nil + } + + log.WithField("mount_path", c.MountPath).Warn("lark user access token auth failed, refreshing and retrying once") + if err = c.forceRefreshUserAccessToken(ctx); err != nil { + return resp, nil + } + opts, err = c.requestOpts(ctx) + if err != nil { + var zero T + return zero, err + } + return call(opts...) +} + +func isLarkAuthFailed(resp larkResp) bool { + if resp == nil || resp.Success() { + return false + } + switch v := any(resp).(type) { + case *larkdrive.ListFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.CreateFolderFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.MoveFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.CopyFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.DeleteFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadPrepareFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadPartFileResp: + return isLarkAuthFailedCode(v.Code) + case *larkdrive.UploadFinishFileResp: + return isLarkAuthFailedCode(v.Code) + default: + return strings.Contains(resp.Error(), "1061005") || + strings.Contains(strings.ToLower(resp.Error()), "auth") + } +} + +func isLarkAuthFailedCode(code int) bool { + return code == 1061005 || code == 99991663 || code == 99991664 || code == 99991668 +} + +func isHTTPAuthFailed(statusCode int) bool { + return statusCode == http.StatusUnauthorized || statusCode == http.StatusForbidden +} + +func (c *Lark) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + token, ok := c.getObjToken(ctx, file.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + if isLarkCloudDocName(file.GetName()) { + return &model.Link{ + URL: c.filePreviewURL(token), + }, nil + } + + if !c.WebProxy || c.ExternalMode { + return &model.Link{ + URL: c.filePreviewURL(token), + }, nil + } + + if c.WebProxy { + accessToken, err := c.downloadAccessToken(ctx, false) + if err != nil { + return nil, err + } + + url := fmt.Sprintf("https://open.feishu.cn/open-apis/drive/v1/files/%s/download", token) + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + req.Header.Set("Range", "bytes=0-1") + + ar, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + _ = ar.Body.Close() + + if isHTTPAuthFailed(ar.StatusCode) && strings.TrimSpace(c.RefreshToken) != "" { + accessToken, err = c.downloadAccessToken(ctx, true) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + ar, err = http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + _ = ar.Body.Close() + } + + if ar.StatusCode != http.StatusPartialContent { + return nil, errors.New("failed to get download link") + } + + return &model.Link{ + URL: url, + Header: http.Header{ + "Authorization": []string{fmt.Sprintf("Bearer %s", accessToken)}, + }, + }, nil + } + + return nil, errors.New("lark download requires web proxy") +} + +func (c *Lark) filePreviewURL(token string) string { + prefix := strings.TrimRight(strings.TrimSpace(c.TenantUrlPrefix), "/") + if prefix == "" { + prefix = "https://www.feishu.cn" + } + return prefix + "/file/" + token +} + +func (c *Lark) downloadAccessToken(ctx context.Context, forceRefresh bool) (string, error) { + var accessToken string + var err error + if strings.TrimSpace(c.RefreshToken) != "" || strings.TrimSpace(c.UserAccessToken) != "" { + accessToken, err = c.ensureUserAccessToken(ctx, forceRefresh) + if err != nil { + return "", err + } + } + if accessToken != "" { + return accessToken, nil + } + + resp, err := c.client.GetTenantAccessTokenBySelfBuiltApp(ctx, &larkcore.SelfBuiltTenantAccessTokenReq{ + AppID: c.AppId, + AppSecret: c.AppSecret, + }) + if err != nil { + return "", err + } + if !resp.Success() { + return "", errors.New(resp.Error()) + } + if resp.TenantAccessToken == "" { + return "", errors.New("lark tenant access token is empty") + } + return resp.TenantAccessToken, nil +} + +func (c *Lark) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + token, ok := c.getObjToken(ctx, parentDir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + body, err := larkdrive.NewCreateFolderFilePathReqBodyBuilder().FolderToken(token).Name(dirName).Build() + if err != nil { + return nil, err + } + + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CreateFolderFileResp, error) { + return c.client.Drive.File.CreateFolder(ctx, + larkdrive.NewCreateFolderFileReqBuilder().Body(body).Build(), opts...) + }) + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + return &model.Object{ + ID: *resp.Data.Token, + Path: strings.Join([]string{c.RootFolderPath, parentDir.GetPath(), dirName}, "/"), + Name: dirName, + Size: 0, + IsFolder: true, + }, nil +} + +func (c *Lark) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcToken, ok := c.getObjToken(ctx, srcObj.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + dstDirToken, ok := c.getObjToken(ctx, dstDir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + req := larkdrive.NewMoveFileReqBuilder(). + Body(larkdrive.NewMoveFileReqBodyBuilder(). + Type("file"). + FolderToken(dstDirToken). + Build()).FileToken(srcToken). + Build() + + // 发起请求 + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.MoveFileResp, error) { + return c.client.Drive.File.Move(ctx, req, opts...) + }) + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + return nil, nil +} + +func (c *Lark) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + // TODO rename obj, optional + return nil, errs.NotImplement +} + +func (c *Lark) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + srcToken, ok := c.getObjToken(ctx, srcObj.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + dstDirToken, ok := c.getObjToken(ctx, dstDir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + req := larkdrive.NewCopyFileReqBuilder(). + Body(larkdrive.NewCopyFileReqBodyBuilder(). + Name(srcObj.GetName()). + Type("file"). + FolderToken(dstDirToken). + Build()).FileToken(srcToken). + Build() + + // 发起请求 + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CopyFileResp, error) { + return c.client.Drive.File.Copy(ctx, req, opts...) + }) + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + return nil, nil +} + +func (c *Lark) Remove(ctx context.Context, obj model.Obj) error { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return errs.ObjectNotFound + } + + req := larkdrive.NewDeleteFileReqBuilder(). + FileToken(token). + Type("file"). + Build() + + // 发起请求 + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.DeleteFileResp, error) { + return c.client.Drive.File.Delete(ctx, req, opts...) + }) + if err != nil { + return err + } + + if !resp.Success() { + return errors.New(resp.Error()) + } + + return nil +} + +var uploadLimit = rate.NewLimiter(rate.Every(time.Second), 5) + +func (c *Lark) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + token, ok := c.getObjToken(ctx, dstDir.GetPath()) + if !ok { + return nil, errs.ObjectNotFound + } + + // prepare + req := larkdrive.NewUploadPrepareFileReqBuilder(). + FileUploadInfo(larkdrive.NewFileUploadInfoBuilder(). + FileName(stream.GetName()). + ParentType(`explorer`). + ParentNode(token). + Size(int(stream.GetSize())). + Build()). + Build() + + // 发起请求 + err := uploadLimit.Wait(ctx) + if err != nil { + return nil, err + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadPrepareFileResp, error) { + return c.client.Drive.File.UploadPrepare(ctx, req, opts...) + }) + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + uploadId := *resp.Data.UploadId + blockSize := *resp.Data.BlockSize + blockCount := *resp.Data.BlockNum + + // upload + for i := 0; i < blockCount; i++ { + length := int64(blockSize) + if i == blockCount-1 { + length = stream.GetSize() - int64(i*blockSize) + } + + reader := driver.NewLimitedUploadStream(ctx, io.LimitReader(stream, length)) + + req := larkdrive.NewUploadPartFileReqBuilder(). + Body(larkdrive.NewUploadPartFileReqBodyBuilder(). + UploadId(uploadId). + Seq(i). + Size(int(length)). + File(reader). + Build()). + Build() + + // 发起请求 + err = uploadLimit.Wait(ctx) + if err != nil { + return nil, err + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadPartFileResp, error) { + return c.client.Drive.File.UploadPart(ctx, req, opts...) + }) + + if err != nil { + return nil, err + } + + if !resp.Success() { + return nil, errors.New(resp.Error()) + } + + up(float64(i) / float64(blockCount)) + } + + //close + closeReq := larkdrive.NewUploadFinishFileReqBuilder(). + Body(larkdrive.NewUploadFinishFileReqBodyBuilder(). + UploadId(uploadId). + BlockNum(blockCount). + Build()). + Build() + + // 发起请求 + closeResp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.UploadFinishFileResp, error) { + return c.client.Drive.File.UploadFinish(ctx, closeReq, opts...) + }) + if err != nil { + return nil, err + } + + if !closeResp.Success() { + return nil, errors.New(closeResp.Error()) + } + + return &model.Object{ + ID: *closeResp.Data.FileToken, + }, nil +} + +//func (d *Lark) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Lark)(nil) diff --git a/drivers/lark/meta.go b/drivers/lark/meta.go new file mode 100644 index 00000000000..fe6149d99a4 --- /dev/null +++ b/drivers/lark/meta.go @@ -0,0 +1,40 @@ +package lark + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // define other + AppId string `json:"app_id" type:"text" help:"app id"` + AppSecret string `json:"app_secret" type:"text" help:"app secret"` + UserAccessToken string `json:"user_access_token" type:"text" help:"optional cached user access token for personal drive access"` + RefreshToken string `json:"refresh_token" type:"text" help:"optional refresh token for user access token auto refresh"` + UserAccessTokenExpiresAt int64 `json:"user_access_token_expires_at" type:"number" help:"user access token expires at unix timestamp"` + RefreshTokenExpiresAt int64 `json:"refresh_token_expires_at" type:"number" help:"refresh token expires at unix timestamp"` + ExternalMode bool `json:"external_mode" type:"bool" help:"external mode"` + TenantUrlPrefix string `json:"tenant_url_prefix" type:"text" help:"tenant url prefix"` +} + +var config = driver.Config{ + Name: "Lark", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Lark{} + }) +} diff --git a/drivers/lark/other.go b/drivers/lark/other.go new file mode 100644 index 00000000000..f0217b72621 --- /dev/null +++ b/drivers/lark/other.go @@ -0,0 +1,433 @@ +package lark + +import ( + "context" + "encoding/json" + "fmt" + "io" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + larkcore "github.com/larksuite/oapi-sdk-go/v3/core" + larkbitable "github.com/larksuite/oapi-sdk-go/v3/service/bitable/v1" + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" + larksheets "github.com/larksuite/oapi-sdk-go/v3/service/sheets/v3" + "github.com/pkg/errors" +) + +const ( + larkExportOptionsMethod = "lark_export_options" + larkExportCreateMethod = "lark_export_create" + larkExportStatusMethod = "lark_export_status" + + larkExportStatusPending = "pending" + larkExportStatusProcessing = "processing" + larkExportStatusSuccess = "success" + larkExportStatusFailed = "failed" +) + +type larkExportCreateReq struct { + Format string `json:"format"` + SubID string `json:"sub_id"` +} + +type larkExportStatusReq struct { + Ticket string `json:"ticket"` +} + +type LarkExportCreateResp struct { + Ticket string `json:"ticket"` + Token string `json:"token"` + Type string `json:"type"` + Format string `json:"format"` + SubID string `json:"sub_id,omitempty"` +} + +type LarkExportOption struct { + Value string `json:"value"` + Label string `json:"label"` + RequiresSubID bool `json:"requires_sub_id"` +} + +type LarkExportSubResource struct { + ID string `json:"id"` + Name string `json:"name"` + Type string `json:"type"` +} + +type LarkExportOptionsResp struct { + Type string `json:"type"` + Formats []LarkExportOption `json:"formats"` + SubResources []LarkExportSubResource `json:"sub_resources,omitempty"` + SubResourceError string `json:"sub_resource_error,omitempty"` +} + +type LarkExportStatusResp struct { + Status string `json:"status"` + FileToken string `json:"file_token,omitempty"` + FileSize int `json:"file_size,omitempty"` + JobStatus int `json:"job_status,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ErrorDetail string `json:"error_detail,omitempty"` +} + +type larkAPIErrorResp interface { + Error() string + ErrorResp() string +} + +func (c *Lark) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch strings.ToLower(strings.TrimSpace(args.Method)) { + case larkExportOptionsMethod: + return c.getExportOptions(ctx, args.Obj) + case larkExportCreateMethod: + var req larkExportCreateReq + if err := decodeOtherData(args.Data, &req); err != nil { + return nil, err + } + return c.createExportTask(ctx, args.Obj, req) + case larkExportStatusMethod: + var req larkExportStatusReq + if err := decodeOtherData(args.Data, &req); err != nil { + return nil, err + } + return c.getExportTask(ctx, args.Obj, req) + default: + return nil, fmt.Errorf("unsupported lark method: %s", args.Method) + } +} + +func decodeOtherData(data interface{}, v interface{}) error { + b, err := json.Marshal(data) + if err != nil { + return errors.WithMessage(err, "failed to encode request data") + } + if err = json.Unmarshal(b, v); err != nil { + return errors.WithMessage(err, "failed to decode request data") + } + return nil +} + +func (c *Lark) createExportTask(ctx context.Context, obj model.Obj, req larkExportCreateReq) (*LarkExportCreateResp, error) { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + docType, err := larkExportType(obj.GetName()) + if err != nil { + return nil, err + } + format := strings.ToLower(strings.TrimSpace(req.Format)) + if !larkExportFormatAllowed(docType, format) { + return nil, fmt.Errorf("unsupported export format %q for lark type %q", format, docType) + } + subID := strings.TrimSpace(req.SubID) + if larkExportFormatRequiresSubID(docType, format) && subID == "" { + return nil, fmt.Errorf("sub_id is required when exporting lark type %q as %q", docType, format) + } + + builder := larkdrive.NewExportTaskBuilder(). + Token(token). + Type(docType). + FileExtension(format). + FileName(larkExportBaseName(obj.GetName())) + if subID != "" { + builder.SubId(subID) + } + exportTask := builder.Build() + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.CreateExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Create(ctx, + larkdrive.NewCreateExportTaskReqBuilder().ExportTask(exportTask).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil || resp.Data.Ticket == nil || *resp.Data.Ticket == "" { + return nil, errors.New("lark export task response missing ticket") + } + return &LarkExportCreateResp{ + Ticket: *resp.Data.Ticket, + Token: token, + Type: docType, + Format: format, + SubID: subID, + }, nil +} + +func (c *Lark) getExportOptions(ctx context.Context, obj model.Obj) (*LarkExportOptionsResp, error) { + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + docType, err := larkExportType(obj.GetName()) + if err != nil { + return nil, err + } + out := &LarkExportOptionsResp{ + Type: docType, + Formats: larkExportOptions(docType), + } + switch docType { + case larkdrive.TypeSheet: + out.SubResources, err = c.listSheetSubResources(ctx, token) + case larkdrive.TypeBitable: + out.SubResources, err = c.listBitableSubResources(ctx, token) + } + if err != nil { + out.SubResourceError = err.Error() + } + return out, nil +} + +func (c *Lark) getExportTask(ctx context.Context, obj model.Obj, req larkExportStatusReq) (*LarkExportStatusResp, error) { + ticket := strings.TrimSpace(req.Ticket) + if ticket == "" { + return nil, errors.New("ticket is required") + } + token, ok := c.getObjToken(ctx, obj.GetPath()) + if !ok { + return nil, errors.WithStack(errors.New("lark file token not found")) + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.GetExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Get(ctx, + larkdrive.NewGetExportTaskReqBuilder().Ticket(ticket).Token(token).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil || resp.Data.Result == nil { + return nil, errors.New("lark export task response missing result") + } + result := resp.Data.Result + out := &LarkExportStatusResp{ + Status: larkExportJobStatus(result), + } + if result.FileToken != nil { + out.FileToken = *result.FileToken + } + if result.FileSize != nil { + out.FileSize = *result.FileSize + } + if result.JobStatus != nil { + out.JobStatus = *result.JobStatus + } + if out.Status == larkExportStatusFailed { + out.ErrorMessage = larkExportTaskErrorMessage(result) + out.ErrorDetail = larkExportTaskErrorDetail(result) + } else if result.JobErrorMsg != nil { + out.ErrorMessage = strings.TrimSpace(*result.JobErrorMsg) + } + return out, nil +} + +func (c *Lark) DownloadExportFile(ctx context.Context, fileToken string) (io.Reader, string, error) { + fileToken = strings.TrimSpace(fileToken) + if fileToken == "" { + return nil, "", errors.New("file_token is required") + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkdrive.DownloadExportTaskResp, error) { + return c.client.Drive.V1.ExportTask.Download(ctx, + larkdrive.NewDownloadExportTaskReqBuilder().FileToken(fileToken).Build(), opts...) + }) + if err != nil { + return nil, "", err + } + if !resp.Success() { + return nil, "", larkAPIError(resp) + } + if resp.File == nil { + return nil, "", errors.New("lark export download response missing file") + } + return resp.File, resp.FileName, nil +} + +func larkExportType(name string) (string, error) { + switch { + case strings.HasSuffix(name, ".lark-doc"): + return larkdrive.TypeDoc, nil + case strings.HasSuffix(name, ".lark-docx"): + return larkdrive.TypeDocx, nil + case strings.HasSuffix(name, ".lark-sheet"): + return larkdrive.TypeSheet, nil + case strings.HasSuffix(name, ".lark-bitable"): + return larkdrive.TypeBitable, nil + default: + return "", fmt.Errorf("unsupported lark export file type: %s", name) + } +} + +func larkExportFormatAllowed(docType, format string) bool { + switch docType { + case larkdrive.TypeDoc, larkdrive.TypeDocx: + return format == larkdrive.FileExtensionPdf || format == larkdrive.FileExtensionDocx + case larkdrive.TypeSheet, larkdrive.TypeBitable: + return format == larkdrive.FileExtensionXlsx || format == larkdrive.FileExtensionCsv + default: + return false + } +} + +func larkExportFormatRequiresSubID(docType, format string) bool { + return (docType == larkdrive.TypeSheet || docType == larkdrive.TypeBitable) && format == larkdrive.FileExtensionCsv +} + +func larkExportOptions(docType string) []LarkExportOption { + switch docType { + case larkdrive.TypeDoc, larkdrive.TypeDocx: + return []LarkExportOption{ + {Value: larkdrive.FileExtensionPdf, Label: "PDF"}, + {Value: larkdrive.FileExtensionDocx, Label: "DOCX"}, + } + case larkdrive.TypeSheet, larkdrive.TypeBitable: + return []LarkExportOption{ + {Value: larkdrive.FileExtensionXlsx, Label: "XLSX"}, + {Value: larkdrive.FileExtensionCsv, Label: "CSV", RequiresSubID: true}, + } + default: + return nil + } +} + +func larkExportBaseName(name string) string { + return trimLarkDisplayExt(name) +} + +func larkExportJobStatus(result *larkdrive.ExportTask) string { + if result == nil { + return larkExportStatusProcessing + } + if result.FileToken != nil && *result.FileToken != "" { + return larkExportStatusSuccess + } + if result.JobStatus != nil && *result.JobStatus == 2 { + return larkExportStatusFailed + } + if result.JobErrorMsg != nil && *result.JobErrorMsg != "" && !strings.EqualFold(*result.JobErrorMsg, "success") { + return larkExportStatusFailed + } + return larkExportStatusProcessing +} + +func larkAPIError(resp larkAPIErrorResp) error { + msg := strings.TrimSpace(resp.ErrorResp()) + if msg == "" || msg == "{}" || msg == "null" { + msg = strings.TrimSpace(resp.Error()) + } + return errors.New(msg) +} + +func larkExportTaskErrorMessage(result *larkdrive.ExportTask) string { + if result == nil { + return "" + } + if result.JobErrorMsg != nil { + msg := strings.TrimSpace(*result.JobErrorMsg) + if msg != "" && !strings.EqualFold(msg, "success") { + return msg + } + } + if result.JobStatus != nil { + return fmt.Sprintf("job_status=%d", *result.JobStatus) + } + return "" +} + +func larkExportTaskErrorDetail(result *larkdrive.ExportTask) string { + if result == nil { + return "" + } + detail := map[string]interface{}{} + if result.JobStatus != nil { + detail["job_status"] = *result.JobStatus + } + if result.JobErrorMsg != nil { + detail["job_error_msg"] = strings.TrimSpace(*result.JobErrorMsg) + } + if len(detail) == 0 { + return "" + } + b, err := json.Marshal(detail) + if err != nil { + return "" + } + return string(b) +} + +func (c *Lark) listSheetSubResources(ctx context.Context, token string) ([]LarkExportSubResource, error) { + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larksheets.QuerySpreadsheetSheetResp, error) { + return c.client.Sheets.SpreadsheetSheet.Query(ctx, + larksheets.NewQuerySpreadsheetSheetReqBuilder().SpreadsheetToken(token).Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil { + return nil, nil + } + var res []LarkExportSubResource + for _, sheet := range resp.Data.Sheets { + if sheet == nil || sheet.SheetId == nil || strings.TrimSpace(*sheet.SheetId) == "" { + continue + } + name := strings.TrimSpace(larkString(sheet.Title)) + if name == "" { + name = *sheet.SheetId + } + res = append(res, LarkExportSubResource{ + ID: *sheet.SheetId, + Name: name, + Type: strings.TrimSpace(larkString(sheet.ResourceType)), + }) + } + return res, nil +} + +func (c *Lark) listBitableSubResources(ctx context.Context, token string) ([]LarkExportSubResource, error) { + var res []LarkExportSubResource + pageToken := "" + for { + builder := larkbitable.NewListAppTableReqBuilder().AppToken(token).PageSize(100) + if pageToken != "" { + builder.PageToken(pageToken) + } + resp, err := doDrive(ctx, c, func(opts ...larkcore.RequestOptionFunc) (*larkbitable.ListAppTableResp, error) { + return c.client.Bitable.AppTable.List(ctx, builder.Build(), opts...) + }) + if err != nil { + return nil, err + } + if !resp.Success() { + return nil, larkAPIError(resp) + } + if resp.Data == nil { + return res, nil + } + for _, table := range resp.Data.Items { + if table == nil || table.TableId == nil || strings.TrimSpace(*table.TableId) == "" { + continue + } + name := strings.TrimSpace(larkString(table.Name)) + if name == "" { + name = *table.TableId + } + res = append(res, LarkExportSubResource{ + ID: *table.TableId, + Name: name, + Type: "table", + }) + } + if resp.Data.HasMore == nil || !*resp.Data.HasMore || resp.Data.PageToken == nil || *resp.Data.PageToken == "" { + return res, nil + } + pageToken = *resp.Data.PageToken + } +} diff --git a/drivers/lark/other_test.go b/drivers/lark/other_test.go new file mode 100644 index 00000000000..6887c6bcd65 --- /dev/null +++ b/drivers/lark/other_test.go @@ -0,0 +1,176 @@ +package lark + +import ( + "testing" + + larkdrive "github.com/larksuite/oapi-sdk-go/v3/service/drive/v1" +) + +func TestLarkExportType(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + }{ + {name: "doc.lark-doc", want: larkdrive.TypeDoc}, + {name: "doc.lark-docx", want: larkdrive.TypeDocx}, + {name: "sheet.lark-sheet", want: larkdrive.TypeSheet}, + {name: "base.lark-bitable", want: larkdrive.TypeBitable}, + {name: "file.pdf", wantErr: true}, + } + for _, tt := range tests { + got, err := larkExportType(tt.name) + if tt.wantErr { + if err == nil { + t.Fatalf("larkExportType(%q) expected error", tt.name) + } + continue + } + if err != nil { + t.Fatalf("larkExportType(%q) unexpected error: %v", tt.name, err) + } + if got != tt.want { + t.Fatalf("larkExportType(%q) = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportFormatAllowed(t *testing.T) { + tests := []struct { + docType string + format string + want bool + }{ + {docType: larkdrive.TypeDoc, format: larkdrive.FileExtensionPdf, want: true}, + {docType: larkdrive.TypeDocx, format: larkdrive.FileExtensionDocx, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionXlsx, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionXlsx, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionPdf, want: false}, + } + for _, tt := range tests { + if got := larkExportFormatAllowed(tt.docType, tt.format); got != tt.want { + t.Fatalf("larkExportFormatAllowed(%q, %q) = %v, want %v", tt.docType, tt.format, got, tt.want) + } + } +} + +func TestLarkExportFormatRequiresSubID(t *testing.T) { + tests := []struct { + docType string + format string + want bool + }{ + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeBitable, format: larkdrive.FileExtensionCsv, want: true}, + {docType: larkdrive.TypeSheet, format: larkdrive.FileExtensionXlsx, want: false}, + {docType: larkdrive.TypeDocx, format: larkdrive.FileExtensionDocx, want: false}, + } + for _, tt := range tests { + if got := larkExportFormatRequiresSubID(tt.docType, tt.format); got != tt.want { + t.Fatalf("larkExportFormatRequiresSubID(%q, %q) = %v, want %v", tt.docType, tt.format, got, tt.want) + } + } +} + +func TestLarkExportOptions(t *testing.T) { + tests := []struct { + docType string + want []LarkExportOption + }{ + {docType: larkdrive.TypeDocx, want: []LarkExportOption{ + {Value: larkdrive.FileExtensionPdf, Label: "PDF"}, + {Value: larkdrive.FileExtensionDocx, Label: "DOCX"}, + }}, + {docType: larkdrive.TypeSheet, want: []LarkExportOption{ + {Value: larkdrive.FileExtensionXlsx, Label: "XLSX"}, + {Value: larkdrive.FileExtensionCsv, Label: "CSV", RequiresSubID: true}, + }}, + } + for _, tt := range tests { + got := larkExportOptions(tt.docType) + if len(got) != len(tt.want) { + t.Fatalf("larkExportOptions(%q) len = %d, want %d", tt.docType, len(got), len(tt.want)) + } + for i := range got { + if got[i] != tt.want[i] { + t.Fatalf("larkExportOptions(%q)[%d] = %+v, want %+v", tt.docType, i, got[i], tt.want[i]) + } + } + } +} + +func TestLarkExportBaseName(t *testing.T) { + tests := map[string]string{ + "weekly.lark-docx": "weekly", + "weekly.report.lark-doc": "weekly.report", + "table.lark-sheet": "table", + "plain.pdf": "plain.pdf", + } + for name, want := range tests { + if got := larkExportBaseName(name); got != want { + t.Fatalf("larkExportBaseName(%q) = %q, want %q", name, got, want) + } + } +} + +func TestLarkExportJobStatus(t *testing.T) { + successToken := "box_token" + successCode := 0 + failCode := 2 + failMsg := "failed" + successMsg := "success" + + tests := []struct { + name string + result *larkdrive.ExportTask + want string + }{ + {name: "nil", result: nil, want: larkExportStatusProcessing}, + {name: "file token wins", result: &larkdrive.ExportTask{FileToken: &successToken, JobStatus: &failCode}, want: larkExportStatusSuccess}, + {name: "status failure", result: &larkdrive.ExportTask{JobStatus: &failCode}, want: larkExportStatusFailed}, + {name: "error message failure", result: &larkdrive.ExportTask{JobStatus: &successCode, JobErrorMsg: &failMsg}, want: larkExportStatusFailed}, + {name: "success message without token still processing", result: &larkdrive.ExportTask{JobStatus: &successCode, JobErrorMsg: &successMsg}, want: larkExportStatusProcessing}, + } + for _, tt := range tests { + if got := larkExportJobStatus(tt.result); got != tt.want { + t.Fatalf("%s: larkExportJobStatus() = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportTaskErrorMessage(t *testing.T) { + failCode := 2 + failMsg := "source document cannot be exported" + successMsg := "success" + + tests := []struct { + name string + result *larkdrive.ExportTask + want string + }{ + {name: "nil", result: nil, want: ""}, + {name: "job error message", result: &larkdrive.ExportTask{JobStatus: &failCode, JobErrorMsg: &failMsg}, want: failMsg}, + {name: "status fallback", result: &larkdrive.ExportTask{JobStatus: &failCode, JobErrorMsg: &successMsg}, want: "job_status=2"}, + } + for _, tt := range tests { + if got := larkExportTaskErrorMessage(tt.result); got != tt.want { + t.Fatalf("%s: larkExportTaskErrorMessage() = %q, want %q", tt.name, got, tt.want) + } + } +} + +func TestLarkExportTaskErrorDetail(t *testing.T) { + failCode := 2 + failMsg := "source document cannot be exported" + + got := larkExportTaskErrorDetail(&larkdrive.ExportTask{ + JobStatus: &failCode, + JobErrorMsg: &failMsg, + }) + want := `{"job_error_msg":"source document cannot be exported","job_status":2}` + if got != want { + t.Fatalf("larkExportTaskErrorDetail() = %q, want %q", got, want) + } +} diff --git a/drivers/lark/types.go b/drivers/lark/types.go new file mode 100644 index 00000000000..3ebefd556dc --- /dev/null +++ b/drivers/lark/types.go @@ -0,0 +1,32 @@ +package lark + +import ( + "context" + "github.com/Xhofe/go-cache" + "time" +) + +type TokenCache struct { + cache.ICache[string] +} + +func (t *TokenCache) Set(_ context.Context, key string, value string, expireTime time.Duration) error { + t.ICache.Set(key, value, cache.WithEx[string](expireTime)) + + return nil +} + +func (t *TokenCache) Get(_ context.Context, key string) (string, error) { + v, ok := t.ICache.Get(key) + if ok { + return v, nil + } + + return "", nil +} + +func newTokenCache() *TokenCache { + c := cache.NewMemCache[string]() + + return &TokenCache{c} +} diff --git a/drivers/lark/util.go b/drivers/lark/util.go new file mode 100644 index 00000000000..ebef4300c49 --- /dev/null +++ b/drivers/lark/util.go @@ -0,0 +1,82 @@ +package lark + +import ( + "context" + "github.com/Xhofe/go-cache" + log "github.com/sirupsen/logrus" + "path" + "strings" + "time" +) + +const objTokenCacheDuration = 5 * time.Minute +const emptyFolderToken = "empty" + +var objTokenCache = cache.NewMemCache[string]() +var exOpts = cache.WithEx[string](objTokenCacheDuration) + +func (c *Lark) getObjToken(ctx context.Context, folderPath string) (string, bool) { + if token, ok := objTokenCache.Get(folderPath); ok { + return token, true + } + + dir, name := path.Split(folderPath) + // strip the last slash of dir if it exists + if len(dir) > 0 && dir[len(dir)-1] == '/' { + dir = dir[:len(dir)-1] + } + if name == "" { + return c.rootFolderToken, true + } + + var parentToken string + var found bool + parentToken, found = c.getObjToken(ctx, dir) + if !found { + return emptyFolderToken, false + } + + files, err := c.listFiles(ctx, parentToken) + if err != nil { + log.WithError(err).Error("failed to list files") + return emptyFolderToken, false + } + + for _, file := range files { + if *file.Name == name || *file.Name == trimLarkDisplayExt(name) { + objTokenCache.Set(folderPath, *file.Token, exOpts) + return *file.Token, true + } + } + + return emptyFolderToken, false +} + +func trimLarkDisplayExt(name string) string { + for _, suffix := range larkCloudDocSuffixes() { + if strings.HasSuffix(name, suffix) { + return strings.TrimSuffix(name, suffix) + } + } + return name +} + +func isLarkCloudDocName(name string) bool { + for _, suffix := range larkCloudDocSuffixes() { + if strings.HasSuffix(name, suffix) { + return true + } + } + return false +} + +func larkCloudDocSuffixes() []string { + return []string{ + ".lark-doc", + ".lark-docx", + ".lark-sheet", + ".lark-bitable", + ".lark-mindnote", + ".lark-slides", + } +} diff --git a/drivers/lenovonas_share/driver.go b/drivers/lenovonas_share/driver.go new file mode 100644 index 00000000000..684a2dda647 --- /dev/null +++ b/drivers/lenovonas_share/driver.go @@ -0,0 +1,139 @@ +package LenovoNasShare + +import ( + "context" + "net/http" + "time" + + "github.com/go-resty/resty/v2" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type LenovoNasShare struct { + model.Storage + Addition + stoken string + expireAt int64 +} + +func (d *LenovoNasShare) Config() driver.Config { + return config +} + +func (d *LenovoNasShare) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *LenovoNasShare) Init(ctx context.Context) error { + if err := d.getStoken(); err != nil { + return err + } + return nil +} + +func (d *LenovoNasShare) Drop(ctx context.Context) error { + return nil +} + +func (d *LenovoNasShare) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + d.checkStoken() // 检查stoken是否过期 + files := make([]File, 0) + + var resp Files + query := map[string]string{ + "code": d.ShareId, + "num": "5000", + "stoken": d.stoken, + "path": dir.GetPath(), + } + _, err := d.request(d.Host+"/oneproxy/api/share/v1/files", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, &resp) + if err != nil { + return nil, err + } + files = append(files, resp.Data.List...) + + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return src, nil + }) +} + +func (d *LenovoNasShare) checkStoken() { // 检查stoken是否过期 + if d.expireAt < time.Now().Unix() { + d.getStoken() + } +} + +func (d *LenovoNasShare) getStoken() error { // 获取stoken + if d.Host == "" { + d.Host = "https://siot-share.lenovo.com.cn" + } + query := map[string]string{ + "code": d.ShareId, + "password": d.SharePwd, + } + resp, err := d.request(d.Host+"/oneproxy/api/share/v1/access", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, nil) + if err != nil { + return err + } + d.stoken = utils.Json.Get(resp, "data", "stoken").ToString() + d.expireAt = utils.Json.Get(resp, "data", "expires_in").ToInt64() + time.Now().Unix() - 60 + return nil +} + +func (d *LenovoNasShare) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + d.checkStoken() // 检查stoken是否过期 + query := map[string]string{ + "code": d.ShareId, + "stoken": d.stoken, + "path": file.GetPath(), + } + resp, err := d.request(d.Host+"/oneproxy/api/share/v1/file/link", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(query) + }, nil) + if err != nil { + return nil, err + } + downloadUrl := d.Host + "/oneproxy/api/share/v1/file/download?code=" + d.ShareId + "&dtoken=" + utils.Json.Get(resp, "data", "param", "dtoken").ToString() + + link := model.Link{ + URL: downloadUrl, + Header: http.Header{ + "Referer": []string{"https://siot-share.lenovo.com.cn"}, + }, + } + return &link, nil +} + +func (d *LenovoNasShare) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *LenovoNasShare) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *LenovoNasShare) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *LenovoNasShare) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *LenovoNasShare) Remove(ctx context.Context, obj model.Obj) error { + return errs.NotImplement +} + +func (d *LenovoNasShare) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return nil, errs.NotImplement +} + +var _ driver.Driver = (*LenovoNasShare)(nil) diff --git a/drivers/lenovonas_share/meta.go b/drivers/lenovonas_share/meta.go new file mode 100644 index 00000000000..0bf80555739 --- /dev/null +++ b/drivers/lenovonas_share/meta.go @@ -0,0 +1,33 @@ +package LenovoNasShare + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + ShareId string `json:"share_id" required:"true" help:"The part after the last / in the shared link"` + SharePwd string `json:"share_pwd" required:"true" help:"The password of the shared link"` + Host string `json:"host" required:"true" default:"https://siot-share.lenovo.com.cn" help:"You can change it to your local area network"` +} + +var config = driver.Config{ + Name: "LenovoNasShare", + LocalSort: true, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: true, + NeedMs: false, + DefaultRoot: "", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &LenovoNasShare{} + }) +} diff --git a/drivers/lenovonas_share/types.go b/drivers/lenovonas_share/types.go new file mode 100644 index 00000000000..37ff1465524 --- /dev/null +++ b/drivers/lenovonas_share/types.go @@ -0,0 +1,82 @@ +package LenovoNasShare + +import ( + "encoding/json" + "time" + + "github.com/alist-org/alist/v3/pkg/utils" + + _ "github.com/alist-org/alist/v3/internal/model" +) + +func (f *File) UnmarshalJSON(data []byte) error { + type Alias File + aux := &struct { + CreateAt int64 `json:"time"` + UpdateAt int64 `json:"chtime"` + *Alias + }{ + Alias: (*Alias)(f), + } + + if err := json.Unmarshal(data, aux); err != nil { + return err + } + + f.CreateAt = time.Unix(aux.CreateAt, 0) + f.UpdateAt = time.Unix(aux.UpdateAt, 0) + + return nil +} + +type File struct { + FileName string `json:"name"` + Size int64 `json:"size"` + CreateAt time.Time `json:"time"` + UpdateAt time.Time `json:"chtime"` + Path string `json:"path"` + Type string `json:"type"` +} + +func (f File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f File) GetPath() string { + return f.Path +} + +func (f File) GetSize() int64 { + if f.IsDir() { + return 0 + } else { + return f.Size + } +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) ModTime() time.Time { + return f.UpdateAt +} + +func (f File) CreateTime() time.Time { + return f.CreateAt +} + +func (f File) IsDir() bool { + return f.Type == "dir" +} + +func (f File) GetID() string { + return f.GetPath() +} + +type Files struct { + Data struct { + List []File `json:"list"` + HasMore bool `json:"has_more"` + } `json:"data"` +} diff --git a/drivers/lenovonas_share/util.go b/drivers/lenovonas_share/util.go new file mode 100644 index 00000000000..ccf3af042a4 --- /dev/null +++ b/drivers/lenovonas_share/util.go @@ -0,0 +1,36 @@ +package LenovoNasShare + +import ( + "errors" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + jsoniter "github.com/json-iterator/go" +) + +func (d *LenovoNasShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "origin": "https://siot-share.lenovo.com.cn", + "referer": "https://siot-share.lenovo.com.cn/", + "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) alist-client", + "platform": "web", + "app-version": "3", + }) + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + body := res.Body() + result := utils.Json.Get(body, "result").ToBool() + if !result { + return nil, errors.New(jsoniter.Get(body, "error", "msg").ToString()) + } + return body, nil +} diff --git a/drivers/local/driver.go b/drivers/local/driver.go index 04604366f65..ef7ce6b4e2a 100644 --- a/drivers/local/driver.go +++ b/drivers/local/driver.go @@ -18,10 +18,12 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" - "github.com/djherbis/times" + "github.com/alist-org/times" + cp "github.com/otiai10/copy" log "github.com/sirupsen/logrus" _ "golang.org/x/image/webp" ) @@ -30,6 +32,19 @@ type Local struct { model.Storage Addition mkdirPerm int32 + thumbSize int + + // zero means no limit + thumbConcurrency int + thumbTokenBucket TokenBucket + + // video thumb position + videoThumbPos float64 + videoThumbPosIsPercentage bool + thumbPixel int + + // use ffmpeg + useFFmpeg bool } func (d *Local) Config() driver.Config { @@ -56,12 +71,72 @@ func (d *Local) Init(ctx context.Context) error { } d.Addition.RootFolderPath = abs } + + d.useFFmpeg = d.UseFFmpeg + if d.ThumbCacheFolder != "" && !utils.Exists(d.ThumbCacheFolder) { err := os.MkdirAll(d.ThumbCacheFolder, os.FileMode(d.mkdirPerm)) if err != nil { return err } } + d.thumbSize = 144 + if item, err := op.GetSettingItemByKey(conf.ThumbnailSize); err == nil && item != nil && strings.TrimSpace(item.Value) != "" { + v, err := strconv.ParseUint(item.Value, 10, 32) + if err != nil { + return fmt.Errorf("invalid setting %s value: %s, err: %s", conf.ThumbnailSize, item.Value, err) + } + if v == 0 { + return fmt.Errorf("invalid setting %s value: %s, the value must be a positive integer", conf.ThumbnailSize, item.Value) + } + d.thumbSize = int(v) + } + if d.ThumbConcurrency != "" { + v, err := strconv.ParseUint(d.ThumbConcurrency, 10, 32) + if err != nil { + return err + } + d.thumbConcurrency = int(v) + } + if d.ThumbPixel != "" { + v, err := strconv.ParseUint(d.ThumbPixel, 10, 32) + if err != nil { + return err + } + d.thumbPixel = int(v) + } + + if d.thumbConcurrency == 0 { + d.thumbTokenBucket = NewNopTokenBucket() + } else { + d.thumbTokenBucket = NewStaticTokenBucketWithMigration(d.thumbTokenBucket, d.thumbConcurrency) + } + // Check the VideoThumbPos value + if d.VideoThumbPos == "" { + d.VideoThumbPos = "20%" + } + if strings.HasSuffix(d.VideoThumbPos, "%") { + percentage := strings.TrimSuffix(d.VideoThumbPos, "%") + val, err := strconv.ParseFloat(percentage, 64) + if err != nil { + return fmt.Errorf("invalid video_thumb_pos value: %s, err: %s", d.VideoThumbPos, err) + } + if val < 0 || val > 100 { + return fmt.Errorf("invalid video_thumb_pos value: %s, the precentage must be a number between 0 and 100", d.VideoThumbPos) + } + d.videoThumbPosIsPercentage = true + d.videoThumbPos = val / 100 + } else { + val, err := strconv.ParseFloat(d.VideoThumbPos, 64) + if err != nil { + return fmt.Errorf("invalid video_thumb_pos value: %s, err: %s", d.VideoThumbPos, err) + } + if val < 0 { + return fmt.Errorf("invalid video_thumb_pos value: %s, the time must be a positive number", d.VideoThumbPos) + } + d.videoThumbPosIsPercentage = false + d.videoThumbPos = val + } return nil } @@ -84,28 +159,29 @@ func (d *Local) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([ if !d.ShowHidden && strings.HasPrefix(f.Name(), ".") { continue } - file := d.FileInfoToObj(f, args.ReqPath, fullPath) + file := d.FileInfoToObj(ctx, f, args.ReqPath, fullPath) files = append(files, file) } return files, nil } -func (d *Local) FileInfoToObj(f fs.FileInfo, reqPath string, fullPath string) model.Obj { +func (d *Local) FileInfoToObj(ctx context.Context, f fs.FileInfo, reqPath string, fullPath string) model.Obj { thumb := "" if d.Thumbnail { typeName := utils.GetFileType(f.Name()) if typeName == conf.IMAGE || typeName == conf.VIDEO { - thumb = common.GetApiUrl(nil) + stdpath.Join("/d", reqPath, f.Name()) + thumb = common.GetApiUrl(common.GetHttpReq(ctx)) + stdpath.Join("/d", reqPath, f.Name()) thumb = utils.EncodePath(thumb, true) thumb += "?type=thumb&sign=" + sign.Sign(stdpath.Join(reqPath, f.Name())) } } - isFolder := f.IsDir() || isSymlinkDir(f, fullPath) + filePath := filepath.Join(fullPath, f.Name()) + isFolder := f.IsDir() || isLinkedDir(f, filePath) var size int64 if !isFolder { size = f.Size() } var ctime time.Time - t, err := times.Stat(stdpath.Join(fullPath, f.Name())) + t, err := times.Stat(filePath) if err == nil { if t.HasBirthTime() { ctime = t.BirthTime() @@ -114,7 +190,7 @@ func (d *Local) FileInfoToObj(f fs.FileInfo, reqPath string, fullPath string) mo file := model.ObjThumb{ Object: model.Object{ - Path: filepath.Join(fullPath, f.Name()), + Path: filePath, Name: f.Name(), Modified: f.ModTime(), Size: size, @@ -126,14 +202,13 @@ func (d *Local) FileInfoToObj(f fs.FileInfo, reqPath string, fullPath string) mo }, } return &file - } func (d *Local) GetMeta(ctx context.Context, path string) (model.Obj, error) { f, err := os.Stat(path) if err != nil { return nil, err } - file := d.FileInfoToObj(f, path, path) + file := d.FileInfoToObj(ctx, f, path, path) //h := "123123" //if s, ok := f.(model.SetHash); ok && file.GetHash() == ("","") { // s.SetHash(h,"SHA1") @@ -151,7 +226,7 @@ func (d *Local) Get(ctx context.Context, path string) (model.Obj, error) { } return nil, err } - isFolder := f.IsDir() || isSymlinkDir(f, path) + isFolder := f.IsDir() || isLinkedDir(f, path) size := f.Size() if isFolder { size = 0 @@ -178,7 +253,13 @@ func (d *Local) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( fullPath := file.GetPath() var link model.Link if args.Type == "thumb" && utils.Ext(file.GetName()) != "svg" { - buf, thumbPath, err := d.getThumb(file) + var buf *bytes.Buffer + var thumbPath *string + err := d.thumbTokenBucket.Do(ctx, func() error { + var err error + buf, thumbPath, err = d.getThumb(file) + return err + }) if err != nil { return nil, err } @@ -220,11 +301,22 @@ func (d *Local) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } - err := os.Rename(srcPath, dstPath) - if err != nil { + if err := os.Rename(srcPath, dstPath); err != nil && strings.Contains(err.Error(), "invalid cross-device link") { + // Handle cross-device file move in local driver + if err = d.Copy(ctx, srcObj, dstDir); err != nil { + return err + } else { + // Directly remove file without check recycle bin if successfully copied + if srcObj.IsDir() { + err = os.RemoveAll(srcObj.GetPath()) + } else { + err = os.Remove(srcObj.GetPath()) + } + return err + } + } else { return err } - return nil } func (d *Local) Rename(ctx context.Context, srcObj model.Obj, newName string) error { @@ -237,30 +329,34 @@ func (d *Local) Rename(ctx context.Context, srcObj model.Obj, newName string) er return nil } -func (d *Local) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { +func (d *Local) Copy(_ context.Context, srcObj, dstDir model.Obj) error { srcPath := srcObj.GetPath() dstPath := filepath.Join(dstDir.GetPath(), srcObj.GetName()) if utils.IsSubPath(srcPath, dstPath) { return fmt.Errorf("the destination folder is a subfolder of the source folder") } - var err error - if srcObj.IsDir() { - err = utils.CopyDir(srcPath, dstPath) - } else { - err = utils.CopyFile(srcPath, dstPath) - } - if err != nil { - return err - } - return nil + // Copy using otiai10/copy to perform more secure & efficient copy + return cp.Copy(srcPath, dstPath, cp.Options{ + Sync: true, // Sync file to disk after copy, may have performance penalty in filesystem such as ZFS + PreserveTimes: true, + PreserveOwner: true, + }) } func (d *Local) Remove(ctx context.Context, obj model.Obj) error { var err error - if obj.IsDir() { - err = os.RemoveAll(obj.GetPath()) + if utils.SliceContains([]string{"", "delete permanently"}, d.RecycleBinPath) { + if obj.IsDir() { + err = os.RemoveAll(obj.GetPath()) + } else { + err = os.Remove(obj.GetPath()) + } } else { - err = os.Remove(obj.GetPath()) + dstPath := filepath.Join(d.RecycleBinPath, obj.GetName()) + if utils.Exists(dstPath) { + dstPath = filepath.Join(d.RecycleBinPath, obj.GetName()+"_"+time.Now().Format("20060102150405")) + } + err = os.Rename(obj.GetPath(), dstPath) } if err != nil { return err diff --git a/drivers/local/meta.go b/drivers/local/meta.go index 00c8f5ce8b1..70ce090db5f 100644 --- a/drivers/local/meta.go +++ b/drivers/local/meta.go @@ -8,9 +8,14 @@ import ( type Addition struct { driver.RootPath Thumbnail bool `json:"thumbnail" required:"true" help:"enable thumbnail"` + UseFFmpeg bool `json:"use_ffmpeg" required:"true" help:"use ffmpeg to generate thumbnail"` ThumbCacheFolder string `json:"thumb_cache_folder"` + ThumbConcurrency string `json:"thumb_concurrency" default:"16" required:"false" help:"Number of concurrent thumbnail generation goroutines. This controls how many thumbnails can be generated in parallel."` + ThumbPixel string `json:"thumb_pixel" default:"320" required:"false" help:"Specifies the target width for image thumbnails in pixels. The height of the thumbnail will be calculated automatically to maintain the original aspect ratio of the image."` + VideoThumbPos string `json:"video_thumb_pos" default:"20%" required:"false" help:"The position of the video thumbnail. If the value is a number (integer ot floating point), it represents the time in seconds. If the value ends with '%', it represents the percentage of the video duration."` ShowHidden bool `json:"show_hidden" default:"true" required:"false" help:"show hidden directories and files"` MkdirPerm string `json:"mkdir_perm" default:"777"` + RecycleBinPath string `json:"recycle_bin_path" default:"delete permanently" help:"path to recycle bin, delete permanently if empty or keep 'delete permanently'"` } var config = driver.Config{ diff --git a/drivers/local/token_bucket.go b/drivers/local/token_bucket.go new file mode 100644 index 00000000000..23c6ebd63b7 --- /dev/null +++ b/drivers/local/token_bucket.go @@ -0,0 +1,95 @@ +package local + +import "context" + +type TokenBucket interface { + Take() <-chan struct{} + Put() + Do(context.Context, func() error) error +} + +// StaticTokenBucket is a bucket with a fixed number of tokens, +// where the retrieval and return of tokens are manually controlled. +// In the initial state, the bucket is full. +type StaticTokenBucket struct { + bucket chan struct{} +} + +func NewStaticTokenBucket(size int) StaticTokenBucket { + bucket := make(chan struct{}, size) + for range size { + bucket <- struct{}{} + } + return StaticTokenBucket{bucket: bucket} +} + +func NewStaticTokenBucketWithMigration(oldBucket TokenBucket, size int) StaticTokenBucket { + if oldBucket != nil { + oldStaticBucket, ok := oldBucket.(StaticTokenBucket) + if ok { + oldSize := cap(oldStaticBucket.bucket) + migrateSize := oldSize + if size < migrateSize { + migrateSize = size + } + + bucket := make(chan struct{}, size) + for range size - migrateSize { + bucket <- struct{}{} + } + + if migrateSize != 0 { + go func() { + for range migrateSize { + <-oldStaticBucket.bucket + bucket <- struct{}{} + } + close(oldStaticBucket.bucket) + }() + } + return StaticTokenBucket{bucket: bucket} + } + } + return NewStaticTokenBucket(size) +} + +// Take channel maybe closed when local driver is modified. +// don't call Put method after the channel is closed. +func (b StaticTokenBucket) Take() <-chan struct{} { + return b.bucket +} + +func (b StaticTokenBucket) Put() { + b.bucket <- struct{}{} +} + +func (b StaticTokenBucket) Do(ctx context.Context, f func() error) error { + select { + case <-ctx.Done(): + return ctx.Err() + case _, ok := <-b.Take(): + if ok { + defer b.Put() + } + } + return f() +} + +// NopTokenBucket all function calls to this bucket will success immediately +type NopTokenBucket struct { + nop chan struct{} +} + +func NewNopTokenBucket() NopTokenBucket { + nop := make(chan struct{}) + close(nop) + return NopTokenBucket{nop} +} + +func (b NopTokenBucket) Take() <-chan struct{} { + return b.nop +} + +func (b NopTokenBucket) Put() {} + +func (b NopTokenBucket) Do(_ context.Context, f func() error) error { return f() } diff --git a/drivers/local/util.go b/drivers/local/util.go index 84f1822bf24..8940d44e254 100644 --- a/drivers/local/util.go +++ b/drivers/local/util.go @@ -2,11 +2,14 @@ package local import ( "bytes" + "encoding/json" "fmt" "io/fs" "os" "path/filepath" + "runtime" "sort" + "strconv" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -16,14 +19,18 @@ import ( ffmpeg "github.com/u2takey/ffmpeg-go" ) -func isSymlinkDir(f fs.FileInfo, path string) bool { - if f.Mode()&os.ModeSymlink == os.ModeSymlink { - dst, err := os.Readlink(filepath.Join(path, f.Name())) +func isLinkedDir(f fs.FileInfo, path string) bool { + if f.Mode()&os.ModeSymlink == os.ModeSymlink || (runtime.GOOS == "windows" && f.Mode()&os.ModeIrregular != 0) { + dst, err := os.Readlink(path) if err != nil { return false } if !filepath.IsAbs(dst) { - dst = filepath.Join(path, dst) + dst = filepath.Join(filepath.Dir(path), dst) + } + dst, err = filepath.Abs(dst) + if err != nil { + return false } stat, err := os.Stat(dst) if err != nil { @@ -34,16 +41,167 @@ func isSymlinkDir(f fs.FileInfo, path string) bool { return false } -func GetSnapshot(videoPath string, frameNum int) (imgData *bytes.Buffer, err error) { - srcBuf := bytes.NewBuffer(nil) - err = ffmpeg.Input(videoPath).Filter("select", ffmpeg.Args{fmt.Sprintf("gte(n,%d)", frameNum)}). - Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg"}). - WithOutput(srcBuf, os.Stdout). +// sanitizeFilePath validates and sanitizes a file path before passing it to external commands. +// It ensures the path is absolute, clean, and refers to an existing regular file, +// preventing path traversal and command injection via shell metacharacters. +func sanitizeFilePath(path string) (string, error) { + cleaned := filepath.Clean(path) + if !filepath.IsAbs(cleaned) { + return "", fmt.Errorf("file path must be absolute: %s", path) + } + info, err := os.Stat(cleaned) + if err != nil { + return "", fmt.Errorf("file path is not accessible: %w", err) + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("path is not a regular file: %s", cleaned) + } + return cleaned, nil +} + +// resizeImageToBufferWithFFmpegGo 使用 ffmpeg-go 调整图片大小并输出到内存缓冲区 +func resizeImageToBufferWithFFmpegGo(inputFile string, width int, outputFormat string /* e.g., "image2pipe", "png_pipe", "mjpeg" */) (*bytes.Buffer, error) { + sanitized, err := sanitizeFilePath(inputFile) + if err != nil { + return nil, fmt.Errorf("invalid input file path: %w", err) + } + inputFile = sanitized + + outBuffer := bytes.NewBuffer(nil) + + // Determine codec based on desired output format for piping + // For generic image piping, 'image2' is often used with -f image2pipe + // For specific formats to buffer, you might specify the codec directly + var vcodec string + switch outputFormat { + case "png_pipe": // if you want to ensure PNG format in buffer + vcodec = "png" + case "mjpeg": // if you want to ensure JPEG format in buffer + vcodec = "mjpeg" + // default or "image2pipe" could leave codec choice more to ffmpeg or require -c:v later + } + + outputArgs := ffmpeg.KwArgs{ + "vf": fmt.Sprintf("scale=%d:-1:flags=lanczos,format=yuv444p", width), + "vframes": "1", + "f": outputFormat, // Format for piping (e.g., image2pipe, png_pipe) + } + if vcodec != "" { + outputArgs["vcodec"] = vcodec + } + if outputFormat == "mjpeg" { + outputArgs["q:v"] = "3" + } + + err = ffmpeg.Input(inputFile). + Output("pipe:", outputArgs). // Output to pipe (stdout) + GlobalArgs("-loglevel", "error"). + Silent(true). // Suppress ffmpeg's own console output + WithOutput(outBuffer, os.Stderr). // Capture stdout to outBuffer, stderr to os.Stderr + // ErrorToStdOut(). // Alternative: send ffmpeg's stderr to Go's stdout Run() + if err != nil { + return nil, fmt.Errorf("ffmpeg-go failed to resize image %s to buffer: %w", inputFile, err) + } + if outBuffer == nil || outBuffer.Len() == 0 { + return nil, fmt.Errorf("ffmpeg-go produced empty buffer for %s", inputFile) + } + + return outBuffer, nil +} + +func generateThumbnailWithImagingOptimized(imagePath string, targetWidth int, quality int) (*bytes.Buffer, error) { + + file, err := os.Open(imagePath) + if err != nil { + return nil, fmt.Errorf("failed to open image: %w", err) + } + defer file.Close() + + img, err := imaging.Decode(file, imaging.AutoOrientation(true)) + if err != nil { + return nil, fmt.Errorf("failed to decode image: %w", err) + } + + thumbImg := imaging.Resize(img, targetWidth, 0, imaging.Lanczos) + img = nil + + var buf bytes.Buffer + // imaging.Encode + // imaging.PNG, imaging.JPEG, imaging.GIF, imaging.BMP, imaging.TIFF + outputFormat := imaging.JPEG + encodeOptions := []imaging.EncodeOption{imaging.JPEGQuality(quality)} + + // outputFormat := imaging.PNG + // encodeOptions := []imaging.EncodeOption{} + + err = imaging.Encode(&buf, thumbImg, outputFormat, encodeOptions...) + if err != nil { + return nil, fmt.Errorf("failed to encode thumbnail: %w", err) + } + + thumbImg = nil + + return &buf, nil +} + +// Get the snapshot of the video +func (d *Local) GetSnapshot(videoPath string) (imgData *bytes.Buffer, err error) { + sanitized, err := sanitizeFilePath(videoPath) + if err != nil { + return nil, fmt.Errorf("invalid video path: %w", err) + } + videoPath = sanitized + + // Run ffprobe to get the video duration + jsonOutput, err := ffmpeg.Probe(videoPath) + if err != nil { + return nil, err + } + // get format.duration from the json string + type probeFormat struct { + Duration string `json:"duration"` + } + type probeData struct { + Format probeFormat `json:"format"` + } + var probe probeData + err = json.Unmarshal([]byte(jsonOutput), &probe) + if err != nil { + return nil, err + } + totalDuration, err := strconv.ParseFloat(probe.Format.Duration, 64) if err != nil { return nil, err } + + var ss string + if d.videoThumbPosIsPercentage { + ss = fmt.Sprintf("%f", totalDuration*d.videoThumbPos) + } else { + // If the value is greater than the total duration, use the total duration + if d.videoThumbPos > totalDuration { + ss = fmt.Sprintf("%f", totalDuration) + } else { + ss = fmt.Sprintf("%f", d.videoThumbPos) + } + } + + // Run ffmpeg to get the snapshot + srcBuf := bytes.NewBuffer(nil) + // If the remaining time from the seek point to the end of the video is less + // than the duration of a single frame, ffmpeg cannot extract any frames + // within the specified range and will exit with an error. + // The "noaccurate_seek" option prevents this error and would also speed up + // the seek process. + stream := ffmpeg.Input(videoPath, ffmpeg.KwArgs{"ss": ss, "noaccurate_seek": ""}). + Output("pipe:", ffmpeg.KwArgs{"vframes": 1, "format": "image2", "vcodec": "mjpeg", "vf": fmt.Sprintf("scale=%d:-1:flags=lanczos", d.thumbPixel)}). + GlobalArgs("-loglevel", "error").Silent(true). + WithOutput(srcBuf, os.Stdout) + if err = stream.Run(); err != nil { + return nil, err + } return srcBuf, nil } @@ -64,7 +222,7 @@ func readDir(dirname string) ([]fs.FileInfo, error) { func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { fullPath := file.GetPath() thumbPrefix := "alist_thumb_" - thumbName := thumbPrefix + utils.GetMD5EncodeStr(fullPath) + ".png" + thumbName := thumbPrefix + utils.GetMD5EncodeStr(fmt.Sprintf("%s:%d", fullPath, d.thumbSize)) + ".png" if d.ThumbCacheFolder != "" { // skip if the file is a thumbnail if strings.HasPrefix(file.GetName(), thumbPrefix) { @@ -77,35 +235,32 @@ func (d *Local) getThumb(file model.Obj) (*bytes.Buffer, *string, error) { } var srcBuf *bytes.Buffer if utils.GetFileType(file.GetName()) == conf.VIDEO { - videoBuf, err := GetSnapshot(fullPath, 10) + videoBuf, err := d.GetSnapshot(fullPath) if err != nil { return nil, nil, err } srcBuf = videoBuf } else { - imgData, err := os.ReadFile(fullPath) - if err != nil { - return nil, nil, err + if d.useFFmpeg { + imgData, err := resizeImageToBufferWithFFmpegGo(fullPath, d.thumbPixel, "image2pipe") + srcBuf = imgData + if err != nil { + return nil, nil, err + } + } else { + imgData, err := generateThumbnailWithImagingOptimized(fullPath, d.thumbPixel, 70) + srcBuf = imgData + if err != nil { + return nil, nil, err + } } - imgBuf := bytes.NewBuffer(imgData) - srcBuf = imgBuf } - image, err := imaging.Decode(srcBuf, imaging.AutoOrientation(true)) - if err != nil { - return nil, nil, err - } - thumbImg := imaging.Resize(image, 144, 0, imaging.Lanczos) - var buf bytes.Buffer - err = imaging.Encode(&buf, thumbImg, imaging.PNG) - if err != nil { - return nil, nil, err - } if d.ThumbCacheFolder != "" { - err = os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), buf.Bytes(), 0666) + err := os.WriteFile(filepath.Join(d.ThumbCacheFolder, thumbName), srcBuf.Bytes(), 0666) if err != nil { return nil, nil, err } } - return &buf, nil, nil + return srcBuf, nil, nil } diff --git a/drivers/mediafire/driver.go b/drivers/mediafire/driver.go new file mode 100644 index 00000000000..e77510eabc0 --- /dev/null +++ b/drivers/mediafire/driver.go @@ -0,0 +1,433 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "context" + "fmt" + "math/rand" + "net/http" + "os" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Mediafire struct { + model.Storage + Addition + cron *cron.Cron + + actionToken string + + appBase string + apiBase string + hostBase string + maxRetries int + + secChUa string + secChUaPlatform string + userAgent string +} + +func (d *Mediafire) Config() driver.Config { + return config +} + +func (d *Mediafire) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Mediafire) Init(ctx context.Context) error { + if d.SessionToken == "" { + return fmt.Errorf("Init :: [MediaFire] {critical} missing sessionToken") + } + + if d.Cookie == "" { + return fmt.Errorf("Init :: [MediaFire] {critical} missing Cookie") + } + + if _, err := d.getSessionToken(ctx); err != nil { + + d.renewToken(ctx) + + num := rand.Intn(4) + 6 + + d.cron = cron.NewCron(time.Minute * time.Duration(num)) + d.cron.Do(func() { + d.renewToken(ctx) + }) + + } + + return nil +} + +func (d *Mediafire) Drop(ctx context.Context) error { + return nil +} + +func (d *Mediafire) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(ctx, dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return d.fileToObj(src), nil + }) +} + +func (d *Mediafire) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + + downloadUrl, err := d.getDirectDownloadLink(ctx, file.GetID()) + if err != nil { + return nil, err + } + + res, err := base.NoRedirectClient.R().SetDoNotParseResponse(true).SetContext(ctx).Get(downloadUrl) + if err != nil { + return nil, err + } + defer func() { + _ = res.RawBody().Close() + }() + + if res.StatusCode() == 302 { + downloadUrl = res.Header().Get("location") + } + + return &model.Link{ + URL: downloadUrl, + Header: http.Header{ + "Origin": []string{d.appBase}, + "Referer": []string{d.appBase + "/"}, + "sec-ch-ua": []string{d.secChUa}, + "sec-ch-ua-platform": []string{d.secChUaPlatform}, + "User-Agent": []string{d.userAgent}, + //"User-Agent": []string{base.UserAgent}, + }, + }, nil +} + +func (d *Mediafire) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "parent_key": parentDir.GetID(), + "foldername": dirName, + } + + var resp MediafireFolderCreateResponse + _, err := d.postForm("/folder/create.php", data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + created, _ := time.Parse("2006-01-02T15:04:05Z", resp.Response.CreatedUTC) + + return &model.ObjThumb{ + Object: model.Object{ + ID: resp.Response.FolderKey, + Name: resp.Response.Name, + Size: 0, + Modified: created, + Ctime: created, + IsFolder: true, + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/move.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key_src": srcObj.GetID(), + "folder_key_dst": dstDir.GetID(), + } + } else { + + endpoint = "/file/move.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "folder_key": dstDir.GetID(), + } + } + + var resp MediafireMoveResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return srcObj, nil +} + +func (d *Mediafire) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/update.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": srcObj.GetID(), + "foldername": newName, + } + } else { + + endpoint = "/file/update.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "filename": newName, + } + } + + var resp MediafireRenameResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var data map[string]string + var endpoint string + + if srcObj.IsDir() { + + endpoint = "/folder/copy.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key_src": srcObj.GetID(), + "folder_key_dst": dstDir.GetID(), + } + } else { + + endpoint = "/file/copy.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": srcObj.GetID(), + "folder_key": dstDir.GetID(), + } + } + + var resp MediafireCopyResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + var newID string + if srcObj.IsDir() { + if len(resp.Response.NewFolderKeys) > 0 { + newID = resp.Response.NewFolderKeys[0] + } + } else { + if len(resp.Response.NewQuickKeys) > 0 { + newID = resp.Response.NewQuickKeys[0] + } + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: newID, + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) Remove(ctx context.Context, obj model.Obj) error { + var data map[string]string + var endpoint string + + if obj.IsDir() { + + endpoint = "/folder/delete.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": obj.GetID(), + } + } else { + + endpoint = "/file/delete.php" + data = map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "quick_key": obj.GetID(), + } + } + + var resp MediafireRemoveResponse + _, err := d.postForm(endpoint, data, &resp) + if err != nil { + return err + } + + if resp.Response.Result != "Success" { + return fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return nil +} + +func (d *Mediafire) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + _, err := d.PutResult(ctx, dstDir, file, up) + return err +} + +func (d *Mediafire) PutResult(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return nil, err + } + defer tempFile.Close() + + osFile, ok := tempFile.(*os.File) + if !ok { + return nil, fmt.Errorf("expected *os.File, got %T", tempFile) + } + + fileHash, err := d.calculateSHA256(osFile) + if err != nil { + return nil, err + } + + checkResp, err := d.uploadCheck(ctx, file.GetName(), file.GetSize(), fileHash, dstDir.GetID()) + if err != nil { + return nil, err + } + + if checkResp.Response.ResumableUpload.AllUnitsReady == "yes" { + up(100.0) + } + + if checkResp.Response.HashExists == "yes" && checkResp.Response.InAccount == "yes" { + up(100.0) + existingFile, err := d.getExistingFileInfo(ctx, fileHash, file.GetName(), dstDir.GetID()) + if err == nil { + return existingFile, nil + } + } + + var pollKey string + + if checkResp.Response.ResumableUpload.AllUnitsReady != "yes" { + + var err error + + pollKey, err = d.uploadUnits(ctx, osFile, checkResp, file.GetName(), fileHash, dstDir.GetID(), up) + if err != nil { + return nil, err + } + } else { + + pollKey = checkResp.Response.ResumableUpload.UploadKey + } + + //fmt.Printf("pollKey: %+v\n", pollKey) + + pollResp, err := d.pollUpload(ctx, pollKey) + if err != nil { + return nil, err + } + + quickKey := pollResp.Response.Doupload.QuickKey + + return &model.ObjThumb{ + Object: model.Object{ + ID: quickKey, + Name: file.GetName(), + Size: file.GetSize(), + }, + Thumbnail: model.Thumbnail{}, + }, nil +} + +func (d *Mediafire) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Mediafire) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +//func (d *Mediafire) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Mediafire)(nil) diff --git a/drivers/mediafire/meta.go b/drivers/mediafire/meta.go new file mode 100644 index 00000000000..243d55570af --- /dev/null +++ b/drivers/mediafire/meta.go @@ -0,0 +1,54 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + //driver.RootID + + SessionToken string `json:"session_token" required:"true" type:"string" help:"Required for MediaFire API"` + Cookie string `json:"cookie" required:"true" type:"string" help:"Required for navigation"` + + OrderBy string `json:"order_by" type:"select" options:"name,time,size" default:"name"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + ChunkSize int64 `json:"chunk_size" type:"number" default:"100"` +} + +var config = driver.Config{ + Name: "MediaFire", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Mediafire{ + appBase: "https://app.mediafire.com", + apiBase: "https://www.mediafire.com/api/1.5", + hostBase: "https://www.mediafire.com", + maxRetries: 3, + secChUa: "\"Not)A;Brand\";v=\"8\", \"Chromium\";v=\"139\", \"Google Chrome\";v=\"139\"", + secChUaPlatform: "Windows", + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36", + } + }) +} diff --git a/drivers/mediafire/types.go b/drivers/mediafire/types.go new file mode 100644 index 00000000000..0073b58179c --- /dev/null +++ b/drivers/mediafire/types.go @@ -0,0 +1,232 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +type MediafireRenewTokenResponse struct { + Response struct { + Action string `json:"action"` + SessionToken string `json:"session_token"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireResponse struct { + Response struct { + Action string `json:"action"` + FolderContent struct { + ChunkSize string `json:"chunk_size"` + ContentType string `json:"content_type"` + ChunkNumber string `json:"chunk_number"` + FolderKey string `json:"folderkey"` + Folders []MediafireFolder `json:"folders,omitempty"` + Files []MediafireFile `json:"files,omitempty"` + MoreChunks string `json:"more_chunks"` + } `json:"folder_content"` + Result string `json:"result"` + } `json:"response"` +} + +type MediafireFolder struct { + FolderKey string `json:"folderkey"` + Name string `json:"name"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` +} + +type MediafireFile struct { + QuickKey string `json:"quickkey"` + Filename string `json:"filename"` + Size string `json:"size"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + MimeType string `json:"mimetype"` +} + +type File struct { + ID string + Name string + Size int64 + CreatedUTC string + IsFolder bool +} + +type FolderContentResponse struct { + Folders []MediafireFolder + Files []MediafireFile + MoreChunks bool +} + +type MediafireLinksResponse struct { + Response struct { + Action string `json:"action"` + Links []struct { + QuickKey string `json:"quickkey"` + View string `json:"view"` + NormalDownload string `json:"normal_download"` + OneTime struct { + Download string `json:"download"` + View string `json:"view"` + } `json:"one_time"` + } `json:"links"` + OneTimeKeyRequestCount string `json:"one_time_key_request_count"` + OneTimeKeyRequestMaxCount string `json:"one_time_key_request_max_count"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireDirectDownloadResponse struct { + Response struct { + Action string `json:"action"` + Links []struct { + QuickKey string `json:"quickkey"` + DirectDownload string `json:"direct_download"` + } `json:"links"` + DirectDownloadFreeBandwidth string `json:"direct_download_free_bandwidth"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireFolderCreateResponse struct { + Response struct { + Action string `json:"action"` + FolderKey string `json:"folder_key"` + UploadKey string `json:"upload_key"` + ParentFolderKey string `json:"parent_folderkey"` + Name string `json:"name"` + Description string `json:"description"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + Privacy string `json:"privacy"` + FileCount string `json:"file_count"` + FolderCount string `json:"folder_count"` + Revision string `json:"revision"` + DropboxEnabled string `json:"dropbox_enabled"` + Flag string `json:"flag"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireMoveResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + NewNames []string `json:"new_names"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireRenameResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireCopyResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + NewQuickKeys []string `json:"new_quickkeys,omitempty"` + NewFolderKeys []string `json:"new_folderkeys,omitempty"` + SkippedCount string `json:"skipped_count,omitempty"` + OtherCount string `json:"other_count,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireRemoveResponse struct { + Response struct { + Action string `json:"action"` + Asynchronous string `json:"asynchronous,omitempty"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + NewDeviceRevision int `json:"new_device_revision"` + } `json:"response"` +} + +type MediafireCheckResponse struct { + Response struct { + Action string `json:"action"` + HashExists string `json:"hash_exists"` + InAccount string `json:"in_account"` + InFolder string `json:"in_folder"` + FileExists string `json:"file_exists"` + ResumableUpload struct { + AllUnitsReady string `json:"all_units_ready"` + NumberOfUnits string `json:"number_of_units"` + UnitSize string `json:"unit_size"` + Bitmap struct { + Count string `json:"count"` + Words []string `json:"words"` + } `json:"bitmap"` + UploadKey string `json:"upload_key"` + } `json:"resumable_upload"` + AvailableSpace string `json:"available_space"` + UsedStorageSize string `json:"used_storage_size"` + StorageLimit string `json:"storage_limit"` + StorageLimitExceeded string `json:"storage_limit_exceeded"` + UploadURL struct { + Simple string `json:"simple"` + SimpleFallback string `json:"simple_fallback"` + Resumable string `json:"resumable"` + ResumableFallback string `json:"resumable_fallback"` + } `json:"upload_url"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} +type MediafireActionTokenResponse struct { + Response struct { + Action string `json:"action"` + ActionToken string `json:"action_token"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafirePollResponse struct { + Response struct { + Action string `json:"action"` + Doupload struct { + Result string `json:"result"` + Status string `json:"status"` + Description string `json:"description"` + QuickKey string `json:"quickkey"` + Hash string `json:"hash"` + Filename string `json:"filename"` + Size string `json:"size"` + Created string `json:"created"` + CreatedUTC string `json:"created_utc"` + Revision string `json:"revision"` + } `json:"doupload"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} + +type MediafireFileSearchResponse struct { + Response struct { + Action string `json:"action"` + FileInfo []File `json:"file_info"` + Result string `json:"result"` + CurrentAPIVersion string `json:"current_api_version"` + } `json:"response"` +} diff --git a/drivers/mediafire/util.go b/drivers/mediafire/util.go new file mode 100644 index 00000000000..091abd0cd64 --- /dev/null +++ b/drivers/mediafire/util.go @@ -0,0 +1,626 @@ +package mediafire + +/* +Package mediafire +Author: Da3zKi7 +Date: 2025-09-11 + +D@' 3z K!7 - The King Of Cracking +*/ + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *Mediafire) getSessionToken(ctx context.Context) (string, error) { + tokenURL := d.hostBase + "/application/get_session_token.php" + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, tokenURL, nil) + if err != nil { + return "", err + } + + req.Header.Set("Accept", "*/*") + req.Header.Set("Accept-Encoding", "gzip, deflate, br, zstd") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + req.Header.Set("Content-Length", "0") + req.Header.Set("Cookie", d.Cookie) + req.Header.Set("DNT", "1") + req.Header.Set("Origin", d.hostBase) + req.Header.Set("Priority", "u=1, i") + req.Header.Set("Referer", (d.hostBase + "/")) + req.Header.Set("Sec-Ch-Ua", d.secChUa) + req.Header.Set("Sec-Ch-Ua-Mobile", "?0") + req.Header.Set("Sec-Ch-Ua-Platform", d.secChUaPlatform) + req.Header.Set("Sec-Fetch-Dest", "empty") + req.Header.Set("Sec-Fetch-Mode", "cors") + req.Header.Set("Sec-Fetch-Site", "same-site") + req.Header.Set("User-Agent", d.userAgent) + //req.Header.Set("Connection", "keep-alive") + + resp, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + //fmt.Printf("getSessionToken :: Raw response: %s\n", string(body)) + //fmt.Printf("getSessionToken :: Parsed response: %+v\n", resp) + + var tokenResp struct { + Response struct { + SessionToken string `json:"session_token"` + } `json:"response"` + } + + if resp.StatusCode == 200 { + if err := json.Unmarshal(body, &tokenResp); err != nil { + return "", err + } + + if tokenResp.Response.SessionToken == "" { + return "", fmt.Errorf("empty session token received") + } + + cookieMap := make(map[string]string) + for _, cookie := range resp.Cookies() { + cookieMap[cookie.Name] = cookie.Value + } + + if len(cookieMap) > 0 { + + var cookies []string + for name, value := range cookieMap { + cookies = append(cookies, fmt.Sprintf("%s=%s", name, value)) + } + d.Cookie = strings.Join(cookies, "; ") + op.MustSaveDriverStorage(d) + + //fmt.Printf("getSessionToken :: Captured cookies: %s\n", d.Cookie) + } + + } else { + return "", fmt.Errorf("getSessionToken :: failed to get session token, status code: %d", resp.StatusCode) + } + + d.SessionToken = tokenResp.Response.SessionToken + + //fmt.Printf("Init :: Obtain Session Token %v", d.SessionToken) + + op.MustSaveDriverStorage(d) + + return d.SessionToken, nil +} + +func (d *Mediafire) renewToken(_ context.Context) error { + query := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + } + + var resp MediafireRenewTokenResponse + _, err := d.postForm("/user/renew_session_token.php", query, &resp) + if err != nil { + return fmt.Errorf("failed to renew token: %w", err) + } + + //fmt.Printf("getInfo :: Raw response: %s\n", string(body)) + //fmt.Printf("getInfo :: Parsed response: %+v\n", resp) + + if resp.Response.Result != "Success" { + return fmt.Errorf("MediaFire token renewal failed: %s", resp.Response.Result) + } + + d.SessionToken = resp.Response.SessionToken + + //fmt.Printf("Init :: Renew Session Token: %s", resp.Response.Result) + + op.MustSaveDriverStorage(d) + + return nil +} + +func (d *Mediafire) getFiles(ctx context.Context, folderKey string) ([]File, error) { + files := make([]File, 0) + hasMore := true + chunkNumber := 1 + + for hasMore { + resp, err := d.getFolderContent(ctx, folderKey, chunkNumber) + if err != nil { + return nil, err + } + + for _, folder := range resp.Folders { + files = append(files, File{ + ID: folder.FolderKey, + Name: folder.Name, + Size: 0, + CreatedUTC: folder.CreatedUTC, + IsFolder: true, + }) + } + + for _, file := range resp.Files { + size, _ := strconv.ParseInt(file.Size, 10, 64) + files = append(files, File{ + ID: file.QuickKey, + Name: file.Filename, + Size: size, + CreatedUTC: file.CreatedUTC, + IsFolder: false, + }) + } + + hasMore = resp.MoreChunks + chunkNumber++ + } + + return files, nil +} + +func (d *Mediafire) getFolderContent(ctx context.Context, folderKey string, chunkNumber int) (*FolderContentResponse, error) { + + foldersResp, err := d.getFolderContentByType(ctx, folderKey, "folders", chunkNumber) + if err != nil { + return nil, err + } + + filesResp, err := d.getFolderContentByType(ctx, folderKey, "files", chunkNumber) + if err != nil { + return nil, err + } + + return &FolderContentResponse{ + Folders: foldersResp.Response.FolderContent.Folders, + Files: filesResp.Response.FolderContent.Files, + MoreChunks: foldersResp.Response.FolderContent.MoreChunks == "yes" || filesResp.Response.FolderContent.MoreChunks == "yes", + }, nil +} + +func (d *Mediafire) getFolderContentByType(_ context.Context, folderKey, contentType string, chunkNumber int) (*MediafireResponse, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "folder_key": folderKey, + "content_type": contentType, + "chunk": strconv.Itoa(chunkNumber), + "chunk_size": strconv.FormatInt(d.ChunkSize, 10), + "details": "yes", + "order_direction": d.OrderDirection, + "order_by": d.OrderBy, + "filter": "", + } + + var resp MediafireResponse + _, err := d.postForm("/folder/get_content.php", data, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) fileToObj(f File) *model.ObjThumb { + created, _ := time.Parse("2006-01-02T15:04:05Z", f.CreatedUTC) + + var thumbnailURL string + if !f.IsFolder && f.ID != "" { + thumbnailURL = d.hostBase + "/convkey/acaa/" + f.ID + "3g.jpg" + } + + return &model.ObjThumb{ + Object: model.Object{ + ID: f.ID, + //Path: "", + Name: f.Name, + Size: f.Size, + Modified: created, + Ctime: created, + IsFolder: f.IsFolder, + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumbnailURL, + }, + } +} + +func (d *Mediafire) getForm(endpoint string, query map[string]string, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + + req.SetQueryParams(query) + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + //"User-Agent": base.UserAgent, + "User-Agent": d.userAgent, + "Origin": d.appBase, + "Referer": d.appBase + "/", + }) + + // If response OK + if resp != nil { + req.SetResult(resp) + } + + // Targets MediaFire API + res, err := req.Get(d.apiBase + endpoint) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Mediafire) postForm(endpoint string, data map[string]string, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + + req.SetFormData(data) + + req.SetHeaders(map[string]string{ + "Cookie": d.Cookie, + "Content-Type": "application/x-www-form-urlencoded", + //"User-Agent": base.UserAgent, + "User-Agent": d.userAgent, + "Origin": d.appBase, + "Referer": d.appBase + "/", + }) + + // If response OK + if resp != nil { + req.SetResult(resp) + } + + // Targets MediaFire API + res, err := req.Post(d.apiBase + endpoint) + if err != nil { + return nil, err + } + + return res.Body(), nil +} + +func (d *Mediafire) getDirectDownloadLink(_ context.Context, fileID string) (string, error) { + data := map[string]string{ + "session_token": d.SessionToken, + "quick_key": fileID, + "link_type": "direct_download", + "response_format": "json", + } + + var resp MediafireDirectDownloadResponse + _, err := d.getForm("/file/get_links.php", data, &resp) + if err != nil { + return "", err + } + + if resp.Response.Result != "Success" { + return "", fmt.Errorf("MediaFire API error: %s", resp.Response.Result) + } + + if len(resp.Response.Links) == 0 { + return "", fmt.Errorf("no download links found") + } + + return resp.Response.Links[0].DirectDownload, nil +} + +func (d *Mediafire) calculateSHA256(file *os.File) (string, error) { + hasher := sha256.New() + if _, err := file.Seek(0, 0); err != nil { + return "", err + } + if _, err := io.Copy(hasher, file); err != nil { + return "", err + } + return hex.EncodeToString(hasher.Sum(nil)), nil +} + +func (d *Mediafire) uploadCheck(ctx context.Context, filename string, filesize int64, filehash, folderKey string) (*MediafireCheckResponse, error) { + + actionToken, err := d.getActionToken(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get action token: %w", err) + } + + query := map[string]string{ + "session_token": actionToken, /* d.SessionToken */ + "filename": filename, + "size": strconv.FormatInt(filesize, 10), + "hash": filehash, + "folder_key": folderKey, + "resumable": "yes", + "response_format": "json", + } + + var resp MediafireCheckResponse + _, err = d.postForm("/upload/check.php", query, &resp) + if err != nil { + return nil, err + } + + //fmt.Printf("uploadCheck :: Raw response: %s\n", string(body)) + //fmt.Printf("uploadCheck :: Parsed response: %+v\n", resp) + + //fmt.Printf("uploadCheck :: ResumableUpload section: %+v\n", resp.Response.ResumableUpload) + //fmt.Printf("uploadCheck :: Upload key specifically: '%s'\n", resp.Response.ResumableUpload.UploadKey) + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire upload check failed: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) resumableUpload(ctx context.Context, folderKey, uploadKey string, unitData []byte, unitID int, fileHash, filename string, totalFileSize int64) (string, error) { + actionToken, err := d.getActionToken(ctx) + if err != nil { + return "", err + } + + url := d.apiBase + "/upload/resumable.php" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(unitData)) + if err != nil { + return "", err + } + + q := req.URL.Query() + q.Add("folder_key", folderKey) + q.Add("response_format", "json") + q.Add("session_token", actionToken) + q.Add("key", uploadKey) + req.URL.RawQuery = q.Encode() + + req.Header.Set("x-filehash", fileHash) + req.Header.Set("x-filesize", strconv.FormatInt(totalFileSize, 10)) + req.Header.Set("x-unit-id", strconv.Itoa(unitID)) + req.Header.Set("x-unit-size", strconv.FormatInt(int64(len(unitData)), 10)) + req.Header.Set("x-unit-hash", d.sha256Hex(bytes.NewReader(unitData))) + req.Header.Set("x-filename", filename) + req.Header.Set("Content-Type", "application/octet-stream") + req.ContentLength = int64(len(unitData)) + + /* fmt.Printf("Debug resumable upload request:\n") + fmt.Printf(" URL: %s\n", req.URL.String()) + fmt.Printf(" Headers: %+v\n", req.Header) + fmt.Printf(" Unit ID: %d\n", unitID) + fmt.Printf(" Unit Size: %d\n", len(unitData)) + fmt.Printf(" Upload Key: %s\n", uploadKey) + fmt.Printf(" Action Token: %s\n", actionToken) */ + + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + + body, err := io.ReadAll(res.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body: %v", err) + } + + //fmt.Printf("MediaFire resumable upload response (status %d): %s\n", res.StatusCode, string(body)) + + var uploadResp struct { + Response struct { + Doupload struct { + Key string `json:"key"` + } `json:"doupload"` + Result string `json:"result"` + } `json:"response"` + } + + if err := json.Unmarshal(body, &uploadResp); err != nil { + return "", fmt.Errorf("failed to parse response: %v", err) + } + + if res.StatusCode != 200 { + return "", fmt.Errorf("resumable upload failed with status %d", res.StatusCode) + } + + return uploadResp.Response.Doupload.Key, nil +} + +func (d *Mediafire) uploadUnits(ctx context.Context, file *os.File, checkResp *MediafireCheckResponse, filename, fileHash, folderKey string, up driver.UpdateProgress) (string, error) { + unitSize, _ := strconv.ParseInt(checkResp.Response.ResumableUpload.UnitSize, 10, 64) + numUnits, _ := strconv.Atoi(checkResp.Response.ResumableUpload.NumberOfUnits) + uploadKey := checkResp.Response.ResumableUpload.UploadKey + + stringWords := checkResp.Response.ResumableUpload.Bitmap.Words + intWords := make([]int, len(stringWords)) + for i, word := range stringWords { + intWords[i], _ = strconv.Atoi(word) + } + + var finalUploadKey string + + for unitID := 0; unitID < numUnits; unitID++ { + + if utils.IsCanceled(ctx) { + return "", ctx.Err() + } + + if d.isUnitUploaded(intWords, unitID) { + up(float64(unitID+1) * 100 / float64(numUnits)) + continue + } + + uploadKey, err := d.uploadSingleUnit(ctx, file, unitID, unitSize, fileHash, filename, uploadKey, folderKey) + if err != nil { + return "", err + } + + finalUploadKey = uploadKey + + up(float64(unitID+1) * 100 / float64(numUnits)) + } + + return finalUploadKey, nil +} + +func (d *Mediafire) uploadSingleUnit(ctx context.Context, file *os.File, unitID int, unitSize int64, fileHash, filename, uploadKey, folderKey string) (string, error) { + start := int64(unitID) * unitSize + size := unitSize + + stat, err := file.Stat() + if err != nil { + return "", err + } + fileSize := stat.Size() + + if start+size > fileSize { + size = fileSize - start + } + + unitData := make([]byte, size) + if _, err := file.ReadAt(unitData, start); err != nil { + return "", err + } + + return d.resumableUpload(ctx, folderKey, uploadKey, unitData, unitID, fileHash, filename, fileSize) +} + +func (d *Mediafire) getActionToken(_ context.Context) (string, error) { + + if d.actionToken != "" { + return d.actionToken, nil + } + + data := map[string]string{ + "type": "upload", + "lifespan": "1440", + "response_format": "json", + "session_token": d.SessionToken, + } + + var resp MediafireActionTokenResponse + _, err := d.postForm("/user/get_action_token.php", data, &resp) + if err != nil { + return "", err + } + + if resp.Response.Result != "Success" { + return "", fmt.Errorf("MediaFire action token failed: %s", resp.Response.Result) + } + + return resp.Response.ActionToken, nil +} + +func (d *Mediafire) pollUpload(ctx context.Context, key string) (*MediafirePollResponse, error) { + + actionToken, err := d.getActionToken(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get action token: %w", err) + } + + //fmt.Printf("Debug Key: %+v\n", key) + + query := map[string]string{ + "key": key, + "response_format": "json", + "session_token": actionToken, /* d.SessionToken */ + } + + var resp MediafirePollResponse + _, err = d.postForm("/upload/poll_upload.php", query, &resp) + if err != nil { + return nil, err + } + + //fmt.Printf("pollUpload :: Raw response: %s\n", string(body)) + //fmt.Printf("pollUpload :: Parsed response: %+v\n", resp) + + //fmt.Printf("pollUpload :: Debug Result: %+v\n", resp.Response.Result) + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire poll upload failed: %s", resp.Response.Result) + } + + return &resp, nil +} + +func (d *Mediafire) sha256Hex(r io.Reader) string { + h := sha256.New() + io.Copy(h, r) + return hex.EncodeToString(h.Sum(nil)) +} + +func (d *Mediafire) isUnitUploaded(words []int, unitID int) bool { + wordIndex := unitID / 16 + bitIndex := unitID % 16 + if wordIndex >= len(words) { + return false + } + return (words[wordIndex]>>bitIndex)&1 == 1 +} + +func (d *Mediafire) getExistingFileInfo(ctx context.Context, fileHash, filename, folderKey string) (*model.ObjThumb, error) { + + if fileInfo, err := d.getFileByHash(ctx, fileHash); err == nil && fileInfo != nil { + return fileInfo, nil + } + + files, err := d.getFiles(ctx, folderKey) + if err != nil { + return nil, err + } + + for _, file := range files { + if file.Name == filename && !file.IsFolder { + return d.fileToObj(file), nil + } + } + + return nil, fmt.Errorf("existing file not found") +} + +func (d *Mediafire) getFileByHash(_ context.Context, hash string) (*model.ObjThumb, error) { + query := map[string]string{ + "session_token": d.SessionToken, + "response_format": "json", + "hash": hash, + } + + var resp MediafireFileSearchResponse + _, err := d.postForm("/file/get_info.php", query, &resp) + if err != nil { + return nil, err + } + + if resp.Response.Result != "Success" { + return nil, fmt.Errorf("MediaFire file search failed: %s", resp.Response.Result) + } + + if len(resp.Response.FileInfo) == 0 { + return nil, fmt.Errorf("file not found by hash") + } + + file := resp.Response.FileInfo[0] + return d.fileToObj(file), nil +} diff --git a/drivers/mediatrack/driver.go b/drivers/mediatrack/driver.go index 90e66ae0e34..50ef9799506 100644 --- a/drivers/mediatrack/driver.go +++ b/drivers/mediatrack/driver.go @@ -161,7 +161,7 @@ func (d *MediaTrack) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { src := "assets/" + uuid.New().String() var resp UploadResp _, err := d.request("https://jayce.api.mediatrack.cn/v3/storage/tokens/asset", http.MethodGet, func(req *resty.Request) { @@ -180,7 +180,7 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if err != nil { return err } - tempFile, err := stream.CacheFullInTempFile() + tempFile, err := file.CacheFullInTempFile() if err != nil { return err } @@ -188,10 +188,19 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil _ = tempFile.Close() }() uploader := s3manager.NewUploader(s) + if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) + } input := &s3manager.UploadInput{ Bucket: &resp.Data.Bucket, Key: &resp.Data.Object, - Body: tempFile, + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: tempFile, + Size: file.GetSize(), + }, + UpdateProgress: up, + }), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { @@ -203,19 +212,19 @@ func (d *MediaTrack) Put(ctx context.Context, dstDir model.Obj, stream model.Fil return err } h := md5.New() - _, err = io.Copy(h, tempFile) + _, err = utils.CopyWithBuffer(h, tempFile) if err != nil { return err } hash := hex.EncodeToString(h.Sum(nil)) data := base.Json{ "category": 0, - "description": stream.GetName(), + "description": file.GetName(), "hash": hash, - "mime": stream.GetMimetype(), - "size": stream.GetSize(), + "mime": file.GetMimetype(), + "size": file.GetSize(), "src": src, - "title": stream.GetName(), + "title": file.GetName(), "type": 0, } _, err = d.request(url, http.MethodPost, func(req *resty.Request) { diff --git a/drivers/mediatrack/meta.go b/drivers/mediatrack/meta.go index 47f112c3573..ade8ae1c8fe 100644 --- a/drivers/mediatrack/meta.go +++ b/drivers/mediatrack/meta.go @@ -9,8 +9,9 @@ type Addition struct { AccessToken string `json:"access_token" required:"true"` ProjectID string `json:"project_id"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` - OrderDesc bool `json:"order_desc"` + OrderBy string `json:"order_by" type:"select" options:"updated_at,title,size" default:"title"` + OrderDesc bool `json:"order_desc"` + DeviceFingerprint string `json:"device_fingerprint" required:"true"` } var config = driver.Config{ diff --git a/drivers/mediatrack/util.go b/drivers/mediatrack/util.go index 37ca0b3d09c..f5b751111a1 100644 --- a/drivers/mediatrack/util.go +++ b/drivers/mediatrack/util.go @@ -17,6 +17,9 @@ import ( func (d *MediaTrack) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() req.SetHeader("Authorization", "Bearer "+d.AccessToken) + if d.DeviceFingerprint != "" { + req.SetHeader("X-Device-Fingerprint", d.DeviceFingerprint) + } if callback != nil { callback(req) } diff --git a/drivers/mega/driver.go b/drivers/mega/driver.go index c1ae9f7f6c9..dc7b220129c 100644 --- a/drivers/mega/driver.go +++ b/drivers/mega/driver.go @@ -4,11 +4,13 @@ import ( "context" "errors" "fmt" - "github.com/alist-org/alist/v3/pkg/http_range" - "github.com/rclone/rclone/lib/readers" "io" "time" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/pquerna/otp/totp" + "github.com/rclone/rclone/lib/readers" + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -32,8 +34,16 @@ func (d *Mega) GetAddition() driver.Additional { } func (d *Mega) Init(ctx context.Context) error { + var twoFACode = d.TwoFACode d.c = mega.New() - return d.c.Login(d.Email, d.Password) + if d.TwoFASecret != "" { + code, err := totp.GenerateCode(d.TwoFASecret, time.Now()) + if err != nil { + return fmt.Errorf("generate totp code failed: %w", err) + } + twoFACode = code + } + return d.c.MultiFactorLogin(d.Email, d.Password, twoFACode) } func (d *Mega) Drop(ctx context.Context) error { @@ -46,12 +56,21 @@ func (d *Mega) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] if err != nil { return nil, err } - res := make([]model.Obj, 0) + fn := make(map[string]model.Obj) for i := range nodes { n := nodes[i] - if n.GetType() == mega.FILE || n.GetType() == mega.FOLDER { - res = append(res, &MegaNode{n}) + if n.GetType() != mega.FILE && n.GetType() != mega.FOLDER { + continue } + if _, ok := fn[n.GetName()]; !ok { + fn[n.GetName()] = &MegaNode{n} + } else if sameNameObj := fn[n.GetName()]; (&MegaNode{n}).ModTime().After(sameNameObj.ModTime()) { + fn[n.GetName()] = &MegaNode{n} + } + } + res := make([]model.Obj, 0) + for _, v := range fn { + res = append(res, v) } return res, nil } @@ -74,7 +93,6 @@ func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* //} size := file.GetSize() - var finalClosers utils.Closers resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { length := httpRange.Length if httpRange.Length >= 0 && httpRange.Start+httpRange.Length >= size { @@ -93,11 +111,10 @@ func (d *Mega) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* d: down, skip: httpRange.Start, } - finalClosers.Add(oo) return readers.NewLimitedReadCloser(oo, length), nil } - resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: finalClosers} + resultRangeReadCloser := &model.RangeReadCloser{RangeReader: resultRangeReader} resultLink := &model.Link{ RangeReadCloser: resultRangeReadCloser, } @@ -148,6 +165,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea return err } + reader := driver.NewLimitedUploadStream(ctx, stream) for id := 0; id < u.Chunks(); id++ { if utils.IsCanceled(ctx) { return ctx.Err() @@ -157,7 +175,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea return err } chunk := make([]byte, chkSize) - n, err := io.ReadFull(stream, chunk) + n, err := io.ReadFull(reader, chunk) if err != nil && err != io.EOF { return err } @@ -169,7 +187,7 @@ func (d *Mega) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea if err != nil { return err } - up(id * 100 / u.Chunks()) + up(float64(id) * 100 / float64(u.Chunks())) } _, err = u.Finish() diff --git a/drivers/mega/meta.go b/drivers/mega/meta.go index 77e768f0542..d0758637bb3 100644 --- a/drivers/mega/meta.go +++ b/drivers/mega/meta.go @@ -9,8 +9,10 @@ type Addition struct { // Usually one of two //driver.RootPath //driver.RootID - Email string `json:"email" required:"true"` - Password string `json:"password" required:"true"` + Email string `json:"email" required:"true"` + Password string `json:"password" required:"true"` + TwoFACode string `json:"two_fa_code" required:"false" help:"2FA 6-digit code, filling in the 2FA code alone will not support reloading driver"` + TwoFASecret string `json:"two_fa_secret" required:"false" help:"2FA secret"` } var config = driver.Config{ diff --git a/drivers/misskey/driver.go b/drivers/misskey/driver.go new file mode 100644 index 00000000000..b5c753f3c65 --- /dev/null +++ b/drivers/misskey/driver.go @@ -0,0 +1,74 @@ +package misskey + +import ( + "context" + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type Misskey struct { + model.Storage + Addition +} + +func (d *Misskey) Config() driver.Config { + return config +} + +func (d *Misskey) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Misskey) Init(ctx context.Context) error { + d.Endpoint = strings.TrimSuffix(d.Endpoint, "/") + if d.Endpoint == "" || d.AccessToken == "" { + return errs.EmptyToken + } else { + return nil + } +} + +func (d *Misskey) Drop(ctx context.Context) error { + return nil +} + +func (d *Misskey) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.list(dir) +} + +func (d *Misskey) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + return d.link(file) +} + +func (d *Misskey) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return d.makeDir(parentDir, dirName) +} + +func (d *Misskey) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.move(srcObj, dstDir) +} + +func (d *Misskey) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return d.rename(srcObj, newName) +} + +func (d *Misskey) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.copy(srcObj, dstDir) +} + +func (d *Misskey) Remove(ctx context.Context, obj model.Obj) error { + return d.remove(obj) +} + +func (d *Misskey) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return d.put(ctx, dstDir, stream, up) +} + +//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Misskey)(nil) diff --git a/drivers/misskey/meta.go b/drivers/misskey/meta.go new file mode 100644 index 00000000000..b8a80c159ff --- /dev/null +++ b/drivers/misskey/meta.go @@ -0,0 +1,35 @@ +package misskey + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootPath + // define other + // Field string `json:"field" type:"select" required:"true" options:"a,b,c" default:"a"` + Endpoint string `json:"endpoint" required:"true" default:"https://misskey.io"` + AccessToken string `json:"access_token" required:"true"` +} + +var config = driver.Config{ + Name: "Misskey", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Misskey{} + }) +} diff --git a/drivers/misskey/types.go b/drivers/misskey/types.go new file mode 100644 index 00000000000..e9adc8d2c6e --- /dev/null +++ b/drivers/misskey/types.go @@ -0,0 +1,35 @@ +package misskey + +type Resp struct { + Code int + Raw []byte +} + +type Properties struct { + Width int `json:"width"` + Height int `json:"height"` +} + +type MFile struct { + ID string `json:"id"` + CreatedAt string `json:"createdAt"` + Name string `json:"name"` + Type string `json:"type"` + MD5 string `json:"md5"` + Size int64 `json:"size"` + IsSensitive bool `json:"isSensitive"` + Blurhash string `json:"blurhash"` + Properties Properties `json:"properties"` + URL string `json:"url"` + ThumbnailURL string `json:"thumbnailUrl"` + Comment *string `json:"comment"` + FolderID *string `json:"folderId"` + Folder MFolder `json:"folder"` +} + +type MFolder struct { + ID string `json:"id"` + CreatedAt string `json:"createdAt"` + Name string `json:"name"` + ParentID *string `json:"parentId"` +} diff --git a/drivers/misskey/util.go b/drivers/misskey/util.go new file mode 100644 index 00000000000..d301955ec42 --- /dev/null +++ b/drivers/misskey/util.go @@ -0,0 +1,269 @@ +package misskey + +import ( + "context" + "errors" + "io" + "net/http" + "time" + + "github.com/go-resty/resty/v2" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// Base layer methods + +func (d *Misskey) request(path, method string, callback base.ReqCallback, resp interface{}) error { + url := d.Endpoint + "/api/drive" + path + req := base.RestyClient.R() + + req.SetAuthToken(d.AccessToken).SetHeader("Content-Type", "application/json") + + if callback != nil { + callback(req) + } else { + req.SetBody("{}") + } + + req.SetResult(resp) + + // 启用调试模式 + req.EnableTrace() + + response, err := req.Execute(method, url) + if err != nil { + return err + } + if !response.IsSuccess() { + return errors.New(response.String()) + } + return nil +} + +func (d *Misskey) getThumb(ctx context.Context, obj model.Obj) (io.Reader, error) { + // TODO return the thumb of obj, optional + return nil, errs.NotImplement +} + +func setBody(body interface{}) base.ReqCallback { + return func(req *resty.Request) { + req.SetBody(body) + } +} + +func handleFolderId(dir model.Obj) interface{} { + if isRootFolder(dir) { + return nil // Root folder doesn't need folderId + } + return dir.GetID() +} + +func isRootFolder(dir model.Obj) bool { + return dir.GetID() == "" +} + +// API layer methods + +func (d *Misskey) getFiles(dir model.Obj) ([]model.Obj, error) { + var files []MFile + var body map[string]string + if !isRootFolder(dir) { + body = map[string]string{"folderId": dir.GetID()} + } else { + body = map[string]string{} + } + err := d.request("/files", http.MethodPost, setBody(body), &files) + if err != nil { + return []model.Obj{}, err + } + return utils.SliceConvert(files, func(src MFile) (model.Obj, error) { + return mFile2Object(src), nil + }) +} + +func (d *Misskey) getFolders(dir model.Obj) ([]model.Obj, error) { + var folders []MFolder + var body map[string]string + if !isRootFolder(dir) { + body = map[string]string{"folderId": dir.GetID()} + } else { + body = map[string]string{} + } + err := d.request("/folders", http.MethodPost, setBody(body), &folders) + if err != nil { + return []model.Obj{}, err + } + return utils.SliceConvert(folders, func(src MFolder) (model.Obj, error) { + return mFolder2Object(src), nil + }) +} + +func (d *Misskey) list(dir model.Obj) ([]model.Obj, error) { + files, _ := d.getFiles(dir) + folders, _ := d.getFolders(dir) + return append(files, folders...), nil +} + +func (d *Misskey) link(file model.Obj) (*model.Link, error) { + var mFile MFile + err := d.request("/files/show", http.MethodPost, setBody(map[string]string{"fileId": file.GetID()}), &mFile) + if err != nil { + return nil, err + } + return &model.Link{ + URL: mFile.URL, + }, nil +} + +func (d *Misskey) makeDir(parentDir model.Obj, dirName string) (model.Obj, error) { + var folder MFolder + err := d.request("/folders/create", http.MethodPost, setBody(map[string]interface{}{"parentId": handleFolderId(parentDir), "name": dirName}), &folder) + if err != nil { + return nil, err + } + return mFolder2Object(folder), nil +} + +func (d *Misskey) move(srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + var folder MFolder + err := d.request("/folders/update", http.MethodPost, setBody(map[string]interface{}{"folderId": srcObj.GetID(), "parentId": handleFolderId(dstDir)}), &folder) + return mFolder2Object(folder), err + } else { + var file MFile + err := d.request("/files/update", http.MethodPost, setBody(map[string]interface{}{"fileId": srcObj.GetID(), "folderId": handleFolderId(dstDir)}), &file) + return mFile2Object(file), err + } +} + +func (d *Misskey) rename(srcObj model.Obj, newName string) (model.Obj, error) { + if srcObj.IsDir() { + var folder MFolder + err := d.request("/folders/update", http.MethodPost, setBody(map[string]string{"folderId": srcObj.GetID(), "name": newName}), &folder) + return mFolder2Object(folder), err + } else { + var file MFile + err := d.request("/files/update", http.MethodPost, setBody(map[string]string{"fileId": srcObj.GetID(), "name": newName}), &file) + return mFile2Object(file), err + } +} + +func (d *Misskey) copy(srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + folder, err := d.makeDir(dstDir, srcObj.GetName()) + if err != nil { + return nil, err + } + list, err := d.list(srcObj) + if err != nil { + return nil, err + } + for _, obj := range list { + _, err := d.copy(obj, folder) + if err != nil { + return nil, err + } + } + return folder, nil + } else { + var file MFile + url, err := d.link(srcObj) + if err != nil { + return nil, err + } + err = d.request("/files/upload-from-url", http.MethodPost, setBody(map[string]interface{}{"url": url.URL, "folderId": handleFolderId(dstDir)}), &file) + if err != nil { + return nil, err + } + return mFile2Object(file), nil + } +} + +func (d *Misskey) remove(obj model.Obj) error { + if obj.IsDir() { + err := d.request("/folders/delete", http.MethodPost, setBody(map[string]string{"folderId": obj.GetID()}), nil) + return err + } else { + err := d.request("/files/delete", http.MethodPost, setBody(map[string]string{"fileId": obj.GetID()}), nil) + return err + } +} + +func (d *Misskey) put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var file MFile + + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: stream, + UpdateProgress: up, + }) + + // Build form data, only add folderId if not root folder + formData := map[string]string{ + "name": stream.GetName(), + "comment": "", + "isSensitive": "false", + "force": "false", + } + + folderId := handleFolderId(dstDir) + if folderId != nil { + formData["folderId"] = folderId.(string) + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetFileReader("file", stream.GetName(), reader). + SetFormData(formData). + SetResult(&file). + SetAuthToken(d.AccessToken) + + resp, err := req.Post(d.Endpoint + "/api/drive/files/create") + if err != nil { + return nil, err + } + if !resp.IsSuccess() { + return nil, errors.New(resp.String()) + } + + return mFile2Object(file), nil +} + +func mFile2Object(file MFile) *model.ObjThumbURL { + ctime, err := time.Parse(time.RFC3339, file.CreatedAt) + if err != nil { + ctime = time.Time{} + } + return &model.ObjThumbURL{ + Object: model.Object{ + ID: file.ID, + Name: file.Name, + Ctime: ctime, + IsFolder: false, + Size: file.Size, + }, + Thumbnail: model.Thumbnail{ + Thumbnail: file.ThumbnailURL, + }, + Url: model.Url{ + Url: file.URL, + }, + } +} + +func mFolder2Object(folder MFolder) *model.Object { + ctime, err := time.Parse(time.RFC3339, folder.CreatedAt) + if err != nil { + ctime = time.Time{} + } + return &model.Object{ + ID: folder.ID, + Name: folder.Name, + Ctime: ctime, + IsFolder: true, + } +} diff --git a/drivers/mopan/driver.go b/drivers/mopan/driver.go index bd2de2b30af..f8f14300571 100644 --- a/drivers/mopan/driver.go +++ b/drivers/mopan/driver.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "golang.org/x/sync/semaphore" + "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" @@ -43,23 +45,31 @@ func (d *MoPan) Init(ctx context.Context) error { if d.uploadThread < 1 || d.uploadThread > 32 { d.uploadThread, d.UploadThread = 3, "3" } - login := func() error { - data, err := d.client.Login(d.Phone, d.Password) + + defer func() { d.SMSCode = "" }() + + login := func() (err error) { + var loginData *mopan.LoginResp + if d.SMSCode != "" { + loginData, err = d.client.LoginBySmsStep2(d.Phone, d.SMSCode) + } else { + loginData, err = d.client.Login(d.Phone, d.Password) + } if err != nil { return err } - d.client.SetAuthorization(data.Token) + d.client.SetAuthorization(loginData.Token) info, err := d.client.GetUserInfo() if err != nil { return err } d.userID = info.UserID - log.Debugf("[mopan] Phone: %s UserCloudStorageRelations: %+v", d.Phone, data.UserCloudStorageRelations) + log.Debugf("[mopan] Phone: %s UserCloudStorageRelations: %+v", d.Phone, loginData.UserCloudStorageRelations) cloudCircleApp, _ := d.client.QueryAllCloudCircleApp() log.Debugf("[mopan] Phone: %s CloudCircleApp: %+v", d.Phone, cloudCircleApp) if d.RootFolderID == "" { - for _, userCloudStorage := range data.UserCloudStorageRelations { + for _, userCloudStorage := range loginData.UserCloudStorageRelations { if userCloudStorage.Path == "/文件" { d.RootFolderID = userCloudStorage.FolderID } @@ -76,8 +86,20 @@ func (d *MoPan) Init(ctx context.Context) error { op.MustSaveDriverStorage(d) } return err - }).SetDeviceInfo(d.DeviceInfo) - d.DeviceInfo = d.client.GetDeviceInfo() + }) + + var deviceInfo mopan.DeviceInfo + if strings.TrimSpace(d.DeviceInfo) != "" && utils.Json.UnmarshalFromString(d.DeviceInfo, &deviceInfo) == nil { + d.client.SetDeviceInfo(&deviceInfo) + } + d.DeviceInfo, _ = utils.Json.MarshalToString(d.client.GetDeviceInfo()) + + if strings.Contains(d.SMSCode, "send") { + if _, err := d.client.LoginBySms(d.Phone); err != nil { + return err + } + return errors.New("please enter the SMS code") + } return login() } @@ -119,10 +141,13 @@ func (d *MoPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) ( } data.DownloadUrl = strings.Replace(strings.ReplaceAll(data.DownloadUrl, "&", "&"), "http://", "https://", 1) - res, err := base.NoRedirectClient.R().SetContext(ctx).Head(data.DownloadUrl) + res, err := base.NoRedirectClient.R().SetDoNotParseResponse(true).SetContext(ctx).Get(data.DownloadUrl) if err != nil { return nil, err } + defer func() { + _ = res.RawBody().Close() + }() if res.StatusCode() == 302 { data.DownloadUrl = res.Header().Get("location") } @@ -244,9 +269,6 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre if err != nil { return nil, err } - defer func() { - _ = file.Close() - }() // step.1 uploadPartData, err := mopan.InitUploadPartData(ctx, mopan.UpdloadFileParam{ @@ -272,12 +294,13 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre } if !initUpdload.FileDataExists { - utils.Log.Error(d.client.CloudDiskStartBusiness()) + // utils.Log.Error(d.client.CloudDiskStartBusiness()) threadG, upCtx := errgroup.NewGroupWithContext(ctx, d.uploadThread, retry.Attempts(3), retry.Delay(time.Second), retry.DelayType(retry.BackOffDelay)) + sem := semaphore.NewWeighted(3) // step.3 parts, err := d.client.GetAllMultiUploadUrls(initUpdload.UploadFileID, initUpdload.PartInfos) @@ -296,19 +319,25 @@ func (d *MoPan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre // step.4 threadG.Go(func(ctx context.Context) error { - req, err := part.NewRequest(ctx, io.NewSectionReader(file, int64(part.PartNumber-1)*initUpdload.PartSize, byteSize)) + if err = sem.Acquire(ctx, 1); err != nil { + return err + } + defer sem.Release(1) + reader := io.NewSectionReader(file, int64(part.PartNumber-1)*initUpdload.PartSize, byteSize) + req, err := part.NewRequest(ctx, driver.NewLimitedUploadStream(ctx, reader)) if err != nil { return err } + req.ContentLength = byteSize resp, err := base.HttpClient.Do(req) if err != nil { return err } - resp.Body.Close() + _ = resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("upload err,code=%d", resp.StatusCode) } - up(100 * int(threadG.Success()) / len(parts)) + up(100 * float64(threadG.Success()) / float64(len(parts))) initUpdload.PartInfos[i] = "" return nil }) diff --git a/drivers/mopan/meta.go b/drivers/mopan/meta.go index e6583fc1b8c..c111fedc16a 100644 --- a/drivers/mopan/meta.go +++ b/drivers/mopan/meta.go @@ -8,6 +8,7 @@ import ( type Addition struct { Phone string `json:"phone" required:"true"` Password string `json:"password" required:"true"` + SMSCode string `json:"sms_code" help:"input 'send' send sms "` RootFolderID string `json:"root_folder_id" default:""` diff --git a/drivers/netease_music/crypto.go b/drivers/netease_music/crypto.go new file mode 100644 index 00000000000..76ff65486ac --- /dev/null +++ b/drivers/netease_music/crypto.go @@ -0,0 +1,135 @@ +package netease_music + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/md5" + "crypto/rsa" + "crypto/x509" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "math/big" + "strings" + + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" +) + +var ( + linuxapiKey = []byte("rFgB&h#%2?^eDg:Q") + eapiKey = []byte("e82ckenh8dichen8") + iv = []byte("0102030405060708") + presetKey = []byte("0CoJUm6Qyw8W8jud") + publicKey = []byte("-----BEGIN PUBLIC KEY-----\nMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDgtQn2JZ34ZC28NWYpAUd98iZ37BUrX/aKzmFbt7clFSs6sXqHauqKWqdtLkF2KexO40H1YTX8z2lSgBBOAxLsvaklV8k4cBFK9snQXE9/DDaFt6Rr7iVZMldczhC0JNgTz+SHXT6CBHuX3e9SdB1Ua44oncaTWz7OBGLbCiK45wIDAQAB\n-----END PUBLIC KEY-----") + stdChars = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") +) + +func aesKeyPending(key []byte) []byte { + k := len(key) + count := 0 + switch true { + case k <= 16: + count = 16 - k + case k <= 24: + count = 24 - k + case k <= 32: + count = 32 - k + default: + return key[:32] + } + if count == 0 { + return key + } + + return append(key, bytes.Repeat([]byte{0}, count)...) +} + +func pkcs7Padding(src []byte, blockSize int) []byte { + padding := blockSize - len(src)%blockSize + padtext := bytes.Repeat([]byte{byte(padding)}, padding) + return append(src, padtext...) +} + +func aesCBCEncrypt(src, key, iv []byte) []byte { + block, _ := aes.NewCipher(aesKeyPending(key)) + src = pkcs7Padding(src, block.BlockSize()) + dst := make([]byte, len(src)) + + mode := cipher.NewCBCEncrypter(block, iv) + mode.CryptBlocks(dst, src) + + return dst +} + +func aesECBEncrypt(src, key []byte) []byte { + block, _ := aes.NewCipher(aesKeyPending(key)) + + src = pkcs7Padding(src, block.BlockSize()) + dst := make([]byte, len(src)) + + ecbCryptBlocks(block, dst, src) + + return dst +} + +func ecbCryptBlocks(block cipher.Block, dst, src []byte) { + bs := block.BlockSize() + + for len(src) > 0 { + block.Encrypt(dst, src[:bs]) + src = src[bs:] + dst = dst[bs:] + } +} + +func rsaEncrypt(buffer, key []byte) []byte { + buffers := make([]byte, 128-16, 128) + buffers = append(buffers, buffer...) + block, _ := pem.Decode(key) + pubInterface, _ := x509.ParsePKIXPublicKey(block.Bytes) + pub := pubInterface.(*rsa.PublicKey) + c := new(big.Int).SetBytes([]byte(buffers)) + return c.Exp(c, big.NewInt(int64(pub.E)), pub.N).Bytes() +} + +func getSecretKey() ([]byte, []byte) { + key := make([]byte, 16) + reversed := make([]byte, 16) + for i := 0; i < 16; i++ { + result := stdChars[random.RangeInt64(0, 62)] + key[i] = result + reversed[15-i] = result + } + return key, reversed +} + +func weapi(data map[string]string) map[string]string { + text, _ := utils.Json.Marshal(data) + secretKey, reversedKey := getSecretKey() + params := []byte(base64.StdEncoding.EncodeToString(aesCBCEncrypt(text, presetKey, iv))) + return map[string]string{ + "params": base64.StdEncoding.EncodeToString(aesCBCEncrypt(params, reversedKey, iv)), + "encSecKey": hex.EncodeToString(rsaEncrypt(secretKey, publicKey)), + } +} + +func eapi(url string, data map[string]interface{}) map[string]string { + text, _ := utils.Json.Marshal(data) + msg := "nobody" + url + "use" + string(text) + "md5forencrypt" + h := md5.New() + h.Write([]byte(msg)) + digest := hex.EncodeToString(h.Sum(nil)) + params := []byte(url + "-36cd479b6b5-" + string(text) + "-36cd479b6b5-" + digest) + return map[string]string{ + "params": hex.EncodeToString(aesECBEncrypt(params, eapiKey)), + } +} + +func linuxapi(data map[string]interface{}) map[string]string { + text, _ := utils.Json.Marshal(data) + return map[string]string{ + "eparams": strings.ToUpper(hex.EncodeToString(aesECBEncrypt(text, linuxapiKey))), + } +} diff --git a/drivers/netease_music/driver.go b/drivers/netease_music/driver.go new file mode 100644 index 00000000000..08460cceee9 --- /dev/null +++ b/drivers/netease_music/driver.go @@ -0,0 +1,110 @@ +package netease_music + +import ( + "context" + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + _ "golang.org/x/image/webp" +) + +type NeteaseMusic struct { + model.Storage + Addition + + csrfToken string + musicU string + fileMapByName map[string]model.Obj +} + +func (d *NeteaseMusic) Config() driver.Config { + return config +} + +func (d *NeteaseMusic) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *NeteaseMusic) Init(ctx context.Context) error { + d.csrfToken = d.Addition.getCookie("__csrf") + d.musicU = d.Addition.getCookie("MUSIC_U") + + if d.csrfToken == "" || d.musicU == "" { + return errs.EmptyToken + } + + return nil +} + +func (d *NeteaseMusic) Drop(ctx context.Context) error { + return nil +} + +func (d *NeteaseMusic) Get(ctx context.Context, path string) (model.Obj, error) { + if path == "/" { + return &model.Object{ + IsFolder: true, + Path: path, + }, nil + } + + fragments := strings.Split(path, "/") + if len(fragments) > 1 { + fileName := fragments[1] + if strings.HasSuffix(fileName, ".lrc") { + lrc := d.fileMapByName[fileName] + return d.getLyricObj(lrc) + } + if song, ok := d.fileMapByName[fileName]; ok { + return song, nil + } else { + return nil, errs.ObjectNotFound + } + } + + return nil, errs.ObjectNotFound +} + +func (d *NeteaseMusic) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return d.getSongObjs(args) +} + +func (d *NeteaseMusic) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if lrc, ok := file.(*LyricObj); ok { + if args.Type == "parsed" { + return lrc.getLyricLink(), nil + } else { + return lrc.getProxyLink(args), nil + } + } + + return d.getSongLink(file) +} + +func (d *NeteaseMusic) Remove(ctx context.Context, obj model.Obj) error { + return d.removeSongObj(obj) +} + +func (d *NeteaseMusic) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + return d.putSongStream(ctx, stream, up) +} + +func (d *NeteaseMusic) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *NeteaseMusic) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *NeteaseMusic) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + return errs.NotSupport +} + +func (d *NeteaseMusic) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + return errs.NotSupport +} + +var _ driver.Driver = (*NeteaseMusic)(nil) diff --git a/drivers/netease_music/meta.go b/drivers/netease_music/meta.go new file mode 100644 index 00000000000..8ddfd728178 --- /dev/null +++ b/drivers/netease_music/meta.go @@ -0,0 +1,32 @@ +package netease_music + +import ( + "regexp" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + Cookie string `json:"cookie" type:"text" required:"true" help:""` + SongLimit uint64 `json:"song_limit" default:"200" type:"number" help:"only get 200 songs by default"` +} + +func (ad *Addition) getCookie(name string) string { + re := regexp.MustCompile(name + "=([^(;|$)]+)") + matches := re.FindStringSubmatch(ad.Cookie) + if len(matches) < 2 { + return "" + } + return matches[1] +} + +var config = driver.Config{ + Name: "NeteaseMusic", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &NeteaseMusic{} + }) +} diff --git a/drivers/netease_music/types.go b/drivers/netease_music/types.go new file mode 100644 index 00000000000..93ecdf702ff --- /dev/null +++ b/drivers/netease_music/types.go @@ -0,0 +1,118 @@ +package netease_music + +import ( + "context" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/server/common" +) + +type HostsResp struct { + Upload []string `json:"upload"` +} + +type SongResp struct { + Data []struct { + Url string `json:"url"` + } `json:"data"` +} + +type ListResp struct { + Size int64 `json:"size"` + MaxSize int64 `json:"maxSize"` + Data []struct { + AddTime int64 `json:"addTime"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + SongId int64 `json:"songId"` + SimpleSong struct { + Al struct { + PicUrl string `json:"picUrl"` + } `json:"al"` + } `json:"simpleSong"` + } `json:"data"` +} + +type LyricObj struct { + model.Object + lyric string +} + +func (lrc *LyricObj) getProxyLink(args model.LinkArgs) *model.Link { + rawURL := common.GetApiUrl(args.HttpReq) + "/p" + lrc.Path + rawURL = utils.EncodePath(rawURL, true) + "?type=parsed&sign=" + sign.Sign(lrc.Path) + return &model.Link{URL: rawURL} +} + +func (lrc *LyricObj) getLyricLink() *model.Link { + reader := strings.NewReader(lrc.lyric) + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{ + RangeReader: func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + if httpRange.Length < 0 { + return io.NopCloser(reader), nil + } + sr := io.NewSectionReader(reader, httpRange.Start, httpRange.Length) + return io.NopCloser(sr), nil + }, + }, + } +} + +type ReqOption struct { + crypto string + stream model.FileStreamer + up driver.UpdateProgress + ctx context.Context + data map[string]string + headers map[string]string + cookies []*http.Cookie + url string +} + +type Characteristic map[string]string + +func (ch *Characteristic) fromDriver(d *NeteaseMusic) *Characteristic { + *ch = map[string]string{ + "osver": "", + "deviceId": "", + "mobilename": "", + "appver": "6.1.1", + "versioncode": "140", + "buildver": strconv.FormatInt(time.Now().Unix(), 10), + "resolution": "1920x1080", + "os": "android", + "channel": "", + "requestId": strconv.FormatInt(time.Now().Unix()*1000, 10) + strconv.Itoa(int(random.RangeInt64(0, 1000))), + "MUSIC_U": d.musicU, + } + return ch +} + +func (ch Characteristic) toCookies() []*http.Cookie { + cookies := make([]*http.Cookie, 0) + for k, v := range ch { + cookies = append(cookies, &http.Cookie{Name: k, Value: v}) + } + return cookies +} + +func (ch *Characteristic) merge(data map[string]string) map[string]interface{} { + body := map[string]interface{}{ + "header": ch, + } + for k, v := range data { + body[k] = v + } + return body +} diff --git a/drivers/netease_music/upload.go b/drivers/netease_music/upload.go new file mode 100644 index 00000000000..3ff6216b7b9 --- /dev/null +++ b/drivers/netease_music/upload.go @@ -0,0 +1,215 @@ +package netease_music + +import ( + "context" + "crypto/md5" + "encoding/hex" + "github.com/alist-org/alist/v3/internal/driver" + "io" + "net/http" + "strconv" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/dhowden/tag" +) + +type token struct { + resourceId string + objectKey string + token string +} + +type songmeta struct { + needUpload bool + songId string + name string + artist string + album string +} + +type uploader struct { + driver *NeteaseMusic + file model.File + meta songmeta + md5 string + ext string + size string + filename string +} + +func (u *uploader) init(stream model.FileStreamer) error { + u.filename = stream.GetName() + u.size = strconv.FormatInt(stream.GetSize(), 10) + + u.ext = "mp3" + if strings.HasSuffix(stream.GetMimetype(), "flac") { + u.ext = "flac" + } + + h := md5.New() + _, err := utils.CopyWithBuffer(h, stream) + if err != nil { + return err + } + u.md5 = hex.EncodeToString(h.Sum(nil)) + _, err = u.file.Seek(0, io.SeekStart) + if err != nil { + return err + } + + if m, err := tag.ReadFrom(u.file); err != nil { + u.meta = songmeta{} + } else { + u.meta = songmeta{ + name: m.Title(), + artist: m.Artist(), + album: m.Album(), + } + } + if u.meta.name == "" { + u.meta.name = u.filename + } + if u.meta.album == "" { + u.meta.album = "未知专辑" + } + if u.meta.artist == "" { + u.meta.artist = "未知艺术家" + } + _, err = u.file.Seek(0, io.SeekStart) + if err != nil { + return err + } + + return nil +} + +func (u *uploader) checkIfExisted() error { + body, err := u.driver.request("https://interface.music.163.com/api/cloud/upload/check", http.MethodPost, + ReqOption{ + crypto: "weapi", + data: map[string]string{ + "ext": "", + "songId": "0", + "version": "1", + "bitrate": "999000", + "length": u.size, + "md5": u.md5, + }, + cookies: []*http.Cookie{ + {Name: "os", Value: "pc"}, + {Name: "appver", Value: "2.9.7"}, + }, + }, + ) + if err != nil { + return err + } + + u.meta.songId = utils.Json.Get(body, "songId").ToString() + u.meta.needUpload = utils.Json.Get(body, "needUpload").ToBool() + + return nil +} + +func (u *uploader) allocToken(bucket ...string) (token, error) { + if len(bucket) == 0 { + bucket = []string{""} + } + + body, err := u.driver.request("https://music.163.com/weapi/nos/token/alloc", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "bucket": bucket[0], + "local": "false", + "type": "audio", + "nos_product": "3", + "filename": u.filename, + "md5": u.md5, + "ext": u.ext, + }, + }) + if err != nil { + return token{}, err + } + + return token{ + resourceId: utils.Json.Get(body, "result", "resourceId").ToString(), + objectKey: utils.Json.Get(body, "result", "objectKey").ToString(), + token: utils.Json.Get(body, "result", "token").ToString(), + }, nil +} + +func (u *uploader) publishInfo(resourceId string) error { + body, err := u.driver.request("https://music.163.com/api/upload/cloud/info/v2", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "md5": u.md5, + "filename": u.filename, + "song": u.meta.name, + "album": u.meta.album, + "artist": u.meta.artist, + "songid": u.meta.songId, + "resourceId": resourceId, + "bitrate": "999000", + }, + }) + if err != nil { + return err + } + + _, err = u.driver.request("https://interface.music.163.com/api/cloud/pub/v2", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "songid": utils.Json.Get(body, "songId").ToString(), + }, + }) + if err != nil { + return err + } + + return nil +} + +func (u *uploader) upload(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) error { + bucket := "jd-musicrep-privatecloud-audio-public" + token, err := u.allocToken(bucket) + if err != nil { + return err + } + + body, err := u.driver.request("https://wanproxy.127.net/lbs?version=1.0&bucketname="+bucket, http.MethodGet, + ReqOption{}, + ) + if err != nil { + return err + } + var resp HostsResp + err = utils.Json.Unmarshal(body, &resp) + if err != nil { + return err + } + + objectKey := strings.ReplaceAll(token.objectKey, "/", "%2F") + _, err = u.driver.request( + resp.Upload[0]+"/"+bucket+"/"+objectKey+"?offset=0&complete=true&version=1.0", + http.MethodPost, + ReqOption{ + stream: stream, + up: up, + ctx: ctx, + headers: map[string]string{ + "x-nos-token": token.token, + "Content-Type": "audio/mpeg", + "Content-Length": u.size, + "Content-MD5": u.md5, + }, + }, + ) + if err != nil { + return err + } + + return nil +} diff --git a/drivers/netease_music/util.go b/drivers/netease_music/util.go new file mode 100644 index 00000000000..217181062ca --- /dev/null +++ b/drivers/netease_music/util.go @@ -0,0 +1,261 @@ +package netease_music + +import ( + "context" + "net/http" + "path" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func (d *NeteaseMusic) request(url, method string, opt ReqOption) ([]byte, error) { + req := base.RestyClient.R() + + req.SetHeader("Cookie", d.Addition.Cookie) + + if strings.Contains(url, "music.163.com") { + req.SetHeader("Referer", "https://music.163.com") + } + + if opt.cookies != nil { + for _, cookie := range opt.cookies { + req.SetCookie(cookie) + } + } + + if opt.headers != nil { + for header, value := range opt.headers { + req.SetHeader(header, value) + } + } + + data := opt.data + if opt.crypto == "weapi" { + data = weapi(data) + re, _ := regexp.Compile(`/\w*api/`) + url = re.ReplaceAllString(url, "/weapi/") + } else if opt.crypto == "eapi" { + ch := new(Characteristic).fromDriver(d) + req.SetCookies(ch.toCookies()) + data = eapi(opt.url, ch.merge(data)) + re, _ := regexp.Compile(`/\w*api/`) + url = re.ReplaceAllString(url, "/eapi/") + } else if opt.crypto == "linuxapi" { + re, _ := regexp.Compile(`/\w*api/`) + data = linuxapi(map[string]interface{}{ + "url": re.ReplaceAllString(url, "/api/"), + "method": method, + "params": data, + }) + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36") + url = "https://music.163.com/api/linux/forward" + } + + if opt.ctx != nil { + req.SetContext(opt.ctx) + } + if method == http.MethodPost { + if opt.stream != nil { + if opt.up == nil { + opt.up = func(_ float64) {} + } + req.SetContentLength(true) + req.SetBody(driver.NewLimitedUploadStream(opt.ctx, &driver.ReaderUpdatingProgress{ + Reader: opt.stream, + UpdateProgress: opt.up, + })) + } else { + req.SetFormData(data) + } + res, err := req.Post(url) + if err != nil { + return nil, err + } + return res.Body(), nil + } + + if method == http.MethodGet { + res, err := req.Get(url) + if err != nil { + return nil, err + } + return res.Body(), nil + } + + return nil, errs.NotImplement +} + +func (d *NeteaseMusic) getSongObjs(args model.ListArgs) ([]model.Obj, error) { + body, err := d.request("https://music.163.com/weapi/v1/cloud/get", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "limit": strconv.FormatUint(d.Addition.SongLimit, 10), + "offset": "0", + }, + cookies: []*http.Cookie{ + {Name: "os", Value: "pc"}, + }, + }) + if err != nil { + return nil, err + } + + var resp ListResp + err = utils.Json.Unmarshal(body, &resp) + if err != nil { + return nil, err + } + + d.fileMapByName = make(map[string]model.Obj) + files := make([]model.Obj, 0, len(resp.Data)) + for _, f := range resp.Data { + song := &model.ObjThumb{ + Object: model.Object{ + IsFolder: false, + Size: f.FileSize, + Name: f.FileName, + Modified: time.UnixMilli(f.AddTime), + ID: strconv.FormatInt(f.SongId, 10), + }, + Thumbnail: model.Thumbnail{Thumbnail: f.SimpleSong.Al.PicUrl}, + } + d.fileMapByName[song.Name] = song + files = append(files, song) + + // map song id for lyric + lrcName := strings.Split(f.FileName, ".")[0] + ".lrc" + lrc := &model.Object{ + IsFolder: false, + Name: lrcName, + Path: path.Join(args.ReqPath, lrcName), + ID: strconv.FormatInt(f.SongId, 10), + } + d.fileMapByName[lrc.Name] = lrc + } + + return files, nil +} + +func (d *NeteaseMusic) getSongLink(file model.Obj) (*model.Link, error) { + body, err := d.request( + "https://music.163.com/api/song/enhance/player/url", http.MethodPost, ReqOption{ + crypto: "linuxapi", + data: map[string]string{ + "ids": "[" + file.GetID() + "]", + "br": "999000", + }, + cookies: []*http.Cookie{ + {Name: "os", Value: "pc"}, + }, + }, + ) + if err != nil { + return nil, err + } + + var resp SongResp + err = utils.Json.Unmarshal(body, &resp) + if err != nil { + return nil, err + } + + if len(resp.Data) < 1 { + return nil, errs.ObjectNotFound + } + + return &model.Link{URL: resp.Data[0].Url}, nil +} + +func (d *NeteaseMusic) getLyricObj(file model.Obj) (model.Obj, error) { + if lrc, ok := file.(*LyricObj); ok { + return lrc, nil + } + + body, err := d.request( + "https://music.163.com/api/song/lyric?_nmclfl=1", http.MethodPost, ReqOption{ + data: map[string]string{ + "id": file.GetID(), + "tv": "-1", + "lv": "-1", + "rv": "-1", + "kv": "-1", + }, + cookies: []*http.Cookie{ + {Name: "os", Value: "ios"}, + }, + }, + ) + if err != nil { + return nil, err + } + + lyric := utils.Json.Get(body, "lrc", "lyric").ToString() + + return &LyricObj{ + lyric: lyric, + Object: model.Object{ + IsFolder: false, + ID: file.GetID(), + Name: file.GetName(), + Path: file.GetPath(), + Size: int64(len(lyric)), + }, + }, nil +} + +func (d *NeteaseMusic) removeSongObj(file model.Obj) error { + _, err := d.request("http://music.163.com/weapi/cloud/del", http.MethodPost, ReqOption{ + crypto: "weapi", + data: map[string]string{ + "songIds": "[" + file.GetID() + "]", + }, + }) + + return err +} + +func (d *NeteaseMusic) putSongStream(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) error { + tmp, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + + u := uploader{driver: d, file: tmp} + + err = u.init(stream) + if err != nil { + return err + } + + err = u.checkIfExisted() + if err != nil { + return err + } + + token, err := u.allocToken() + if err != nil { + return err + } + + if u.meta.needUpload { + err = u.upload(ctx, stream, up) + if err != nil { + return err + } + } + + err = u.publishInfo(token.resourceId) + if err != nil { + return err + } + + return nil +} diff --git a/drivers/onedrive/driver.go b/drivers/onedrive/driver.go index 50e129d99c5..adbe0342e1c 100644 --- a/drivers/onedrive/driver.go +++ b/drivers/onedrive/driver.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "path" + "sync" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -19,6 +20,8 @@ type Onedrive struct { model.Storage Addition AccessToken string + root *Object + mutex sync.Mutex } func (d *Onedrive) Config() driver.Config { @@ -40,6 +43,42 @@ func (d *Onedrive) Drop(ctx context.Context) error { return nil } +func (d *Onedrive) GetRoot(ctx context.Context) (model.Obj, error) { + if d.root != nil { + return d.root, nil + } + d.mutex.Lock() + defer d.mutex.Unlock() + root := &Object{ + ObjThumb: model.ObjThumb{ + Object: model.Object{ + ID: "root", + Path: d.RootFolderPath, + Name: "root", + Size: 0, + Modified: d.Modified, + Ctime: d.Modified, + IsFolder: true, + }, + }, + ParentID: "", + } + if !utils.PathEqual(d.RootFolderPath, "/") { + // get root folder id + url := d.GetMetaUrl(false, d.RootFolderPath) + var resp struct { + Id string `json:"id"` + } + _, err := d.Request(url, http.MethodGet, nil, &resp) + if err != nil { + return nil, err + } + root.ID = resp.Id + } + d.root = root + return d.root, nil +} + func (d *Onedrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { files, err := d.getFiles(dir.GetPath()) if err != nil { @@ -79,6 +118,7 @@ func (d *Onedrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName str "folder": base.Json{}, "@microsoft.graph.conflictBehavior": "rename", } + // todo 修复文件夹 ctime/mtime, onedrive 可在 data 里设置 fileSystemInfo 字段, 但是此接口未提供 ctime/mtime _, err := d.Request(url, http.MethodPost, func(req *resty.Request) { req.SetBody(data) }, nil) diff --git a/drivers/onedrive/meta.go b/drivers/onedrive/meta.go index a60e5f33a93..54a7340a942 100644 --- a/drivers/onedrive/meta.go +++ b/drivers/onedrive/meta.go @@ -11,7 +11,7 @@ type Addition struct { IsSharepoint bool `json:"is_sharepoint"` ClientID string `json:"client_id" required:"true"` ClientSecret string `json:"client_secret" required:"true"` - RedirectUri string `json:"redirect_uri" required:"true" default:"https://alist.nn.ci/tool/onedrive/callback"` + RedirectUri string `json:"redirect_uri" required:"true" default:"https://alistgo.com/tool/onedrive/callback"` RefreshToken string `json:"refresh_token" required:"true"` SiteId string `json:"site_id"` ChunkSize int64 `json:"chunk_size" type:"number" default:"5"` diff --git a/drivers/onedrive/types.go b/drivers/onedrive/types.go index 69264abcf03..aedd5a06802 100644 --- a/drivers/onedrive/types.go +++ b/drivers/onedrive/types.go @@ -24,12 +24,12 @@ type RespErr struct { } type File struct { - Id string `json:"id"` - Name string `json:"name"` - Size int64 `json:"size"` - LastModifiedDateTime time.Time `json:"lastModifiedDateTime"` - Url string `json:"@microsoft.graph.downloadUrl"` - File *struct { + Id string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo"` + Url string `json:"@microsoft.graph.downloadUrl"` + File *struct { MimeType string `json:"mimeType"` } `json:"file"` Thumbnails []struct { @@ -58,7 +58,7 @@ func fileToObj(f File, parentID string) *Object { ID: f.Id, Name: f.Name, Size: f.Size, - Modified: f.LastModifiedDateTime, + Modified: f.FileSystemInfo.LastModifiedDateTime, IsFolder: f.File == nil, }, Thumbnail: model.Thumbnail{Thumbnail: thumb}, @@ -72,3 +72,20 @@ type Files struct { Value []File `json:"value"` NextLink string `json:"@odata.nextLink"` } + +// Metadata represents a request to update Metadata. +// It includes only the writeable properties. +// omitempty is intentionally included for all, per https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update?view=odsp-graph-online#request-body +type Metadata struct { + Description string `json:"description,omitempty"` // Provides a user-visible description of the item. Read-write. Only on OneDrive Personal. Undocumented limit of 1024 characters. + FileSystemInfo *FileSystemInfoFacet `json:"fileSystemInfo,omitempty"` // File system information on client. Read-write. +} + +// FileSystemInfoFacet contains properties that are reported by the +// device's local file system for the local version of an item. This +// facet can be used to specify the last modified date or created date +// of the item as it was on the local device. +type FileSystemInfoFacet struct { + CreatedDateTime time.Time `json:"createdDateTime,omitempty"` // The UTC date and time the file was created on a client. + LastModifiedDateTime time.Time `json:"lastModifiedDateTime,omitempty"` // The UTC date and time the file was last modified on a client. +} diff --git a/drivers/onedrive/util.go b/drivers/onedrive/util.go index 0539e098682..28ed5ccc3cc 100644 --- a/drivers/onedrive/util.go +++ b/drivers/onedrive/util.go @@ -8,7 +8,7 @@ import ( "io" "net/http" stdpath "path" - "strconv" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" @@ -18,7 +18,6 @@ import ( "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" - log "github.com/sirupsen/logrus" ) var onedriveHostMap = map[string]Host{ @@ -127,7 +126,7 @@ func (d *Onedrive) Request(url string, method string, callback base.ReqCallback, func (d *Onedrive) getFiles(path string) ([]File, error) { var res []File - nextLink := d.GetMetaUrl(false, path) + "/children?$top=5000&$expand=thumbnails($select=medium)&$select=id,name,size,lastModifiedDateTime,content.downloadUrl,file,parentReference" + nextLink := d.GetMetaUrl(false, path) + "/children?$top=1000&$expand=thumbnails($select=medium)&$select=id,name,size,fileSystemInfo,content.downloadUrl,file,parentReference" for nextLink != "" { var files Files _, err := d.Request(nextLink, http.MethodGet, nil, &files) @@ -148,62 +147,111 @@ func (d *Onedrive) GetFile(path string) (*File, error) { } func (d *Onedrive) upSmall(ctx context.Context, dstDir model.Obj, stream model.FileStreamer) error { - url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/content" - data, err := io.ReadAll(stream) + filepath := stdpath.Join(dstDir.GetPath(), stream.GetName()) + // 1. upload new file + // ApiDoc: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online + url := d.GetMetaUrl(false, filepath) + "/content" + _, err := d.Request(url, http.MethodPut, func(req *resty.Request) { + req.SetBody(driver.NewLimitedUploadStream(ctx, stream)).SetContext(ctx) + }, nil) if err != nil { - return err + return fmt.Errorf("onedrive: Failed to upload new file(path=%v): %w", filepath, err) + } + + // 2. update metadata + err = d.updateMetadata(ctx, stream, filepath) + if err != nil { + return fmt.Errorf("onedrive: Failed to update file(path=%v) metadata: %w", filepath, err) } - _, err = d.Request(url, http.MethodPut, func(req *resty.Request) { - req.SetBody(data).SetContext(ctx) + return nil +} + +func (d *Onedrive) updateMetadata(ctx context.Context, stream model.FileStreamer, filepath string) error { + url := d.GetMetaUrl(false, filepath) + metadata := toAPIMetadata(stream) + // ApiDoc: https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_update?view=odsp-graph-online + _, err := d.Request(url, http.MethodPatch, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) }, nil) return err } +func toAPIMetadata(stream model.FileStreamer) Metadata { + metadata := Metadata{ + FileSystemInfo: &FileSystemInfoFacet{}, + } + if !stream.ModTime().IsZero() { + metadata.FileSystemInfo.LastModifiedDateTime = stream.ModTime() + } + if !stream.CreateTime().IsZero() { + metadata.FileSystemInfo.CreatedDateTime = stream.CreateTime() + } + if stream.CreateTime().IsZero() && !stream.ModTime().IsZero() { + metadata.FileSystemInfo.CreatedDateTime = stream.CreateTime() + } + return metadata +} + func (d *Onedrive) upBig(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { url := d.GetMetaUrl(false, stdpath.Join(dstDir.GetPath(), stream.GetName())) + "/createUploadSession" - res, err := d.Request(url, http.MethodPost, nil, nil) + metadata := map[string]interface{}{"item": toAPIMetadata(stream)} + res, err := d.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(metadata).SetContext(ctx) + }, nil) if err != nil { return err } uploadUrl := jsoniter.Get(res, "uploadUrl").ToString() var finish int64 = 0 DEFAULT := d.ChunkSize * 1024 * 1024 + retryCount := 0 + maxRetries := 3 for finish < stream.GetSize() { if utils.IsCanceled(ctx) { return ctx.Err() } - log.Debugf("upload: %d", finish) - var byteSize int64 = DEFAULT left := stream.GetSize() - finish - if left < DEFAULT { - byteSize = left - } + byteSize := min(left, DEFAULT) + utils.Log.Debugf("[Onedrive] upload range: %d-%d/%d", finish, finish+byteSize-1, stream.GetSize()) byteData := make([]byte, byteSize) n, err := io.ReadFull(stream, byteData) - log.Debug(err, n) + utils.Log.Debug(err, n) if err != nil { return err } - req, err := http.NewRequest("PUT", uploadUrl, bytes.NewBuffer(byteData)) + req, err := http.NewRequest("PUT", uploadUrl, driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))) if err != nil { return err } req = req.WithContext(ctx) - req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) + req.ContentLength = byteSize + // req.Header.Set("Content-Length", strconv.Itoa(int(byteSize))) req.Header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", finish, finish+byteSize-1, stream.GetSize())) - finish += byteSize res, err := base.HttpClient.Do(req) if err != nil { return err } // https://learn.microsoft.com/zh-cn/onedrive/developer/rest-api/api/driveitem_createuploadsession - if res.StatusCode != 201 && res.StatusCode != 202 && res.StatusCode != 200 { + switch { + case res.StatusCode >= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1<= 500 && res.StatusCode <= 504: + retryCount++ + if retryCount > maxRetries { + res.Body.Close() + return fmt.Errorf("upload failed after %d retries due to server errors, error %d", maxRetries, res.StatusCode) + } + backoff := time.Duration(1< 1 && (id[0] == 'd' || id[0] == 'f') { + return id[1:] + } + return id +} + +// Get folder ID from path, return "0" for root +func getFolderID(path string) string { + if path == "/" || path == "" { + return "0" + } + return extractID(path) +} \ No newline at end of file diff --git a/drivers/pcloud/util.go b/drivers/pcloud/util.go new file mode 100644 index 00000000000..f2c1875e133 --- /dev/null +++ b/drivers/pcloud/util.go @@ -0,0 +1,297 @@ +package pcloud + +import ( + "context" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + defaultClientID = "DnONSzyJXpm" + defaultClientSecret = "VKEnd3ze4jsKFGg8TJiznwFG8" +) + +// Get API base URL +func (d *PCloud) getAPIURL() string { + return "https://" + d.Hostname +} + +// Get OAuth client credentials +func (d *PCloud) getClientCredentials() (string, string) { + clientID := d.ClientID + clientSecret := d.ClientSecret + + if clientID == "" { + clientID = defaultClientID + } + if clientSecret == "" { + clientSecret = defaultClientSecret + } + + return clientID, clientSecret +} + +// Refresh OAuth access token +func (d *PCloud) refreshToken() error { + clientID, clientSecret := d.getClientCredentials() + + var resp TokenResponse + _, err := base.RestyClient.R(). + SetFormData(map[string]string{ + "client_id": clientID, + "client_secret": clientSecret, + "grant_type": "refresh_token", + "refresh_token": d.RefreshToken, + }). + SetResult(&resp). + Post(d.getAPIURL() + "/oauth2_token") + + if err != nil { + return err + } + + d.AccessToken = resp.AccessToken + return nil +} + +// shouldRetry determines if an error should be retried based on pCloud-specific logic +func (d *PCloud) shouldRetry(statusCode int, apiError *ErrorResult) bool { + // HTTP-level retry conditions + if statusCode == 429 || statusCode >= 500 { + return true + } + + // pCloud API-specific retry conditions (like rclone) + if apiError != nil && apiError.Result != 0 { + // 4xxx: rate limiting + if apiError.Result/1000 == 4 { + return true + } + // 5xxx: internal errors + if apiError.Result/1000 == 5 { + return true + } + } + + return false +} + +// requestWithRetry makes authenticated API request with retry logic +func (d *PCloud) requestWithRetry(endpoint string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + maxRetries := 3 + baseDelay := 500 * time.Millisecond + + for attempt := 0; attempt <= maxRetries; attempt++ { + body, err := d.request(endpoint, method, callback, resp) + if err == nil { + return body, nil + } + + // If this is the last attempt, return the error + if attempt == maxRetries { + return nil, err + } + + // Check if we should retry based on error type + if !d.shouldRetryError(err) { + return nil, err + } + + // Exponential backoff + delay := baseDelay * time.Duration(1< 0 && resp.Medias[0].Link.Url != "" { log.Debugln("use media link") - link.URL = resp.Medias[0].Link.Url + url = resp.Medias[0].Link.Url } - return &link, nil + + return &model.Link{ + URL: url, + }, nil } func (d *PikPak) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "kind": "drive#folder", "parent_id": parentDir.GetID(), @@ -81,7 +168,7 @@ func (d *PikPak) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } func (d *PikPak) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchMove", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchMove", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{srcObj.GetID()}, "to": base.Json{ @@ -93,7 +180,7 @@ func (d *PikPak) Move(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *PikPak) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files/"+srcObj.GetID(), http.MethodPatch, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files/"+srcObj.GetID(), http.MethodPatch, func(req *resty.Request) { req.SetBody(base.Json{ "name": newName, }) @@ -102,7 +189,7 @@ func (d *PikPak) Rename(ctx context.Context, srcObj model.Obj, newName string) e } func (d *PikPak) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchCopy", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchCopy", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{srcObj.GetID()}, "to": base.Json{ @@ -114,7 +201,7 @@ func (d *PikPak) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *PikPak) Remove(ctx context.Context, obj model.Obj) error { - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files:batchTrash", http.MethodPost, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files:batchTrash", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "ids": []string{obj.GetID()}, }) @@ -138,7 +225,7 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } var resp UploadTaskData - res, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodPost, func(req *resty.Request) { + res, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { req.SetBody(base.Json{ "kind": "drive#file", "name": stream.GetName(), @@ -161,24 +248,105 @@ func (d *PikPak) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } params := resp.Resumable.Params - endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") - cfg := &aws.Config{ - Credentials: credentials.NewStaticCredentials(params.AccessKeyID, params.AccessKeySecret, params.SecurityToken), - Region: aws.String("pikpak"), - Endpoint: &endpoint, + //endpoint := strings.Join(strings.Split(params.Endpoint, ".")[1:], ".") + // web 端上传 返回的endpoint 为 `mypikpak.net` | android 端上传 返回的endpoint 为 `vip-lixian-07.mypikpak.net`· + if d.Addition.Platform == "android" { + params.Endpoint = "mypikpak.net" } - ss, err := session.NewSession(cfg) + + if stream.GetSize() <= 10*utils.MB { // 文件大小 小于10MB,改用普通模式上传 + return d.UploadByOSS(ctx, ¶ms, stream, up) + } + // 分片上传 + return d.UploadByMultipart(ctx, ¶ms, stream.GetSize(), stream, up) +} + +// 离线下载文件 +func (d *PikPak) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + requestBody := base.Json{ + "kind": "drive#file", + "name": fileName, + "upload_type": "UPLOAD_TYPE_URL", + "url": base.Json{ + "url": fileUrl, + }, + "parent_id": parentDir.GetID(), + "folder_type": "", + } + + var resp OfflineDownloadResp + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) { + req.SetBody(requestBody) + }, &resp) + if err != nil { - return err + return nil, err + } + + return &resp.Task, err +} + +/* +获取离线下载任务列表 +phase 可能的取值: +PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING +*/ +func (d *PikPak) OfflineList(ctx context.Context, nextPageToken string, phase []string) ([]OfflineTask, error) { + res := make([]OfflineTask, 0) + url := "https://api-drive.mypikpak.net/drive/v1/tasks" + + if len(phase) == 0 { + phase = []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_COMPLETE", "PHASE_TYPE_PENDING"} } - uploader := s3manager.NewUploader(ss) - input := &s3manager.UploadInput{ - Bucket: ¶ms.Bucket, - Key: ¶ms.Key, - Body: stream, + params := map[string]string{ + "type": "offline", + "thumbnail_size": "SIZE_SMALL", + "limit": "10000", + "page_token": nextPageToken, + "with": "reference_resource", } - _, err = uploader.UploadWithContext(ctx, input) - return err + + // 处理 phase 参数 + if len(phase) > 0 { + filters := base.Json{ + "phase": map[string]string{ + "in": strings.Join(phase, ","), + }, + } + filtersJSON, err := json.Marshal(filters) + if err != nil { + return nil, fmt.Errorf("failed to marshal filters: %w", err) + } + params["filters"] = string(filtersJSON) + } + + var resp OfflineListResp + _, err := d.request(url, http.MethodGet, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(params) + }, &resp) + + if err != nil { + return nil, fmt.Errorf("failed to get offline list: %w", err) + } + res = append(res, resp.Tasks...) + return res, nil +} + +func (d *PikPak) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + url := "https://api-drive.mypikpak.net/drive/v1/tasks" + params := map[string]string{ + "task_ids": strings.Join(taskIDs, ","), + "delete_files": strconv.FormatBool(deleteFiles), + } + _, err := d.request(url, http.MethodDelete, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(params) + }, nil) + if err != nil { + return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) + } + return nil } var _ driver.Driver = (*PikPak)(nil) diff --git a/drivers/pikpak/meta.go b/drivers/pikpak/meta.go index 3512c44b93b..5abbc8796ca 100644 --- a/drivers/pikpak/meta.go +++ b/drivers/pikpak/meta.go @@ -9,7 +9,11 @@ type Addition struct { driver.RootID Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` - DisableMediaLink bool `json:"disable_media_link"` + Platform string `json:"platform" required:"true" default:"web" type:"select" options:"android,web,pc"` + RefreshToken string `json:"refresh_token" required:"true" default:""` + CaptchaToken string `json:"captcha_token" default:""` + DeviceID string `json:"device_id" required:"false" default:""` + DisableMediaLink bool `json:"disable_media_link" default:"true"` } var config = driver.Config{ diff --git a/drivers/pikpak/types.go b/drivers/pikpak/types.go index 489a1efe713..2a959ebf05d 100644 --- a/drivers/pikpak/types.go +++ b/drivers/pikpak/types.go @@ -1,6 +1,7 @@ package pikpak import ( + "fmt" "strconv" "time" @@ -9,11 +10,6 @@ import ( hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" ) -type RespErr struct { - ErrorCode int `json:"error_code"` - Error string `json:"error"` -} - type Files struct { Files []File `json:"files"` NextPageToken string `json:"next_page_token"` @@ -84,18 +80,118 @@ type UploadTaskData struct { UploadType string `json:"upload_type"` //UPLOAD_TYPE_RESUMABLE Resumable *struct { - Kind string `json:"kind"` - Params struct { - AccessKeyID string `json:"access_key_id"` - AccessKeySecret string `json:"access_key_secret"` - Bucket string `json:"bucket"` - Endpoint string `json:"endpoint"` - Expiration time.Time `json:"expiration"` - Key string `json:"key"` - SecurityToken string `json:"security_token"` - } `json:"params"` - Provider string `json:"provider"` + Kind string `json:"kind"` + Params S3Params `json:"params"` + Provider string `json:"provider"` } `json:"resumable"` File File `json:"file"` } + +type S3Params struct { + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + Bucket string `json:"bucket"` + Endpoint string `json:"endpoint"` + Expiration time.Time `json:"expiration"` + Key string `json:"key"` + SecurityToken string `json:"security_token"` +} + +// 添加离线下载响应 +type OfflineDownloadResp struct { + File *string `json:"file"` + Task OfflineTask `json:"task"` + UploadType string `json:"upload_type"` + URL struct { + Kind string `json:"kind"` + } `json:"url"` +} + +// 离线下载列表 +type OfflineListResp struct { + ExpiresIn int64 `json:"expires_in"` + NextPageToken string `json:"next_page_token"` + Tasks []OfflineTask `json:"tasks"` +} + +// offlineTask +type OfflineTask struct { + Callback string `json:"callback"` + CreatedTime string `json:"created_time"` + FileID string `json:"file_id"` + FileName string `json:"file_name"` + FileSize string `json:"file_size"` + IconLink string `json:"icon_link"` + ID string `json:"id"` + Kind string `json:"kind"` + Message string `json:"message"` + Name string `json:"name"` + Params Params `json:"params"` + Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING + Progress int64 `json:"progress"` + ReferenceResource ReferenceResource `json:"reference_resource"` + Space string `json:"space"` + StatusSize int64 `json:"status_size"` + Statuses []string `json:"statuses"` + ThirdTaskID string `json:"third_task_id"` + Type string `json:"type"` + UpdatedTime string `json:"updated_time"` + UserID string `json:"user_id"` +} + +type Params struct { + Age string `json:"age"` + MIMEType *string `json:"mime_type,omitempty"` + PredictType string `json:"predict_type"` + URL string `json:"url"` +} + +type ReferenceResource struct { + Type string `json:"@type"` + Audit interface{} `json:"audit"` + Hash string `json:"hash"` + IconLink string `json:"icon_link"` + ID string `json:"id"` + Kind string `json:"kind"` + Medias []Media `json:"medias"` + MIMEType string `json:"mime_type"` + Name string `json:"name"` + Params map[string]interface{} `json:"params"` + ParentID string `json:"parent_id"` + Phase string `json:"phase"` + Size string `json:"size"` + Space string `json:"space"` + Starred bool `json:"starred"` + Tags []string `json:"tags"` + ThumbnailLink string `json:"thumbnail_link"` +} + +type ErrResp struct { + ErrorCode int64 `json:"error_code"` + ErrorMsg string `json:"error"` + ErrorDescription string `json:"error_description"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) +} + +type CaptchaTokenRequest struct { + Action string `json:"action"` + CaptchaToken string `json:"captcha_token"` + ClientID string `json:"client_id"` + DeviceID string `json:"device_id"` + Meta map[string]string `json:"meta"` + RedirectUri string `json:"redirect_uri"` +} + +type CaptchaTokenResponse struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Url string `json:"url"` +} diff --git a/drivers/pikpak/util.go b/drivers/pikpak/util.go index 02b988bcd64..4cb3fbc3ec8 100644 --- a/drivers/pikpak/util.go +++ b/drivers/pikpak/util.go @@ -1,52 +1,143 @@ package pikpak import ( + "bytes" + "context" + "crypto/md5" "crypto/sha1" "encoding/hex" - "errors" + "fmt" "io" "net/http" + "path/filepath" + "regexp" + "strings" + "sync" + "sync/atomic" + "time" "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aliyun/aliyun-oss-go-sdk/oss" "github.com/go-resty/resty/v2" jsoniter "github.com/json-iterator/go" + "github.com/pkg/errors" ) -// do others that not defined in Driver interface +var AndroidAlgorithms = []string{ + "SOP04dGzk0TNO7t7t9ekDbAmx+eq0OI1ovEx", + "nVBjhYiND4hZ2NCGyV5beamIr7k6ifAsAbl", + "Ddjpt5B/Cit6EDq2a6cXgxY9lkEIOw4yC1GDF28KrA", + "VVCogcmSNIVvgV6U+AochorydiSymi68YVNGiz", + "u5ujk5sM62gpJOsB/1Gu/zsfgfZO", + "dXYIiBOAHZgzSruaQ2Nhrqc2im", + "z5jUTBSIpBN9g4qSJGlidNAutX6", + "KJE2oveZ34du/g1tiimm", +} + +var WebAlgorithms = []string{ + "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", + "+r6CQVxjzJV6LCV", + "F", + "pFJRC", + "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", + "/750aCr4lm/Sly/c", + "RB+DT/gZCrbV", + "", + "CyLsf7hdkIRxRm215hl", + "7xHvLi2tOYP0Y92b", + "ZGTXXxu8E/MIWaEDB+Sm/", + "1UI3", + "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", + "ihtqpG6FMt65+Xk+tWUH2", + "NhXXU9rg4XXdzo7u5o", +} + +var PCAlgorithms = []string{ + "KHBJ07an7ROXDoK7Db", + "G6n399rSWkl7WcQmw5rpQInurc1DkLmLJqE", + "JZD1A3M4x+jBFN62hkr7VDhkkZxb9g3rWqRZqFAAb", + "fQnw/AmSlbbI91Ik15gpddGgyU7U", + "/Dv9JdPYSj3sHiWjouR95NTQff", + "yGx2zuTjbWENZqecNI+edrQgqmZKP", + "ljrbSzdHLwbqcRn", + "lSHAsqCkGDGxQqqwrVu", + "TsWXI81fD1", + "vk7hBjawK/rOSrSWajtbMk95nfgf3", +} + +const ( + OSSUserAgent = "aliyun-sdk-android/2.9.13(Linux/Android 14/M2004j7ac;UKQ1.231108.001)" + OssSecurityTokenHeaderName = "X-OSS-Security-Token" + ThreadsNum = 10 +) + +const ( + AndroidClientID = "YNxT9w7GMdWvEOKa" + AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + AndroidClientVersion = "1.53.2" + AndroidPackageName = "com.pikcloud.pikpak" + AndroidSdkVersion = "2.0.6.206003" + WebClientID = "YUMx5nI8ZU8Ap8pm" + WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + WebClientVersion = "2.0.0" + WebPackageName = "mypikpak.com" + WebSdkVersion = "8.0.3" + PCClientID = "YvtoWO6GNHiuCl7x" + PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" + PCClientVersion = "undefined" // 2.6.11.4955 + PCPackageName = "mypikpak.com" + PCSdkVersion = "8.0.3" +) func (d *PikPak) login() error { - url := "https://user.mypikpak.com/v1/auth/signin" - var e RespErr - res, err := base.RestyClient.R().SetError(&e).SetBody(base.Json{ - "captcha_token": "", - "client_id": "YNxT9w7GMdWvEOKa", - "client_secret": "dbw2OtmVEeuUvIptb1Coyg", + // 检查用户名和密码是否为空 + if d.Addition.Username == "" || d.Addition.Password == "" { + return errors.New("username or password is empty") + } + + url := "https://user.mypikpak.net/v1/auth/signin" + // 使用 用户填写的 CaptchaToken —————— (验证后的captcha_token) + if d.GetCaptchaToken() == "" { + if err := d.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), d.Username); err != nil { + return err + } + } + + var e ErrResp + res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e).SetBody(base.Json{ + "captcha_token": d.GetCaptchaToken(), + "client_id": d.ClientID, + "client_secret": d.ClientSecret, "username": d.Username, "password": d.Password, - }).Post(url) + }).SetQueryParam("client_id", d.ClientID).Post(url) if err != nil { return err } if e.ErrorCode != 0 { - return errors.New(e.Error) + return &e } data := res.Body() d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() d.AccessToken = jsoniter.Get(data, "access_token").ToString() + d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) return nil } -func (d *PikPak) refreshToken() error { - url := "https://user.mypikpak.com/v1/auth/token" - var e RespErr - res, err := base.RestyClient.R().SetError(&e). +func (d *PikPak) refreshToken(refreshToken string) error { + url := "https://user.mypikpak.net/v1/auth/token" + var e ErrResp + res, err := base.RestyClient.SetRetryCount(1).R().SetError(&e). SetHeader("user-agent", "").SetBody(base.Json{ - "client_id": "YNxT9w7GMdWvEOKa", - "client_secret": "dbw2OtmVEeuUvIptb1Coyg", + "client_id": d.ClientID, + "client_secret": d.ClientSecret, "grant_type": "refresh_token", - "refresh_token": d.RefreshToken, - }).Post(url) + "refresh_token": refreshToken, + }).SetQueryParam("client_id", d.ClientID).Post(url) if err != nil { d.Status = err.Error() op.MustSaveDriverStorage(d) @@ -54,49 +145,72 @@ func (d *PikPak) refreshToken() error { } if e.ErrorCode != 0 { if e.ErrorCode == 4126 { - // refresh_token invalid, re-login - return d.login() + // 1. 未填写 username 或 password + if d.Addition.Username == "" || d.Addition.Password == "" { + return errors.New("refresh_token invalid, please re-provide refresh_token") + } else { + // refresh_token invalid, re-login + return d.login() + } } - d.Status = e.Error + d.Status = e.Error() op.MustSaveDriverStorage(d) - return errors.New(e.Error) + return errors.New(e.Error()) } data := res.Body() d.Status = "work" d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() d.AccessToken = jsoniter.Get(data, "access_token").ToString() + d.Common.SetUserID(jsoniter.Get(data, "sub").ToString()) + d.Addition.RefreshToken = d.RefreshToken op.MustSaveDriverStorage(d) return nil } func (d *PikPak) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() - req.SetHeader("Authorization", "Bearer "+d.AccessToken) + req.SetHeaders(map[string]string{ + //"Authorization": "Bearer " + d.AccessToken, + "User-Agent": d.GetUserAgent(), + "X-Device-ID": d.GetDeviceID(), + "X-Captcha-Token": d.GetCaptchaToken(), + }) + if d.AccessToken != "" { + req.SetHeader("Authorization", "Bearer "+d.AccessToken) + } + if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } - var e RespErr + var e ErrResp req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } - if e.ErrorCode != 0 { - if e.ErrorCode == 16 { - // login / refresh token - err = d.refreshToken() - if err != nil { - return nil, err - } - return d.request(url, method, callback, resp) - } else { - return nil, errors.New(e.Error) + + switch e.ErrorCode { + case 0: + return res.Body(), nil + case 4122, 4121, 16: + // access_token 过期 + if err1 := d.refreshToken(d.RefreshToken); err1 != nil { + return nil, err1 + } + return d.request(url, method, callback, resp) + case 9: // 验证码token过期 + if err = d.RefreshCaptchaTokenAtLogin(GetAction(method, url), d.GetUserID()); err != nil { + return nil, err } + return d.request(url, method, callback, resp) + case 10: // 操作频繁 + return nil, errors.New(e.ErrorDescription) + default: + return nil, errors.New(e.Error()) } - return res.Body(), nil } func (d *PikPak) getFiles(id string) ([]File, error) { @@ -115,7 +229,7 @@ func (d *PikPak) getFiles(id string) ([]File, error) { "page_token": pageToken, } var resp Files - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/files", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -127,27 +241,421 @@ func (d *PikPak) getFiles(id string) ([]File, error) { return res, nil } -func getGcid(r io.Reader, size int64) (string, error) { - calcBlockSize := func(j int64) int64 { - var psize int64 = 0x40000 - for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { - psize = psize << 1 - } - return psize +func GetAction(method string, url string) string { + urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] + return method + ":" + urlpath +} + +type Common struct { + client *resty.Client + CaptchaToken string + UserID string + // 必要值,签名相关 + ClientID string + ClientSecret string + ClientVersion string + PackageName string + Algorithms []string + DeviceID string + UserAgent string + // 验证码token刷新成功回调 + RefreshCTokenCk func(token string) +} + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} + +func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { + deviceSign := generateDeviceSign(deviceID, packageName) + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) + sb.WriteString("protocolVersion/200 ") + sb.WriteString("accesstype/ ") + sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) + sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) + sb.WriteString("action_type/ ") + sb.WriteString("networktype/WIFI ") + sb.WriteString("sessionid/ ") + sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) + sb.WriteString("providername/NONE ") + sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) + sb.WriteString("refresh_token/ ") + sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) + sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) + sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) + sb.WriteString(fmt.Sprintf("appname/android-%s ", appName)) + sb.WriteString(fmt.Sprintf("session_origin/ ")) + sb.WriteString(fmt.Sprintf("grant_type/ ")) + sb.WriteString(fmt.Sprintf("appid/ ")) + sb.WriteString(fmt.Sprintf("clientip/ ")) + sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) + sb.WriteString(fmt.Sprintf("osversion/13 ")) + sb.WriteString(fmt.Sprintf("platformversion/10 ")) + sb.WriteString(fmt.Sprintf("accessmode/ ")) + sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) + + return sb.String() +} + +func (c *Common) SetDeviceID(deviceID string) { + c.DeviceID = deviceID +} + +func (c *Common) SetUserID(userID string) { + c.UserID = userID +} + +func (c *Common) SetUserAgent(userAgent string) { + c.UserAgent = userAgent +} + +func (c *Common) SetCaptchaToken(captchaToken string) { + c.CaptchaToken = captchaToken +} +func (c *Common) GetCaptchaToken() string { + return c.CaptchaToken +} + +func (c *Common) GetUserAgent() string { + return c.UserAgent +} + +func (c *Common) GetDeviceID() string { + return c.DeviceID +} + +func (c *Common) GetUserID() string { + return c.UserID +} + +// RefreshCaptchaTokenAtLogin 刷新验证码token(登录后) +func (d *PikPak) RefreshCaptchaTokenAtLogin(action, userID string) error { + metas := map[string]string{ + "client_version": d.ClientVersion, + "package_name": d.PackageName, + "user_id": userID, } + metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() + return d.refreshCaptchaToken(action, metas) +} - hash1 := sha1.New() - hash2 := sha1.New() - readSize := calcBlockSize(size) +// RefreshCaptchaTokenInLogin 刷新验证码token(登录时) +func (d *PikPak) RefreshCaptchaTokenInLogin(action, username string) error { + metas := make(map[string]string) + if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { + metas["email"] = username + } else if len(username) >= 11 && len(username) <= 18 { + metas["phone_number"] = username + } else { + metas["username"] = username + } + return d.refreshCaptchaToken(action, metas) +} + +// GetCaptchaSign 获取验证码签名 +func (c *Common) GetCaptchaSign() (timestamp, sign string) { + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) + for _, algorithm := range c.Algorithms { + str = utils.GetMD5EncodeStr(str + algorithm) + } + sign = "1." + str + return +} + +// refreshCaptchaToken 刷新CaptchaToken +func (d *PikPak) refreshCaptchaToken(action string, metas map[string]string) error { + param := CaptchaTokenRequest{ + Action: action, + CaptchaToken: d.GetCaptchaToken(), + ClientID: d.ClientID, + DeviceID: d.GetDeviceID(), + Meta: metas, + RedirectUri: "xlaccsdk01://xbase.cloud/callback?state=harbor", + } + var e ErrResp + var resp CaptchaTokenResponse + _, err := d.request("https://user.mypikpak.net/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + req.SetError(&e).SetBody(param).SetQueryParam("client_id", d.ClientID) + }, &resp) + + if err != nil { + return err + } + + if e.IsError() { + return errors.New(e.Error()) + } + + if resp.Url != "" { + return fmt.Errorf(`need verify: Click Here`, resp.Url) + } + + if d.Common.RefreshCTokenCk != nil { + d.Common.RefreshCTokenCk(resp.CaptchaToken) + } + d.Common.SetCaptchaToken(resp.CaptchaToken) + return nil +} + +func (d *PikPak) UploadByOSS(ctx context.Context, params *S3Params, s model.FileStreamer, up driver.UpdateProgress) error { + ossClient, err := oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret) + if err != nil { + return err + } + bucket, err := ossClient.Bucket(params.Bucket) + if err != nil { + return err + } + + err = bucket.PutObject(params.Key, driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }), OssOption(params)...) + if err != nil { + return err + } + return nil +} + +func (d *PikPak) UploadByMultipart(ctx context.Context, params *S3Params, fileSize int64, s model.FileStreamer, up driver.UpdateProgress) error { + var ( + chunks []oss.FileChunk + parts []oss.UploadPart + imur oss.InitiateMultipartUploadResult + ossClient *oss.Client + bucket *oss.Bucket + err error + ) + + tmpF, err := s.CacheFullInTempFile() + if err != nil { + return err + } + + if ossClient, err = oss.New(params.Endpoint, params.AccessKeyID, params.AccessKeySecret); err != nil { + return err + } + + if bucket, err = ossClient.Bucket(params.Bucket); err != nil { + return err + } + + ticker := time.NewTicker(time.Hour * 12) + defer ticker.Stop() + // 设置超时 + timeout := time.NewTimer(time.Hour * 24) + + if chunks, err = SplitFile(fileSize); err != nil { + return err + } + + if imur, err = bucket.InitiateMultipartUpload(params.Key, + oss.SetHeader(OssSecurityTokenHeaderName, params.SecurityToken), + oss.UserAgentHeader(OSSUserAgent), + ); err != nil { + return err + } + + wg := sync.WaitGroup{} + wg.Add(len(chunks)) + + chunksCh := make(chan oss.FileChunk) + errCh := make(chan error) + UploadedPartsCh := make(chan oss.UploadPart) + quit := make(chan struct{}) + + // producer + go chunksProducer(chunksCh, chunks) + go func() { + wg.Wait() + quit <- struct{}{} + }() + + completedNum := atomic.Int32{} + // consumers + for i := 0; i < ThreadsNum; i++ { + go func(threadId int) { + defer func() { + if r := recover(); r != nil { + errCh <- fmt.Errorf("recovered in %v", r) + } + }() + for chunk := range chunksCh { + var part oss.UploadPart // 出现错误就继续尝试,共尝试3次 + for retry := 0; retry < 3; retry++ { + select { + case <-ctx.Done(): + break + case <-ticker.C: + errCh <- errors.Wrap(err, "ossToken 过期") + default: + } + + buf := make([]byte, chunk.Size) + if _, err = tmpF.ReadAt(buf, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { + continue + } + + b := driver.NewLimitedUploadStream(ctx, bytes.NewReader(buf)) + if part, err = bucket.UploadPart(imur, b, chunk.Size, chunk.Number, OssOption(params)...); err == nil { + break + } + } + if err != nil { + errCh <- errors.Wrap(err, fmt.Sprintf("上传 %s 的第%d个分片时出现错误:%v", s.GetName(), chunk.Number, err)) + } else { + num := completedNum.Add(1) + up(float64(num) * 100.0 / float64(len(chunks))) + } + UploadedPartsCh <- part + } + }(i) + } + + go func() { + for part := range UploadedPartsCh { + parts = append(parts, part) + wg.Done() + } + }() +LOOP: for { - hash2.Reset() - if n, err := io.CopyN(hash2, r, readSize); err != nil && n == 0 { - if err != io.EOF { - return "", err + select { + case <-ticker.C: + // ossToken 过期 + return err + case <-quit: + break LOOP + case <-errCh: + return err + case <-timeout.C: + return fmt.Errorf("time out") + } + } + + // EOF错误是xml的Unmarshal导致的,响应其实是json格式,所以实际上上传是成功的 + if _, err = bucket.CompleteMultipartUpload(imur, parts, OssOption(params)...); err != nil && !errors.Is(err, io.EOF) { + // 当文件名含有 &< 这两个字符之一时响应的xml解析会出现错误,实际上上传是成功的 + if filename := filepath.Base(s.GetName()); !strings.ContainsAny(filename, "&<") { + return err + } + } + return nil +} + +func chunksProducer(ch chan oss.FileChunk, chunks []oss.FileChunk) { + for _, chunk := range chunks { + ch <- chunk + } +} + +func SplitFile(fileSize int64) (chunks []oss.FileChunk, err error) { + for i := int64(1); i < 10; i++ { + if fileSize < i*utils.GB { // 文件大小小于iGB时分为i*100片 + if chunks, err = SplitFileByPartNum(fileSize, int(i*100)); err != nil { + return } break } - hash1.Write(hash2.Sum(nil)) } - return hex.EncodeToString(hash1.Sum(nil)), nil + if fileSize > 9*utils.GB { // 文件大小大于9GB时分为1000片 + if chunks, err = SplitFileByPartNum(fileSize, 1000); err != nil { + return + } + } + // 单个分片大小不能小于1MB + if chunks[0].Size < 1*utils.MB { + if chunks, err = SplitFileByPartSize(fileSize, 1*utils.MB); err != nil { + return + } + } + return +} + +// SplitFileByPartNum splits big file into parts by the num of parts. +// Split the file with specified parts count, returns the split result when error is nil. +func SplitFileByPartNum(fileSize int64, chunkNum int) ([]oss.FileChunk, error) { + if chunkNum <= 0 || chunkNum > 10000 { + return nil, errors.New("chunkNum invalid") + } + + if int64(chunkNum) > fileSize { + return nil, errors.New("oss: chunkNum invalid") + } + + var chunks []oss.FileChunk + chunk := oss.FileChunk{} + chunkN := (int64)(chunkNum) + for i := int64(0); i < chunkN; i++ { + chunk.Number = int(i + 1) + chunk.Offset = i * (fileSize / chunkN) + if i == chunkN-1 { + chunk.Size = fileSize/chunkN + fileSize%chunkN + } else { + chunk.Size = fileSize / chunkN + } + chunks = append(chunks, chunk) + } + + return chunks, nil +} + +// SplitFileByPartSize splits big file into parts by the size of parts. +// Splits the file by the part size. Returns the FileChunk when error is nil. +func SplitFileByPartSize(fileSize int64, chunkSize int64) ([]oss.FileChunk, error) { + if chunkSize <= 0 { + return nil, errors.New("chunkSize invalid") + } + + chunkN := fileSize / chunkSize + if chunkN >= 10000 { + return nil, errors.New("Too many parts, please increase part size") + } + + var chunks []oss.FileChunk + chunk := oss.FileChunk{} + for i := int64(0); i < chunkN; i++ { + chunk.Number = int(i + 1) + chunk.Offset = i * chunkSize + chunk.Size = chunkSize + chunks = append(chunks, chunk) + } + + if fileSize%chunkSize > 0 { + chunk.Number = len(chunks) + 1 + chunk.Offset = int64(len(chunks)) * chunkSize + chunk.Size = fileSize % chunkSize + chunks = append(chunks, chunk) + } + + return chunks, nil +} + +// OssOption get options +func OssOption(params *S3Params) []oss.Option { + options := []oss.Option{ + oss.SetHeader(OssSecurityTokenHeaderName, params.SecurityToken), + oss.UserAgentHeader(OSSUserAgent), + } + return options } diff --git a/drivers/pikpak_share/driver.go b/drivers/pikpak_share/driver.go index 3003ff48860..d6341bd990a 100644 --- a/drivers/pikpak_share/driver.go +++ b/drivers/pikpak_share/driver.go @@ -2,7 +2,9 @@ package pikpak_share import ( "context" + "github.com/alist-org/alist/v3/internal/op" "net/http" + "time" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" @@ -13,8 +15,7 @@ import ( type PikPakShare struct { model.Storage Addition - RefreshToken string - AccessToken string + *Common PassCodeToken string } @@ -27,16 +28,57 @@ func (d *PikPakShare) GetAddition() driver.Additional { } func (d *PikPakShare) Init(ctx context.Context) error { - err := d.login() + if d.Common == nil { + d.Common = &Common{ + DeviceID: utils.GetMD5EncodeStr(d.Addition.ShareId + d.Addition.SharePwd + time.Now().String()), + UserAgent: "", + RefreshCTokenCk: func(token string) { + d.Common.CaptchaToken = token + op.MustSaveDriverStorage(d) + }, + } + } + + if d.Addition.DeviceID != "" { + d.SetDeviceID(d.Addition.DeviceID) + } else { + d.Addition.DeviceID = d.Common.DeviceID + op.MustSaveDriverStorage(d) + } + + if d.Platform == "android" { + d.ClientID = AndroidClientID + d.ClientSecret = AndroidClientSecret + d.ClientVersion = AndroidClientVersion + d.PackageName = AndroidPackageName + d.Algorithms = AndroidAlgorithms + d.UserAgent = BuildCustomUserAgent(d.GetDeviceID(), AndroidClientID, AndroidPackageName, AndroidSdkVersion, AndroidClientVersion, AndroidPackageName, "") + } else if d.Platform == "web" { + d.ClientID = WebClientID + d.ClientSecret = WebClientSecret + d.ClientVersion = WebClientVersion + d.PackageName = WebPackageName + d.Algorithms = WebAlgorithms + d.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" + } else if d.Platform == "pc" { + d.ClientID = PCClientID + d.ClientSecret = PCClientSecret + d.ClientVersion = PCClientVersion + d.PackageName = PCPackageName + d.Algorithms = PCAlgorithms + d.UserAgent = "MainWindow Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) PikPak/2.6.11.4955 Chrome/100.0.4896.160 Electron/18.3.15 Safari/537.36" + } + + // 获取CaptchaToken + err := d.RefreshCaptchaToken(GetAction(http.MethodGet, "https://api-drive.mypikpak.net/drive/v1/share:batch_file_info"), "") if err != nil { return err } + if d.SharePwd != "" { - err = d.getSharePassToken() - if err != nil { - return err - } + return d.getSharePassToken() } + return nil } @@ -61,16 +103,27 @@ func (d *PikPakShare) Link(ctx context.Context, file model.Obj, args model.LinkA "file_id": file.GetID(), "pass_code_token": d.PassCodeToken, } - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share/file_info", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share/file_info", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { return nil, err } - link := model.Link{ - URL: resp.FileInfo.WebContentLink, + + downloadUrl := resp.FileInfo.WebContentLink + if downloadUrl == "" && len(resp.FileInfo.Medias) > 0 { + // 使用转码后的链接 + if d.Addition.UseTransCodingAddress && len(resp.FileInfo.Medias) > 1 { + downloadUrl = resp.FileInfo.Medias[1].Link.Url + } else { + downloadUrl = resp.FileInfo.Medias[0].Link.Url + } + } - return &link, nil + + return &model.Link{ + URL: downloadUrl, + }, nil } var _ driver.Driver = (*PikPakShare)(nil) diff --git a/drivers/pikpak_share/meta.go b/drivers/pikpak_share/meta.go index bf77e22b3cf..30bccbdce87 100644 --- a/drivers/pikpak_share/meta.go +++ b/drivers/pikpak_share/meta.go @@ -7,10 +7,11 @@ import ( type Addition struct { driver.RootID - Username string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - ShareId string `json:"share_id" required:"true"` - SharePwd string `json:"share_pwd"` + ShareId string `json:"share_id" required:"true"` + SharePwd string `json:"share_pwd"` + Platform string `json:"platform" default:"web" required:"true" type:"select" options:"android,web,pc"` + DeviceID string `json:"device_id" required:"false" default:""` + UseTransCodingAddress bool `json:"use_transcoding_address" required:"true" default:"false"` } var config = driver.Config{ diff --git a/drivers/pikpak_share/types.go b/drivers/pikpak_share/types.go index 144a05a8744..78ea2ff8bbc 100644 --- a/drivers/pikpak_share/types.go +++ b/drivers/pikpak_share/types.go @@ -1,20 +1,16 @@ package pikpak_share import ( + "fmt" "strconv" "time" "github.com/alist-org/alist/v3/internal/model" ) -type RespErr struct { - ErrorCode int `json:"error_code"` - Error string `json:"error"` -} - type ShareResp struct { - ShareStatus string `json:"share_status"` - ShareStatusText string `json:"share_status_text"` + ShareStatus string `json:"share_status"` + ShareStatusText string `json:"share_status_text"` FileInfo File `json:"file_info"` Files []File `json:"files"` NextPageToken string `json:"next_page_token"` @@ -78,3 +74,32 @@ type Media struct { IsVisible bool `json:"is_visible"` Category string `json:"category"` } + +type CaptchaTokenRequest struct { + Action string `json:"action"` + CaptchaToken string `json:"captcha_token"` + ClientID string `json:"client_id"` + DeviceID string `json:"device_id"` + Meta map[string]string `json:"meta"` + RedirectUri string `json:"redirect_uri"` +} + +type CaptchaTokenResponse struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Url string `json:"url"` +} + +type ErrResp struct { + ErrorCode int64 `json:"error_code"` + ErrorMsg string `json:"error"` + ErrorDescription string `json:"error_description"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) +} diff --git a/drivers/pikpak_share/util.go b/drivers/pikpak_share/util.go index e62f17842c6..2980069e5f8 100644 --- a/drivers/pikpak_share/util.go +++ b/drivers/pikpak_share/util.go @@ -1,98 +1,115 @@ package pikpak_share import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" "errors" + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" "net/http" + "regexp" + "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" - "github.com/alist-org/alist/v3/internal/op" "github.com/go-resty/resty/v2" - jsoniter "github.com/json-iterator/go" ) -// do others that not defined in Driver interface - -func (d *PikPakShare) login() error { - url := "https://user.mypikpak.com/v1/auth/signin" - var e RespErr - res, err := base.RestyClient.R().SetError(&e).SetBody(base.Json{ - "captcha_token": "", - "client_id": "YNxT9w7GMdWvEOKa", - "client_secret": "dbw2OtmVEeuUvIptb1Coyg", - "username": d.Username, - "password": d.Password, - }).Post(url) - if err != nil { - return err - } - if e.ErrorCode != 0 { - return errors.New(e.Error) - } - data := res.Body() - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() - return nil +var AndroidAlgorithms = []string{ + "SOP04dGzk0TNO7t7t9ekDbAmx+eq0OI1ovEx", + "nVBjhYiND4hZ2NCGyV5beamIr7k6ifAsAbl", + "Ddjpt5B/Cit6EDq2a6cXgxY9lkEIOw4yC1GDF28KrA", + "VVCogcmSNIVvgV6U+AochorydiSymi68YVNGiz", + "u5ujk5sM62gpJOsB/1Gu/zsfgfZO", + "dXYIiBOAHZgzSruaQ2Nhrqc2im", + "z5jUTBSIpBN9g4qSJGlidNAutX6", + "KJE2oveZ34du/g1tiimm", } -func (d *PikPakShare) refreshToken() error { - url := "https://user.mypikpak.com/v1/auth/token" - var e RespErr - res, err := base.RestyClient.R().SetError(&e). - SetHeader("user-agent", "").SetBody(base.Json{ - "client_id": "YNxT9w7GMdWvEOKa", - "client_secret": "dbw2OtmVEeuUvIptb1Coyg", - "grant_type": "refresh_token", - "refresh_token": d.RefreshToken, - }).Post(url) - if err != nil { - d.Status = err.Error() - op.MustSaveDriverStorage(d) - return err - } - if e.ErrorCode != 0 { - if e.ErrorCode == 4126 { - // refresh_token invalid, re-login - return d.login() - } - d.Status = e.Error - op.MustSaveDriverStorage(d) - return errors.New(e.Error) - } - data := res.Body() - d.Status = "work" - d.RefreshToken = jsoniter.Get(data, "refresh_token").ToString() - d.AccessToken = jsoniter.Get(data, "access_token").ToString() - op.MustSaveDriverStorage(d) - return nil +var WebAlgorithms = []string{ + "C9qPpZLN8ucRTaTiUMWYS9cQvWOE", + "+r6CQVxjzJV6LCV", + "F", + "pFJRC", + "9WXYIDGrwTCz2OiVlgZa90qpECPD6olt", + "/750aCr4lm/Sly/c", + "RB+DT/gZCrbV", + "", + "CyLsf7hdkIRxRm215hl", + "7xHvLi2tOYP0Y92b", + "ZGTXXxu8E/MIWaEDB+Sm/", + "1UI3", + "E7fP5Pfijd+7K+t6Tg/NhuLq0eEUVChpJSkrKxpO", + "ihtqpG6FMt65+Xk+tWUH2", + "NhXXU9rg4XXdzo7u5o", } +var PCAlgorithms = []string{ + "KHBJ07an7ROXDoK7Db", + "G6n399rSWkl7WcQmw5rpQInurc1DkLmLJqE", + "JZD1A3M4x+jBFN62hkr7VDhkkZxb9g3rWqRZqFAAb", + "fQnw/AmSlbbI91Ik15gpddGgyU7U", + "/Dv9JdPYSj3sHiWjouR95NTQff", + "yGx2zuTjbWENZqecNI+edrQgqmZKP", + "ljrbSzdHLwbqcRn", + "lSHAsqCkGDGxQqqwrVu", + "TsWXI81fD1", + "vk7hBjawK/rOSrSWajtbMk95nfgf3", +} + +const ( + AndroidClientID = "YNxT9w7GMdWvEOKa" + AndroidClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + AndroidClientVersion = "1.53.2" + AndroidPackageName = "com.pikcloud.pikpak" + AndroidSdkVersion = "2.0.6.206003" + WebClientID = "YUMx5nI8ZU8Ap8pm" + WebClientSecret = "dbw2OtmVEeuUvIptb1Coyg" + WebClientVersion = "2.0.0" + WebPackageName = "mypikpak.com" + WebSdkVersion = "8.0.3" + PCClientID = "YvtoWO6GNHiuCl7x" + PCClientSecret = "1NIH5R1IEe2pAxZE3hv3uA" + PCClientVersion = "undefined" // 2.6.11.4955 + PCPackageName = "mypikpak.com" + PCSdkVersion = "8.0.3" +) + func (d *PikPakShare) request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { req := base.RestyClient.R() - req.SetHeader("Authorization", "Bearer "+d.AccessToken) + req.SetHeaders(map[string]string{ + "User-Agent": d.GetUserAgent(), + "X-Client-ID": d.GetClientID(), + "X-Device-ID": d.GetDeviceID(), + "X-Captcha-Token": d.GetCaptchaToken(), + }) + if callback != nil { callback(req) } if resp != nil { req.SetResult(resp) } - var e RespErr + var e ErrResp req.SetError(&e) res, err := req.Execute(method, url) if err != nil { return nil, err } - if e.ErrorCode != 0 { - if e.ErrorCode == 16 { - // login / refresh token - err = d.refreshToken() - if err != nil { - return nil, err - } - return d.request(url, method, callback, resp) + switch e.ErrorCode { + case 0: + return res.Body(), nil + case 9: // 验证码token过期 + if err = d.RefreshCaptchaToken(GetAction(method, url), ""); err != nil { + return nil, err } - return nil, errors.New(e.Error) + return d.request(url, method, callback, resp) + case 10: // 操作频繁 + return nil, errors.New(e.ErrorDescription) + default: + return nil, errors.New(e.Error()) } - return res.Body(), nil } func (d *PikPakShare) getSharePassToken() error { @@ -103,7 +120,7 @@ func (d *PikPakShare) getSharePassToken() error { "limit": "100", } var resp ShareResp - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -131,7 +148,7 @@ func (d *PikPakShare) getFiles(id string) ([]File, error) { "pass_code_token": d.PassCodeToken, } var resp ShareResp - _, err := d.request("https://api-drive.mypikpak.com/drive/v1/share/detail", http.MethodGet, func(req *resty.Request) { + _, err := d.request("https://api-drive.mypikpak.net/drive/v1/share/detail", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(query) }, &resp) if err != nil { @@ -152,3 +169,161 @@ func (d *PikPakShare) getFiles(id string) ([]File, error) { } return res, nil } + +func GetAction(method string, url string) string { + urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] + return method + ":" + urlpath +} + +type Common struct { + client *resty.Client + CaptchaToken string + // 必要值,签名相关 + ClientID string + ClientSecret string + ClientVersion string + PackageName string + Algorithms []string + DeviceID string + UserAgent string + // 验证码token刷新成功回调 + RefreshCTokenCk func(token string) +} + +func (c *Common) SetUserAgent(userAgent string) { + c.UserAgent = userAgent +} + +func (c *Common) SetCaptchaToken(captchaToken string) { + c.CaptchaToken = captchaToken +} + +func (c *Common) SetDeviceID(deviceID string) { + c.DeviceID = deviceID +} + +func (c *Common) GetCaptchaToken() string { + return c.CaptchaToken +} + +func (c *Common) GetClientID() string { + return c.ClientID +} + +func (c *Common) GetUserAgent() string { + return c.UserAgent +} + +func (c *Common) GetDeviceID() string { + return c.DeviceID +} + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} + +func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { + deviceSign := generateDeviceSign(deviceID, packageName) + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) + sb.WriteString("protocolVersion/200 ") + sb.WriteString("accesstype/ ") + sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) + sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) + sb.WriteString("action_type/ ") + sb.WriteString("networktype/WIFI ") + sb.WriteString("sessionid/ ") + sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) + sb.WriteString("providername/NONE ") + sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) + sb.WriteString("refresh_token/ ") + sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) + sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) + sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) + sb.WriteString(fmt.Sprintf("appname/android-%s ", appName)) + sb.WriteString(fmt.Sprintf("session_origin/ ")) + sb.WriteString(fmt.Sprintf("grant_type/ ")) + sb.WriteString(fmt.Sprintf("appid/ ")) + sb.WriteString(fmt.Sprintf("clientip/ ")) + sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) + sb.WriteString(fmt.Sprintf("osversion/13 ")) + sb.WriteString(fmt.Sprintf("platformversion/10 ")) + sb.WriteString(fmt.Sprintf("accessmode/ ")) + sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) + + return sb.String() +} + +// RefreshCaptchaToken 刷新验证码token +func (d *PikPakShare) RefreshCaptchaToken(action, userID string) error { + metas := map[string]string{ + "client_version": d.ClientVersion, + "package_name": d.PackageName, + "user_id": userID, + } + metas["timestamp"], metas["captcha_sign"] = d.Common.GetCaptchaSign() + return d.refreshCaptchaToken(action, metas) +} + +// GetCaptchaSign 获取验证码签名 +func (c *Common) GetCaptchaSign() (timestamp, sign string) { + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) + for _, algorithm := range c.Algorithms { + str = utils.GetMD5EncodeStr(str + algorithm) + } + sign = "1." + str + return +} + +// refreshCaptchaToken 刷新CaptchaToken +func (d *PikPakShare) refreshCaptchaToken(action string, metas map[string]string) error { + param := CaptchaTokenRequest{ + Action: action, + CaptchaToken: d.GetCaptchaToken(), + ClientID: d.ClientID, + DeviceID: d.GetDeviceID(), + Meta: metas, + } + var e ErrResp + var resp CaptchaTokenResponse + _, err := d.request("https://user.mypikpak.net/v1/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + req.SetError(&e).SetBody(param) + }, &resp) + + if err != nil { + return err + } + + if e.IsError() { + return errors.New(e.Error()) + } + + //if resp.Url != "" { + // return fmt.Errorf(`need verify: Click Here`, resp.Url) + //} + + if d.Common.RefreshCTokenCk != nil { + d.Common.RefreshCTokenCk(resp.CaptchaToken) + } + d.Common.SetCaptchaToken(resp.CaptchaToken) + return nil +} diff --git a/drivers/proton_drive/driver.go b/drivers/proton_drive/driver.go new file mode 100644 index 00000000000..6ff8b069cb3 --- /dev/null +++ b/drivers/proton_drive/driver.go @@ -0,0 +1,418 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "context" + "encoding/base64" + "fmt" + "net/http" + "sync" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + proton_api_bridge "github.com/henrybear327/Proton-API-Bridge" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +type ProtonDrive struct { + model.Storage + Addition + + protonDrive *proton_api_bridge.ProtonDrive + credentials *common.ProtonDriveCredential + + apiBase string + appVersion string + protonJson string + userAgent string + sdkVersion string + webDriveAV string + + tempServer *http.Server + tempServerPort int + downloadTokens map[string]*downloadInfo + tokenMutex sync.RWMutex + + c *proton.Client + //m *proton.Manager + + credentialCacheFile string + + //userKR *crypto.KeyRing + addrKRs map[string]*crypto.KeyRing + addrData map[string]proton.Address + + MainShare *proton.Share + RootLink *proton.Link + + DefaultAddrKR *crypto.KeyRing + MainShareKR *crypto.KeyRing +} + +func (d *ProtonDrive) Config() driver.Config { + return config +} + +func (d *ProtonDrive) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *ProtonDrive) Init(ctx context.Context) error { + + defer func() { + if r := recover(); r != nil { + fmt.Printf("ProtonDrive initialization panic: %v", r) + } + }() + + if d.Username == "" { + return fmt.Errorf("username is required") + } + if d.Password == "" { + return fmt.Errorf("password is required") + } + + //fmt.Printf("ProtonDrive Init: Username=%s, TwoFACode=%s", d.Username, d.TwoFACode) + + if ctx == nil { + return fmt.Errorf("context cannot be nil") + } + + cachedCredentials, err := d.loadCachedCredentials() + useReusableLogin := false + var reusableCredential *common.ReusableCredentialData + + if err == nil && cachedCredentials != nil && + cachedCredentials.UID != "" && cachedCredentials.AccessToken != "" && + cachedCredentials.RefreshToken != "" && cachedCredentials.SaltedKeyPass != "" { + useReusableLogin = true + reusableCredential = cachedCredentials + } else { + useReusableLogin = false + reusableCredential = &common.ReusableCredentialData{} + } + + config := &common.Config{ + AppVersion: d.appVersion, + UserAgent: d.userAgent, + FirstLoginCredential: &common.FirstLoginCredentialData{ + Username: d.Username, + Password: d.Password, + TwoFA: d.TwoFACode, + }, + EnableCaching: true, + ConcurrentBlockUploadCount: 5, + ConcurrentFileCryptoCount: 2, + UseReusableLogin: false, + ReplaceExistingDraft: true, + ReusableCredential: reusableCredential, + CredentialCacheFile: d.credentialCacheFile, + } + + if config.FirstLoginCredential == nil { + return fmt.Errorf("failed to create login credentials, FirstLoginCredential cannot be nil") + } + + //fmt.Printf("Calling NewProtonDrive...") + + protonDrive, credentials, err := proton_api_bridge.NewProtonDrive( + ctx, + config, + func(auth proton.Auth) {}, + func() {}, + ) + + if credentials == nil && !useReusableLogin { + return fmt.Errorf("failed to get credentials from NewProtonDrive") + } + + if err != nil { + return fmt.Errorf("failed to initialize ProtonDrive: %w", err) + } + + d.protonDrive = protonDrive + + var finalCredentials *common.ProtonDriveCredential + + if useReusableLogin { + + // For reusable login, create credentials from cached data + finalCredentials = &common.ProtonDriveCredential{ + UID: reusableCredential.UID, + AccessToken: reusableCredential.AccessToken, + RefreshToken: reusableCredential.RefreshToken, + SaltedKeyPass: reusableCredential.SaltedKeyPass, + } + + d.credentials = finalCredentials + } else { + d.credentials = credentials + } + + clientOptions := []proton.Option{ + proton.WithAppVersion(d.appVersion), + proton.WithUserAgent(d.userAgent), + } + manager := proton.New(clientOptions...) + d.c = manager.NewClient(d.credentials.UID, d.credentials.AccessToken, d.credentials.RefreshToken) + + saltedKeyPassBytes, err := base64.StdEncoding.DecodeString(d.credentials.SaltedKeyPass) + if err != nil { + return fmt.Errorf("failed to decode salted key pass: %w", err) + } + + _, addrKRs, addrs, _, err := getAccountKRs(ctx, d.c, nil, saltedKeyPassBytes) + if err != nil { + return fmt.Errorf("failed to get account keyrings: %w", err) + } + + d.MainShare = protonDrive.MainShare + d.RootLink = protonDrive.RootLink + d.MainShareKR = protonDrive.MainShareKR + d.DefaultAddrKR = protonDrive.DefaultAddrKR + d.addrKRs = addrKRs + d.addrData = addrs + + return nil +} + +func (d *ProtonDrive) Drop(ctx context.Context) error { + if d.tempServer != nil { + d.tempServer.Shutdown(ctx) + } + return nil +} + +func (d *ProtonDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var linkID string + + if dir.GetPath() == "/" { + linkID = d.protonDrive.RootLink.LinkID + } else { + + link, err := d.searchByPath(ctx, dir.GetPath(), true) + if err != nil { + return nil, err + } + linkID = link.LinkID + } + + entries, err := d.protonDrive.ListDirectory(ctx, linkID) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + } + + //fmt.Printf("Found %d entries for path %s\n", len(entries), dir.GetPath()) + //fmt.Printf("Found %d entries\n", len(entries)) + + if len(entries) == 0 { + emptySlice := []model.Obj{} + + //fmt.Printf("Returning empty slice (entries): %+v\n", emptySlice) + + return emptySlice, nil + } + + var objects []model.Obj + for _, entry := range entries { + obj := &model.Object{ + Name: entry.Name, + Size: entry.Link.Size, + Modified: time.Unix(entry.Link.ModifyTime, 0), + IsFolder: entry.IsFolder, + } + objects = append(objects, obj) + } + + return objects, nil +} + +func (d *ProtonDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + link, err := d.searchByPath(ctx, file.GetPath(), false) + if err != nil { + return nil, err + } + + if err := d.ensureTempServer(); err != nil { + return nil, fmt.Errorf("failed to start temp server: %w", err) + } + + token := d.generateDownloadToken(link.LinkID, file.GetName()) + + /* return &model.Link{ + URL: fmt.Sprintf("protondrive://download/%s", link.LinkID), + }, nil */ + + return &model.Link{ + URL: fmt.Sprintf("http://localhost:%d/temp/%s", d.tempServerPort, token), + }, nil +} + +func (d *ProtonDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var parentLinkID string + + if parentDir.GetPath() == "/" { + parentLinkID = d.protonDrive.RootLink.LinkID + } else { + link, err := d.searchByPath(ctx, parentDir.GetPath(), true) + if err != nil { + return nil, err + } + parentLinkID = link.LinkID + } + + _, err := d.protonDrive.CreateNewFolderByID(ctx, parentLinkID, dirName) + if err != nil { + return nil, fmt.Errorf("failed to create directory: %w", err) + } + + newDir := &model.Object{ + Name: dirName, + IsFolder: true, + Modified: time.Now(), + } + return newDir, nil +} + +func (d *ProtonDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return d.DirectMove(ctx, srcObj, dstDir) +} + +func (d *ProtonDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + return d.DirectRename(ctx, srcObj, newName) +} + +func (d *ProtonDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, fmt.Errorf("directory copy not supported") + } + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), false) + if err != nil { + return nil, err + } + + reader, linkSize, fileSystemAttrs, err := d.protonDrive.DownloadFile(ctx, srcLink, 0) + if err != nil { + return nil, fmt.Errorf("failed to download source file: %w", err) + } + defer reader.Close() + + actualSize := linkSize + if fileSystemAttrs != nil && fileSystemAttrs.Size > 0 { + actualSize = fileSystemAttrs.Size + } + + tempFile, err := utils.CreateTempFile(reader, actualSize) + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer tempFile.Close() + + updatedObj := &model.Object{ + Name: srcObj.GetName(), + // Use the accurate and real size + Size: actualSize, + Modified: srcObj.ModTime(), + IsFolder: false, + } + + return d.Put(ctx, dstDir, &fileStreamer{ + ReadCloser: tempFile, + obj: updatedObj, + }, nil) +} + +func (d *ProtonDrive) Remove(ctx context.Context, obj model.Obj) error { + link, err := d.searchByPath(ctx, obj.GetPath(), obj.IsDir()) + if err != nil { + return err + } + + if obj.IsDir() { + return d.protonDrive.MoveFolderToTrashByID(ctx, link.LinkID, false) + } else { + return d.protonDrive.MoveFileToTrashByID(ctx, link.LinkID) + } +} + +func (d *ProtonDrive) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + var parentLinkID string + + if dstDir.GetPath() == "/" { + parentLinkID = d.protonDrive.RootLink.LinkID + } else { + link, err := d.searchByPath(ctx, dstDir.GetPath(), true) + if err != nil { + return nil, err + } + parentLinkID = link.LinkID + } + + tempFile, err := utils.CreateTempFile(file, file.GetSize()) + if err != nil { + return nil, fmt.Errorf("failed to create temp file: %w", err) + } + defer tempFile.Close() + + err = d.uploadFile(ctx, parentLinkID, file.GetName(), tempFile, file.GetSize(), up) + if err != nil { + return nil, err + } + + uploadedObj := &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + Modified: file.ModTime(), + IsFolder: false, + } + return uploadedObj, nil +} + +func (d *ProtonDrive) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *ProtonDrive) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + +var _ driver.Driver = (*ProtonDrive)(nil) diff --git a/drivers/proton_drive/meta.go b/drivers/proton_drive/meta.go new file mode 100644 index 00000000000..ed33a41a2f8 --- /dev/null +++ b/drivers/proton_drive/meta.go @@ -0,0 +1,69 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + //driver.RootID + + Username string `json:"username" required:"true" type:"string"` + Password string `json:"password" required:"true" type:"string"` + TwoFACode string `json:"two_fa_code,omitempty" type:"string"` +} + +type Config struct { + Name string `json:"name"` + LocalSort bool `json:"local_sort"` + OnlyLocal bool `json:"only_local"` + OnlyProxy bool `json:"only_proxy"` + NoCache bool `json:"no_cache"` + NoUpload bool `json:"no_upload"` + NeedMs bool `json:"need_ms"` + DefaultRoot string `json:"default_root"` +} + +var config = driver.Config{ + Name: "ProtonDrive", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "/", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: false, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ProtonDrive{ + apiBase: "https://drive.proton.me/api", + appVersion: "windows-drive@1.11.3+rclone+proton", + credentialCacheFile: ".prtcrd", + protonJson: "application/vnd.protonmail.v1+json", + sdkVersion: "js@0.3.0", + userAgent: "ProtonDrive/v1.70.0 (Windows NT 10.0.22000; Win64; x64)", + webDriveAV: "web-drive@5.2.0+0f69f7a8", + } + }) +} diff --git a/drivers/proton_drive/types.go b/drivers/proton_drive/types.go new file mode 100644 index 00000000000..37a9edc6c46 --- /dev/null +++ b/drivers/proton_drive/types.go @@ -0,0 +1,124 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "errors" + "io" + "os" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/henrybear327/go-proton-api" +) + +type ProtonFile struct { + *proton.Link + Name string + IsFolder bool +} + +func (p *ProtonFile) GetName() string { + return p.Name +} + +func (p *ProtonFile) GetSize() int64 { + return p.Link.Size +} + +func (p *ProtonFile) GetPath() string { + return p.Name +} + +func (p *ProtonFile) IsDir() bool { + return p.IsFolder +} + +func (p *ProtonFile) ModTime() time.Time { + return time.Unix(p.Link.ModifyTime, 0) +} + +func (p *ProtonFile) CreateTime() time.Time { + return time.Unix(p.Link.CreateTime, 0) +} + +type downloadInfo struct { + LinkID string + FileName string +} + +type fileStreamer struct { + io.ReadCloser + obj model.Obj +} + +func (fs *fileStreamer) GetMimetype() string { return "" } +func (fs *fileStreamer) NeedStore() bool { return false } +func (fs *fileStreamer) IsForceStreamUpload() bool { return false } +func (fs *fileStreamer) GetExist() model.Obj { return nil } +func (fs *fileStreamer) SetExist(model.Obj) {} +func (fs *fileStreamer) RangeRead(http_range.Range) (io.Reader, error) { + return nil, errors.New("not supported") +} +func (fs *fileStreamer) CacheFullInTempFile() (model.File, error) { + return nil, errors.New("not supported") +} +func (fs *fileStreamer) SetTmpFile(r *os.File) {} +func (fs *fileStreamer) GetFile() model.File { return nil } +func (fs *fileStreamer) GetName() string { return fs.obj.GetName() } +func (fs *fileStreamer) GetSize() int64 { return fs.obj.GetSize() } +func (fs *fileStreamer) GetPath() string { return fs.obj.GetPath() } +func (fs *fileStreamer) IsDir() bool { return fs.obj.IsDir() } +func (fs *fileStreamer) ModTime() time.Time { return fs.obj.ModTime() } +func (fs *fileStreamer) CreateTime() time.Time { return fs.obj.ModTime() } +func (fs *fileStreamer) GetHash() utils.HashInfo { return fs.obj.GetHash() } +func (fs *fileStreamer) GetID() string { return fs.obj.GetID() } + +type httpRange struct { + start, end int64 +} + +type MoveRequest struct { + ParentLinkID string `json:"ParentLinkID"` + NodePassphrase string `json:"NodePassphrase"` + NodePassphraseSignature *string `json:"NodePassphraseSignature"` + Name string `json:"Name"` + NameSignatureEmail string `json:"NameSignatureEmail"` + Hash string `json:"Hash"` + OriginalHash string `json:"OriginalHash"` + ContentHash *string `json:"ContentHash"` // Maybe null +} + +type progressReader struct { + reader io.Reader + total int64 + current int64 + callback driver.UpdateProgress +} + +type RenameRequest struct { + Name string `json:"Name"` // PGP encrypted name + NameSignatureEmail string `json:"NameSignatureEmail"` // User's signature email + Hash string `json:"Hash"` // New name hash + OriginalHash string `json:"OriginalHash"` // Current name hash +} + +type RenameResponse struct { + Code int `json:"Code"` +} diff --git a/drivers/proton_drive/util.go b/drivers/proton_drive/util.go new file mode 100644 index 00000000000..7bd52fcaa33 --- /dev/null +++ b/drivers/proton_drive/util.go @@ -0,0 +1,918 @@ +package protondrive + +/* +Package protondrive +Author: Da3zKi7 +Date: 2025-09-18 + +Thanks to @henrybear327 for modded go-proton-api & Proton-API-Bridge + +The power of open-source, the force of teamwork and the magic of reverse engineering! + + +D@' 3z K!7 - The King Of Cracking + +Да здравствует Родина)) +*/ + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "net" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/ProtonMail/gopenpgp/v2/crypto" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/henrybear327/Proton-API-Bridge/common" + "github.com/henrybear327/go-proton-api" +) + +func (d *ProtonDrive) loadCachedCredentials() (*common.ReusableCredentialData, error) { + if d.credentialCacheFile == "" { + return nil, nil + } + + if _, err := os.Stat(d.credentialCacheFile); os.IsNotExist(err) { + return nil, nil + } + + data, err := os.ReadFile(d.credentialCacheFile) + if err != nil { + return nil, fmt.Errorf("failed to read credential cache file: %w", err) + } + + var credentials common.ReusableCredentialData + if err := json.Unmarshal(data, &credentials); err != nil { + return nil, fmt.Errorf("failed to parse cached credentials: %w", err) + } + + if credentials.UID == "" || credentials.AccessToken == "" || + credentials.RefreshToken == "" || credentials.SaltedKeyPass == "" { + return nil, fmt.Errorf("cached credentials are incomplete") + } + + return &credentials, nil +} + +func (d *ProtonDrive) searchByPath(ctx context.Context, fullPath string, isFolder bool) (*proton.Link, error) { + if fullPath == "/" { + return d.protonDrive.RootLink, nil + } + + cleanPath := strings.Trim(fullPath, "/") + pathParts := strings.Split(cleanPath, "/") + + currentLink := d.protonDrive.RootLink + + for i, part := range pathParts { + isLastPart := i == len(pathParts)-1 + searchForFolder := !isLastPart || isFolder + + entries, err := d.protonDrive.ListDirectory(ctx, currentLink.LinkID) + if err != nil { + return nil, fmt.Errorf("failed to list directory: %w", err) + + } + + found := false + for _, entry := range entries { + // entry.Name is already decrypted! + if entry.Name == part && entry.IsFolder == searchForFolder { + currentLink = entry.Link + found = true + break + } + } + + if !found { + return nil, fmt.Errorf("path not found: %s (looking for part: %s)", fullPath, part) + } + } + + return currentLink, nil +} + +func (pr *progressReader) Read(p []byte) (int, error) { + n, err := pr.reader.Read(p) + pr.current += int64(n) + + if pr.callback != nil { + percentage := float64(pr.current) / float64(pr.total) * 100 + pr.callback(percentage) + } + + return n, err +} + +func (d *ProtonDrive) uploadFile(ctx context.Context, parentLinkID, fileName string, file *os.File, size int64, up driver.UpdateProgress) error { + + fileInfo, err := file.Stat() + if err != nil { + return fmt.Errorf("failed to get file info: %w", err) + } + + _, err = d.protonDrive.GetLink(ctx, parentLinkID) + if err != nil { + return fmt.Errorf("failed to get parent link: %w", err) + } + + reader := &progressReader{ + reader: bufio.NewReader(file), + total: size, + current: 0, + callback: up, + } + + _, _, err = d.protonDrive.UploadFileByReader(ctx, parentLinkID, fileName, fileInfo.ModTime(), reader, 0) + if err != nil { + return fmt.Errorf("failed to upload file: %w", err) + } + + return nil +} + +func (d *ProtonDrive) ensureTempServer() error { + if d.tempServer != nil { + + // Already running + return nil + } + + listener, err := net.Listen("tcp", ":0") + if err != nil { + return err + } + d.tempServerPort = listener.Addr().(*net.TCPAddr).Port + + mux := http.NewServeMux() + mux.HandleFunc("/temp/", d.handleTempDownload) + + d.tempServer = &http.Server{ + Handler: mux, + } + + go func() { + d.tempServer.Serve(listener) + }() + + return nil +} + +func (d *ProtonDrive) handleTempDownload(w http.ResponseWriter, r *http.Request) { + token := strings.TrimPrefix(r.URL.Path, "/temp/") + + d.tokenMutex.RLock() + info, exists := d.downloadTokens[token] + d.tokenMutex.RUnlock() + + if !exists { + http.Error(w, "Invalid or expired token", http.StatusNotFound) + return + } + + link, err := d.protonDrive.GetLink(r.Context(), info.LinkID) + if err != nil { + http.Error(w, "Failed to get file link", http.StatusInternalServerError) + return + } + + // Get file size for range calculations + _, _, attrs, err := d.protonDrive.DownloadFile(r.Context(), link, 0) + if err != nil { + http.Error(w, "Failed to get file info", http.StatusInternalServerError) + return + } + + fileSize := attrs.Size + + rangeHeader := r.Header.Get("Range") + if rangeHeader != "" { + + // Parse range header like "bytes=0-1023" or "bytes=1024-" + ranges, err := parseRange(rangeHeader, fileSize) + if err != nil { + http.Error(w, "Invalid range", http.StatusRequestedRangeNotSatisfiable) + return + } + + if len(ranges) == 1 { + + // Single range request, small + start, end := ranges[0].start, ranges[0].end + contentLength := end - start + 1 + + // Start download from offset + reader, _, _, err := d.protonDrive.DownloadFile(r.Context(), link, start) + if err != nil { + http.Error(w, "Failed to start download", http.StatusInternalServerError) + return + } + defer reader.Close() + + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fileSize)) + w.Header().Set("Content-Length", fmt.Sprintf("%d", contentLength)) + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(link.Name))) + + // Partial content... + // Setting fileName is more cosmetical here + //.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", link.Name)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", info.FileName)) + w.Header().Set("Accept-Ranges", "bytes") + + w.WriteHeader(http.StatusPartialContent) + + io.CopyN(w, reader, contentLength) + return + } + } + + // Full file download (non-range request) + reader, _, _, err := d.protonDrive.DownloadFile(r.Context(), link, 0) + if err != nil { + http.Error(w, "Failed to start download", http.StatusInternalServerError) + return + } + defer reader.Close() + + // Set headers for full content + w.Header().Set("Content-Length", fmt.Sprintf("%d", fileSize)) + w.Header().Set("Content-Type", mime.TypeByExtension(filepath.Ext(link.Name))) + + // Setting fileName is needed since ProtonDrive fileName is more like a random string + //w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", link.Name)) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", info.FileName)) + + w.Header().Set("Accept-Ranges", "bytes") + + // Stream the full file + io.Copy(w, reader) +} + +func (d *ProtonDrive) generateDownloadToken(linkID, fileName string) string { + token := fmt.Sprintf("%d_%s", time.Now().UnixNano(), linkID[:8]) + + d.tokenMutex.Lock() + if d.downloadTokens == nil { + d.downloadTokens = make(map[string]*downloadInfo) + } + + d.downloadTokens[token] = &downloadInfo{ + LinkID: linkID, + FileName: fileName, + } + + d.tokenMutex.Unlock() + + go func() { + + // Token expires in 1 hour + time.Sleep(1 * time.Hour) + d.tokenMutex.Lock() + + delete(d.downloadTokens, token) + d.tokenMutex.Unlock() + }() + + return token +} + +func parseRange(rangeHeader string, size int64) ([]httpRange, error) { + if !strings.HasPrefix(rangeHeader, "bytes=") { + return nil, fmt.Errorf("invalid range header") + } + + rangeSpec := strings.TrimPrefix(rangeHeader, "bytes=") + ranges := strings.Split(rangeSpec, ",") + + var result []httpRange + for _, r := range ranges { + r = strings.TrimSpace(r) + if strings.Contains(r, "-") { + parts := strings.Split(r, "-") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid range format") + } + + var start, end int64 + var err error + + if parts[0] == "" { + + // Suffix range (e.g., "-500") + if parts[1] == "" { + return nil, fmt.Errorf("invalid range format") + } + end = size - 1 + start, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + start = size - start + if start < 0 { + start = 0 + } + } else if parts[1] == "" { + + // Prefix range (e.g., "500-") + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, err + } + end = size - 1 + } else { + // Full range (e.g., "0-1023") + start, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return nil, err + } + end, err = strconv.ParseInt(parts[1], 10, 64) + if err != nil { + return nil, err + } + } + + if start >= size || end >= size || start > end { + return nil, fmt.Errorf("range out of bounds") + } + + result = append(result, httpRange{start: start, end: end}) + } + } + + return result, nil +} + +func (d *ProtonDrive) encryptFileName(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Temporary file (request) + tempReq := proton.CreateFileReq{ + SignatureAddress: d.MainShare.Creator, + } + + // Encrypt the filename + err = tempReq.SetName(name, d.DefaultAddrKR, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to encrypt filename: %w", err) + } + + return tempReq.Name, nil +} + +func (d *ProtonDrive) generateFileNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func (d *ProtonDrive) getOriginalNameHash(link *proton.Link) (string, error) { + if link == nil { + return "", fmt.Errorf("link cannot be nil") + } + + if link.Hash == "" { + return "", fmt.Errorf("link hash is empty") + } + + return link.Hash, nil +} + +func (d *ProtonDrive) getLink(ctx context.Context, linkID string) (*proton.Link, error) { + if linkID == "" { + return nil, fmt.Errorf("linkID cannot be empty") + } + + link, err := d.c.GetLink(ctx, d.MainShare.ShareID, linkID) + if err != nil { + return nil, err + } + + return &link, nil +} + +func (d *ProtonDrive) getLinkKR(ctx context.Context, link *proton.Link) (*crypto.KeyRing, error) { + if link == nil { + return nil, fmt.Errorf("link cannot be nil") + } + + // Root Link or Root Dir + if link.ParentLinkID == "" { + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + return link.GetKeyRing(d.MainShareKR, signatureVerificationKR) + } + + // Get parent keyring recursively + parentLink, err := d.getLink(ctx, link.ParentLinkID) + if err != nil { + return nil, err + } + + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return nil, err + } + + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{link.SignatureEmail}) + if err != nil { + return nil, err + } + + return link.GetKeyRing(parentNodeKR, signatureVerificationKR) +} + +var ( + ErrKeyPassOrSaltedKeyPassMustBeNotNil = errors.New("either keyPass or saltedKeyPass must be not nil") + ErrFailedToUnlockUserKeys = errors.New("failed to unlock user keys") +) + +func getAccountKRs(ctx context.Context, c *proton.Client, keyPass, saltedKeyPass []byte) (*crypto.KeyRing, map[string]*crypto.KeyRing, map[string]proton.Address, []byte, error) { + + user, err := c.GetUser(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("user %#v", user) + + addrsArr, err := c.GetAddresses(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("addr %#v", addr) + + if saltedKeyPass == nil { + if keyPass == nil { + return nil, nil, nil, nil, ErrKeyPassOrSaltedKeyPassMustBeNotNil + } + + // Due to limitations, salts are stored using cacheCredentialToFile + salts, err := c.GetSalts(ctx) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("salts %#v", salts) + + saltedKeyPass, err = salts.SaltForKey(keyPass, user.Keys.Primary().ID) + if err != nil { + return nil, nil, nil, nil, err + } + // fmt.Printf("saltedKeyPass ok") + } + + userKR, addrKRs, err := proton.Unlock(user, addrsArr, saltedKeyPass, nil) + if err != nil { + return nil, nil, nil, nil, err + + } else if userKR.CountDecryptionEntities() == 0 { + return nil, nil, nil, nil, ErrFailedToUnlockUserKeys + } + + addrs := make(map[string]proton.Address) + for _, addr := range addrsArr { + addrs[addr.Email] = addr + } + + return userKR, addrKRs, addrs, saltedKeyPass, nil +} + +func (d *ProtonDrive) getSignatureVerificationKeyring(emailAddresses []string, verificationAddrKRs ...*crypto.KeyRing) (*crypto.KeyRing, error) { + ret, err := crypto.NewKeyRing(nil) + if err != nil { + return nil, err + } + + for _, emailAddress := range emailAddresses { + if addr, ok := d.addrData[emailAddress]; ok { + if addrKR, exists := d.addrKRs[addr.ID]; exists { + err = d.addKeysFromKR(ret, addrKR) + if err != nil { + return nil, err + } + } + } + } + + for _, kr := range verificationAddrKRs { + err = d.addKeysFromKR(ret, kr) + if err != nil { + return nil, err + } + } + + if ret.CountEntities() == 0 { + return nil, fmt.Errorf("no keyring for signature verification") + } + + return ret, nil +} + +func (d *ProtonDrive) addKeysFromKR(kr *crypto.KeyRing, newKRs ...*crypto.KeyRing) error { + for i := range newKRs { + for _, key := range newKRs[i].GetKeys() { + err := kr.AddKey(key) + if err != nil { + return err + } + } + } + return nil +} + +func (d *ProtonDrive) DirectRename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + //fmt.Printf("DEBUG DirectRename: path=%s, newName=%s", srcObj.GetPath(), newName) + + if d.MainShare == nil || d.DefaultAddrKR == nil { + return nil, fmt.Errorf("missing required fields: MainShare=%v, DefaultAddrKR=%v", + d.MainShare != nil, d.DefaultAddrKR != nil) + } + + if d.protonDrive == nil { + return nil, fmt.Errorf("protonDrive bridge is nil") + } + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), srcObj.IsDir()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + parentLinkID := srcLink.ParentLinkID + if parentLinkID == "" { + return nil, fmt.Errorf("cannot rename root folder") + } + + encryptedName, err := d.encryptFileName(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateFileNameHash(ctx, newName, parentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + renameReq := RenameRequest{ + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + } + + err = d.executeRenameAPI(ctx, srcLink.LinkID, renameReq) + if err != nil { + return nil, fmt.Errorf("rename API call failed: %w", err) + } + + return &model.Object{ + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) executeRenameAPI(ctx context.Context, linkID string, req RenameRequest) error { + + renameURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/rename", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal rename request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", renameURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.credentials.UID) + httpReq.Header.Set("Authorization", "Bearer "+d.credentials.AccessToken) + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute rename request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("rename failed with status %d", resp.StatusCode) + } + + var renameResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&renameResp); err != nil { + return fmt.Errorf("failed to decode rename response: %w", err) + } + + if renameResp.Code != 1000 { + return fmt.Errorf("rename failed with code %d", renameResp.Code) + } + + return nil +} + +func (d *ProtonDrive) executeMoveAPI(ctx context.Context, linkID string, req MoveRequest) error { + //fmt.Printf("DEBUG Move Request - Name: %s\n", req.Name) + //fmt.Printf("DEBUG Move Request - Hash: %s\n", req.Hash) + //fmt.Printf("DEBUG Move Request - OriginalHash: %s\n", req.OriginalHash) + //fmt.Printf("DEBUG Move Request - ParentLinkID: %s\n", req.ParentLinkID) + + //fmt.Printf("DEBUG Move Request - Name length: %d\n", len(req.Name)) + //fmt.Printf("DEBUG Move Request - NameSignatureEmail: %s\n", req.NameSignatureEmail) + //fmt.Printf("DEBUG Move Request - ContentHash: %v\n", req.ContentHash) + //fmt.Printf("DEBUG Move Request - NodePassphrase length: %d\n", len(req.NodePassphrase)) + //fmt.Printf("DEBUG Move Request - NodePassphraseSignature length: %d\n", len(req.NodePassphraseSignature)) + + //fmt.Printf("DEBUG Move Request - SrcLinkID: %s\n", linkID) + //fmt.Printf("DEBUG Move Request - DstParentLinkID: %s\n", req.ParentLinkID) + //fmt.Printf("DEBUG Move Request - ShareID: %s\n", d.MainShare.ShareID) + + srcLink, _ := d.getLink(ctx, linkID) + if srcLink != nil && srcLink.ParentLinkID == req.ParentLinkID { + return fmt.Errorf("cannot move to same parent directory") + } + + moveURL := fmt.Sprintf(d.apiBase+"/drive/v2/volumes/%s/links/%s/move", + d.MainShare.VolumeID, linkID) + + reqBody, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal move request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, "PUT", moveURL, bytes.NewReader(reqBody)) + if err != nil { + return fmt.Errorf("failed to create HTTP request: %w", err) + } + + httpReq.Header.Set("Authorization", "Bearer "+d.credentials.AccessToken) + httpReq.Header.Set("Accept", d.protonJson) + httpReq.Header.Set("X-Pm-Appversion", d.webDriveAV) + httpReq.Header.Set("X-Pm-Drive-Sdk-Version", d.sdkVersion) + httpReq.Header.Set("X-Pm-Uid", d.credentials.UID) + httpReq.Header.Set("Content-Type", "application/json") + + client := &http.Client{} + resp, err := client.Do(httpReq) + if err != nil { + return fmt.Errorf("failed to execute move request: %w", err) + } + defer resp.Body.Close() + + var moveResp RenameResponse + if err := json.NewDecoder(resp.Body).Decode(&moveResp); err != nil { + return fmt.Errorf("failed to decode move response: %w", err) + } + + if moveResp.Code != 1000 { + return fmt.Errorf("move operation failed with code: %d", moveResp.Code) + } + + return nil +} + +func (d *ProtonDrive) DirectMove(ctx context.Context, srcObj model.Obj, dstDir model.Obj) (model.Obj, error) { + //fmt.Printf("DEBUG DirectMove: srcPath=%s, dstPath=%s", srcObj.GetPath(), dstDir.GetPath()) + + srcLink, err := d.searchByPath(ctx, srcObj.GetPath(), srcObj.IsDir()) + if err != nil { + return nil, fmt.Errorf("failed to find source: %w", err) + } + + var dstParentLinkID string + if dstDir.GetPath() == "/" { + dstParentLinkID = d.RootLink.LinkID + } else { + dstLink, err := d.searchByPath(ctx, dstDir.GetPath(), true) + if err != nil { + return nil, fmt.Errorf("failed to find destination: %w", err) + } + dstParentLinkID = dstLink.LinkID + } + + if srcObj.IsDir() { + + // Check if destination is a descendant of source + if err := d.checkCircularMove(ctx, srcLink.LinkID, dstParentLinkID); err != nil { + return nil, err + } + } + + // Encrypt the filename for the new location + encryptedName, err := d.encryptFileName(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to encrypt filename: %w", err) + } + + newHash, err := d.generateNameHash(ctx, srcObj.GetName(), dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to generate new hash: %w", err) + } + + originalHash, err := d.getOriginalNameHash(srcLink) + if err != nil { + return nil, fmt.Errorf("failed to get original hash: %w", err) + } + + // Re-encrypt node passphrase for new parent context + reencryptedPassphrase, err := d.reencryptNodePassphrase(ctx, srcLink, dstParentLinkID) + if err != nil { + return nil, fmt.Errorf("failed to re-encrypt node passphrase: %w", err) + } + + moveReq := MoveRequest{ + ParentLinkID: dstParentLinkID, + NodePassphrase: reencryptedPassphrase, + Name: encryptedName, + NameSignatureEmail: d.MainShare.Creator, + Hash: newHash, + OriginalHash: originalHash, + ContentHash: nil, + + // *** Causes rejection *** + /* NodePassphraseSignature: srcLink.NodePassphraseSignature, */ + } + + //fmt.Printf("DEBUG MoveRequest validation:\n") + //fmt.Printf(" Name length: %d\n", len(moveReq.Name)) + //fmt.Printf(" Hash: %s\n", moveReq.Hash) + //fmt.Printf(" OriginalHash: %s\n", moveReq.OriginalHash) + //fmt.Printf(" NodePassphrase length: %d\n", len(moveReq.NodePassphrase)) + /* fmt.Printf(" NodePassphraseSignature length: %d\n", len(moveReq.NodePassphraseSignature)) */ + //fmt.Printf(" NameSignatureEmail: %s\n", moveReq.NameSignatureEmail) + + err = d.executeMoveAPI(ctx, srcLink.LinkID, moveReq) + if err != nil { + return nil, fmt.Errorf("move API call failed: %w", err) + } + + return &model.Object{ + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *ProtonDrive) reencryptNodePassphrase(ctx context.Context, srcLink *proton.Link, dstParentLinkID string) (string, error) { + // Get source parent link with metadata + srcParentLink, err := d.getLink(ctx, srcLink.ParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get source parent link: %w", err) + } + + // Get source parent keyring using link object + srcParentKR, err := d.getLinkKR(ctx, srcParentLink) + if err != nil { + return "", fmt.Errorf("failed to get source parent keyring: %w", err) + } + + // Get destination parent link with metadata + dstParentLink, err := d.getLink(ctx, dstParentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get destination parent link: %w", err) + } + + // Get destination parent keyring using link object + dstParentKR, err := d.getLinkKR(ctx, dstParentLink) + if err != nil { + return "", fmt.Errorf("failed to get destination parent keyring: %w", err) + } + + // Re-encrypt the node passphrase from source parent context to destination parent context + reencryptedPassphrase, err := reencryptKeyPacket(srcParentKR, dstParentKR, d.DefaultAddrKR, srcLink.NodePassphrase) + if err != nil { + return "", fmt.Errorf("failed to re-encrypt key packet: %w", err) + } + + return reencryptedPassphrase, nil +} + +func (d *ProtonDrive) generateNameHash(ctx context.Context, name string, parentLinkID string) (string, error) { + + parentLink, err := d.getLink(ctx, parentLinkID) + if err != nil { + return "", fmt.Errorf("failed to get parent link: %w", err) + } + + // Get parent node keyring + parentNodeKR, err := d.getLinkKR(ctx, parentLink) + if err != nil { + return "", fmt.Errorf("failed to get parent keyring: %w", err) + } + + // Get signature verification keyring + signatureVerificationKR, err := d.getSignatureVerificationKeyring([]string{parentLink.SignatureEmail}, parentNodeKR) + if err != nil { + return "", fmt.Errorf("failed to get signature verification keyring: %w", err) + } + + parentHashKey, err := parentLink.GetHashKey(parentNodeKR, signatureVerificationKR) + if err != nil { + return "", fmt.Errorf("failed to get parent hash key: %w", err) + } + + nameHash, err := proton.GetNameHash(name, parentHashKey) + if err != nil { + return "", fmt.Errorf("failed to generate name hash: %w", err) + } + + return nameHash, nil +} + +func reencryptKeyPacket(srcKR, dstKR, _ *crypto.KeyRing, passphrase string) (string, error) { // addrKR (3) + oldSplitMessage, err := crypto.NewPGPSplitMessageFromArmored(passphrase) + if err != nil { + return "", err + } + + sessionKey, err := srcKR.DecryptSessionKey(oldSplitMessage.KeyPacket) + if err != nil { + return "", err + } + + newKeyPacket, err := dstKR.EncryptSessionKey(sessionKey) + if err != nil { + return "", err + } + + newSplitMessage := crypto.NewPGPSplitMessage(newKeyPacket, oldSplitMessage.DataPacket) + + return newSplitMessage.GetArmored() +} + +func (d *ProtonDrive) checkCircularMove(ctx context.Context, srcLinkID, dstParentLinkID string) error { + currentLinkID := dstParentLinkID + + for currentLinkID != "" && currentLinkID != d.RootLink.LinkID { + if currentLinkID == srcLinkID { + return fmt.Errorf("cannot move folder into itself or its subfolder") + } + + currentLink, err := d.getLink(ctx, currentLinkID) + if err != nil { + return err + } + currentLinkID = currentLink.ParentLinkID + } + + return nil +} diff --git a/drivers/quark_uc/driver.go b/drivers/quark_uc/driver.go index 7c254022a92..691874c321e 100644 --- a/drivers/quark_uc/driver.go +++ b/drivers/quark_uc/driver.go @@ -1,18 +1,21 @@ package quark import ( + "bytes" "context" - "crypto/md5" - "crypto/sha1" "encoding/hex" + "hash" "io" "net/http" + "strings" "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + streamPkg "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" @@ -35,6 +38,14 @@ func (d *QuarkOrUC) GetAddition() driver.Additional { func (d *QuarkOrUC) Init(ctx context.Context) error { _, err := d.request("/config", http.MethodGet, nil, nil) + if err == nil && d.AdditionVersion != 2 { + d.AdditionVersion = 2 + if !d.UseTransCodingAddress && len(d.DownProxyUrl) == 0 { + d.WebProxy = true + d.WebdavPolicy = "native_proxy" + } + op.MustSaveDriverStorage(d) + } return err } @@ -43,39 +54,23 @@ func (d *QuarkOrUC) Drop(ctx context.Context) error { } func (d *QuarkOrUC) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { - files, err := d.GetFiles(dir.GetID()) - if err != nil { - return nil, err - } - return utils.SliceConvert(files, func(src File) (model.Obj, error) { - return fileToObj(src), nil - }) + return d.GetFiles(dir.GetID()) } func (d *QuarkOrUC) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - data := base.Json{ - "fids": []string{file.GetID()}, - } - var resp DownResp - ua := d.conf.ua - _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { - req.SetHeader("User-Agent", ua). - SetBody(data) - }, &resp) - if err != nil { + f := file.(*File) + if d.UseTransCodingAddress && d.config.Name == "Quark" && f.Category == 1 && f.Size > 0 { + link, err := d.getTranscodingLink(file) + if err == nil { + return link, nil + } + if strings.Contains(err.Error(), "plf_invalid") { + log.Warnf("quark transcoding link invalid for %s, fallback to download link: %v", file.GetName(), err) + return d.getDownloadLink(file) + } return nil, err } - - return &model.Link{ - URL: resp.Data[0].DownloadUrl, - Header: http.Header{ - "Cookie": []string{d.Cookie}, - "Referer": []string{d.conf.referer}, - "User-Agent": []string{ua}, - }, - Concurrency: 2, - PartSize: 10 * utils.MB, - }, nil + return d.getDownloadLink(file) } func (d *QuarkOrUC) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { @@ -135,33 +130,33 @@ func (d *QuarkOrUC) Remove(ctx context.Context, obj model.Obj) error { } func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - tempFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - defer func() { - _ = tempFile.Close() - }() - m := md5.New() - _, err = io.Copy(m, tempFile) - if err != nil { - return err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err - } - md5Str := hex.EncodeToString(m.Sum(nil)) - s := sha1.New() - _, err = io.Copy(s, tempFile) - if err != nil { - return err - } - _, err = tempFile.Seek(0, io.SeekStart) - if err != nil { - return err + md5Str, sha1Str := stream.GetHash().GetHash(utils.MD5), stream.GetHash().GetHash(utils.SHA1) + var ( + md5 hash.Hash + sha1 hash.Hash + ) + writers := []io.Writer{} + if len(md5Str) != utils.MD5.Width { + md5 = utils.MD5.NewFunc() + writers = append(writers, md5) + } + if len(sha1Str) != utils.SHA1.Width { + sha1 = utils.SHA1.NewFunc() + writers = append(writers, sha1) + } + + if len(writers) > 0 { + _, err := streamPkg.CacheFullInTempFileAndWriter(stream, io.MultiWriter(writers...)) + if err != nil { + return err + } + if md5 != nil { + md5Str = hex.EncodeToString(md5.Sum(nil)) + } + if sha1 != nil { + sha1Str = hex.EncodeToString(sha1.Sum(nil)) + } } - sha1Str := hex.EncodeToString(s.Sum(nil)) // pre pre, err := d.upPre(stream, dstDir.GetID()) if err != nil { @@ -177,29 +172,31 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File return nil } // part up - partSize := pre.Metadata.PartSize - var bytes []byte - md5s := make([]string, 0) - defaultBytes := make([]byte, partSize) total := stream.GetSize() left := total + partSize := int64(pre.Metadata.PartSize) + part := make([]byte, partSize) + count := int(total / partSize) + if total%partSize > 0 { + count++ + } + md5s := make([]string, 0, count) partNumber := 1 for left > 0 { if utils.IsCanceled(ctx) { return ctx.Err() } - if left > int64(partSize) { - bytes = defaultBytes - } else { - bytes = make([]byte, left) + if left < partSize { + part = part[:left] } - _, err := io.ReadFull(tempFile, bytes) + n, err := io.ReadFull(stream, part) if err != nil { return err } - left -= int64(len(bytes)) + left -= int64(n) log.Debugf("left: %d", left) - m, err := d.upPart(ctx, pre, stream.GetMimetype(), partNumber, bytes) + reader := driver.NewLimitedUploadStream(ctx, bytes.NewReader(part)) + m, err := d.upPart(ctx, pre, stream.GetMimetype(), partNumber, reader) //m, err := driver.UpPart(pre, file.GetMIMEType(), partNumber, bytes, account, md5Str, sha1Str) if err != nil { return err @@ -209,7 +206,7 @@ func (d *QuarkOrUC) Put(ctx context.Context, dstDir model.Obj, stream model.File } md5s = append(md5s, m) partNumber++ - up(int(100 * (total - left) / total)) + up(100 * float64(total-left) / float64(total)) } err = d.upCommit(pre, md5s) if err != nil { diff --git a/drivers/quark_uc/meta.go b/drivers/quark_uc/meta.go index f3acfe88562..6940c44b9d6 100644 --- a/drivers/quark_uc/meta.go +++ b/drivers/quark_uc/meta.go @@ -8,8 +8,11 @@ import ( type Addition struct { Cookie string `json:"cookie" required:"true"` driver.RootID - OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` - OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + OrderBy string `json:"order_by" type:"select" options:"none,file_type,file_name,updated_at" default:"none"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + UseTransCodingAddress bool `json:"use_transcoding_address" help:"You can watch the transcoded video and support 302 redirection" required:"true" default:"false"` + OnlyListVideoFile bool `json:"only_list_video_file" default:"false"` + AdditionVersion int } type Conf struct { @@ -24,7 +27,6 @@ func init() { return &QuarkOrUC{ config: driver.Config{ Name: "Quark", - OnlyLocal: true, DefaultRoot: "0", NoOverwriteUpload: true, }, @@ -40,7 +42,7 @@ func init() { return &QuarkOrUC{ config: driver.Config{ Name: "UC", - OnlyLocal: true, + OnlyProxy: true, DefaultRoot: "0", NoOverwriteUpload: true, }, diff --git a/drivers/quark_uc/types.go b/drivers/quark_uc/types.go index afbdb3eff89..13bfac2f7d5 100644 --- a/drivers/quark_uc/types.go +++ b/drivers/quark_uc/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" ) type Resp struct { @@ -18,13 +19,13 @@ type File struct { Fid string `json:"fid"` FileName string `json:"file_name"` //PdirFid string `json:"pdir_fid"` - //Category int `json:"category"` + Category int `json:"category"` //FileType int `json:"file_type"` Size int64 `json:"size"` //FormatType string `json:"format_type"` //Status int `json:"status"` //Tags string `json:"tags,omitempty"` - //LCreatedAt int64 `json:"l_created_at"` + LCreatedAt int64 `json:"l_created_at"` LUpdatedAt int64 `json:"l_updated_at"` //NameSpace int `json:"name_space"` //IncludeItems int `json:"include_items,omitempty"` @@ -32,8 +33,8 @@ type File struct { //BackupSign int `json:"backup_sign"` //Duration int `json:"duration"` //FileSource string `json:"file_source"` - File bool `json:"file"` - //CreatedAt int64 `json:"created_at"` + File bool `json:"file"` + CreatedAt int64 `json:"created_at"` UpdatedAt int64 `json:"updated_at"` //PrivateExtra struct {} `json:"_private_extra"` //ObjCategory string `json:"obj_category,omitempty"` @@ -50,6 +51,38 @@ func fileToObj(f File) *model.Object { } } +func (f *File) GetSize() int64 { + return f.Size +} + +func (f *File) GetName() string { + return f.FileName +} + +func (f *File) ModTime() time.Time { + return time.UnixMilli(f.UpdatedAt) +} + +func (f *File) CreateTime() time.Time { + return time.UnixMilli(f.CreatedAt) +} + +func (f *File) IsDir() bool { + return !f.File +} + +func (f *File) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f *File) GetID() string { + return f.Fid +} + +func (f *File) GetPath() string { + return "" +} + type SortResp struct { Resp Data struct { @@ -100,6 +133,39 @@ type DownResp struct { //} `json:"metadata"` } +type TranscodingResp struct { + Resp + Data struct { + DefaultResolution string `json:"default_resolution"` + OriginDefaultResolution string `json:"origin_default_resolution"` + VideoList []struct { + Resolution string `json:"resolution"` + VideoInfo struct { + Duration int `json:"duration"` + Size int64 `json:"size"` + Format string `json:"format"` + Width int `json:"width"` + Height int `json:"height"` + Bitrate float64 `json:"bitrate"` + Codec string `json:"codec"` + Fps float64 `json:"fps"` + Rotate int `json:"rotate"` + UpdateTime int64 `json:"update_time"` + URL string `json:"url"` + Resolution string `json:"resolution"` + HlsType string `json:"hls_type"` + Finish bool `json:"finish"` + Resoultion string `json:"resoultion"` + Success bool `json:"success"` + } `json:"video_info,omitempty"` + } `json:"video_list"` + FileName string `json:"file_name"` + NameSpace int `json:"name_space"` + Size int64 `json:"size"` + Thumbnail string `json:"thumbnail"` + } `json:"data"` +} + type UpPreResp struct { Resp Data struct { diff --git a/drivers/quark_uc/util.go b/drivers/quark_uc/util.go index df27af6714f..2f99c308f33 100644 --- a/drivers/quark_uc/util.go +++ b/drivers/quark_uc/util.go @@ -6,6 +6,8 @@ import ( "encoding/base64" "errors" "fmt" + "html" + "io" "net/http" "strconv" "strings" @@ -49,20 +51,29 @@ func (d *QuarkOrUC) request(pathname string, method string, callback base.ReqCal d.Cookie = cookie.SetStr(d.Cookie, "__puus", __puus.Value) op.MustSaveDriverStorage(d) } + if d.UseTransCodingAddress && d.config.Name == "Quark" { + __pus := cookie.GetCookie(res.Cookies(), "__pus") + if __pus != nil { + d.Cookie = cookie.SetStr(d.Cookie, "__pus", __pus.Value) + op.MustSaveDriverStorage(d) + } + } if e.Status >= 400 || e.Code != 0 { return nil, errors.New(e.Message) } return res.Body(), nil } -func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { - files := make([]File, 0) +func (d *QuarkOrUC) GetFiles(parent string) ([]model.Obj, error) { + files := make([]model.Obj, 0) page := 1 size := 100 query := map[string]string{ - "pdir_fid": parent, - "_size": strconv.Itoa(size), - "_fetch_total": "1", + "pdir_fid": parent, + "_size": strconv.Itoa(size), + "_fetch_total": "1", + "fetch_all_file": "1", + "fetch_risk_file_name": "1", } if d.OrderBy != "none" { query["_sort"] = "file_type:asc," + d.OrderBy + ":" + d.OrderDirection @@ -76,7 +87,16 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { if err != nil { return nil, err } - files = append(files, resp.Data.List...) + for _, file := range resp.Data.List { + file.FileName = html.UnescapeString(file.FileName) + if d.OnlyListVideoFile { + if file.IsDir() || file.Category == 1 { + files = append(files, &file) + } + } else { + files = append(files, &file) + } + } if page*size >= resp.Metadata.Total { break } @@ -85,6 +105,62 @@ func (d *QuarkOrUC) GetFiles(parent string) ([]File, error) { return files, nil } +func (d *QuarkOrUC) getDownloadLink(file model.Obj) (*model.Link, error) { + data := base.Json{ + "fids": []string{file.GetID()}, + } + var resp DownResp + ua := d.conf.ua + _, err := d.request("/file/download", http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua). + SetBody(data) + }, &resp) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: resp.Data[0].DownloadUrl, + Header: http.Header{ + "Cookie": []string{d.Cookie}, + "Referer": []string{d.conf.referer}, + "User-Agent": []string{ua}, + }, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil +} + +func (d *QuarkOrUC) getTranscodingLink(file model.Obj) (*model.Link, error) { + data := base.Json{ + "fid": file.GetID(), + "resolutions": "low,normal,high,super,2k,4k", + "supports": "fmp4_av,m3u8,dolby_vision", + } + var resp TranscodingResp + ua := d.conf.ua + + _, err := d.request("/file/v2/play/project", http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", ua). + SetBody(data) + }, &resp) + if err != nil { + return nil, err + } + + for _, info := range resp.Data.VideoList { + if info.VideoInfo.URL != "" { + return &model.Link{ + URL: info.VideoInfo.URL, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil + } + } + + return nil, errors.New("no link found") +} + func (d *QuarkOrUC) upPre(file model.FileStreamer, parentId string) (UpPreResp, error) { now := time.Now() data := base.Json{ @@ -119,7 +195,7 @@ func (d *QuarkOrUC) upHash(md5, sha1, taskId string) (bool, error) { return resp.Data.Finish, err } -func (d *QuarkOrUC) upPart(ctx context.Context, pre UpPreResp, mineType string, partNumber int, bytes []byte) (string, error) { +func (d *QuarkOrUC) upPart(ctx context.Context, pre UpPreResp, mineType string, partNumber int, bytes io.Reader) (string, error) { //func (driver QuarkOrUC) UpPart(pre UpPreResp, mineType string, partNumber int, bytes []byte, account *model.Account, md5Str, sha1Str string) (string, error) { timeStr := time.Now().UTC().Format(http.TimeFormat) data := base.Json{ @@ -163,10 +239,13 @@ x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit "partNumber": strconv.Itoa(partNumber), "uploadId": pre.Data.UploadId, }).SetBody(bytes).Put(u) + if err != nil { + return "", err + } if res.StatusCode() != 200 { return "", fmt.Errorf("up status: %d, error: %s", res.StatusCode(), res.String()) } - return res.Header().Get("ETag"), nil + return res.Header().Get("Etag"), nil } func (d *QuarkOrUC) upCommit(pre UpPreResp, md5s []string) error { @@ -230,6 +309,9 @@ x-oss-user-agent:aliyun-sdk-js/6.6.1 Chrome 98.0.4758.80 on Windows 10 64-bit SetQueryParams(map[string]string{ "uploadId": pre.Data.UploadId, }).SetBody(body).Post(u) + if err != nil { + return err + } if res.StatusCode() != 200 { return fmt.Errorf("up status: %d, error: %s", res.StatusCode(), res.String()) } diff --git a/drivers/quark_uc_tv/driver.go b/drivers/quark_uc_tv/driver.go new file mode 100644 index 00000000000..a857e2dd841 --- /dev/null +++ b/drivers/quark_uc_tv/driver.go @@ -0,0 +1,177 @@ +package quark_uc_tv + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" +) + +type QuarkUCTV struct { + *QuarkUCTVCommon + model.Storage + Addition + config driver.Config + conf Conf +} + +func (d *QuarkUCTV) Config() driver.Config { + return d.config +} + +func (d *QuarkUCTV) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *QuarkUCTV) Init(ctx context.Context) error { + + if d.Addition.DeviceID == "" { + d.Addition.DeviceID = utils.GetMD5EncodeStr(time.Now().String()) + } + op.MustSaveDriverStorage(d) + + if d.QuarkUCTVCommon == nil { + d.QuarkUCTVCommon = &QuarkUCTVCommon{ + AccessToken: "", + } + } + ctx1, cancelFunc := context.WithTimeout(ctx, 5*time.Second) + defer cancelFunc() + if d.Addition.RefreshToken == "" { + if d.Addition.QueryToken == "" { + qrData, err := d.getLoginCode(ctx1) + if err != nil { + return err + } + // 展示二维码 + qrTemplate := ` + + ` + qrPage := fmt.Sprintf(qrTemplate, qrData) + return fmt.Errorf("need verify: \n%s", qrPage) + } else { + // 通过query token获取code -> refresh token + code, err := d.getCode(ctx1) + if err != nil { + return err + } + // 通过code获取refresh token + err = d.getRefreshTokenByTV(ctx1, code, false) + if err != nil { + return err + } + } + } + // 通过refresh token获取access token + if d.QuarkUCTVCommon.AccessToken == "" { + err := d.getRefreshTokenByTV(ctx1, d.Addition.RefreshToken, true) + if err != nil { + return err + } + } + + // 验证 access token 是否有效 + _, err := d.isLogin(ctx1) + if err != nil { + return err + } + return nil +} + +func (d *QuarkUCTV) Drop(ctx context.Context) error { + return nil +} + +func (d *QuarkUCTV) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files := make([]model.Obj, 0) + pageIndex := int64(0) + pageSize := int64(100) + for { + var filesData FilesData + _, err := d.request(ctx, "/file", "GET", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "method": "list", + "parent_fid": dir.GetID(), + "order_by": "3", + "desc": "1", + "category": "", + "source": "", + "ex_source": "", + "list_all": "0", + "page_size": strconv.FormatInt(pageSize, 10), + "page_index": strconv.FormatInt(pageIndex, 10), + }) + }, &filesData) + if err != nil { + return nil, err + } + for i := range filesData.Data.Files { + files = append(files, &filesData.Data.Files[i]) + } + if pageIndex*pageSize >= filesData.Data.TotalCount { + break + } else { + pageIndex++ + } + } + return files, nil +} + +func (d *QuarkUCTV) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var fileLink FileLink + _, err := d.request(ctx, "/file", "GET", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "method": "download", + "group_by": "source", + "fid": file.GetID(), + "resolution": "low,normal,high,super,2k,4k", + "support": "dolby_vision", + }) + }, &fileLink) + if err != nil { + return nil, err + } + + return &model.Link{ + URL: fileLink.Data.DownloadURL, + Concurrency: 3, + PartSize: 10 * utils.MB, + }, nil +} + +func (d *QuarkUCTV) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *QuarkUCTV) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *QuarkUCTV) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *QuarkUCTV) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *QuarkUCTV) Remove(ctx context.Context, obj model.Obj) error { + return errs.NotImplement +} + +func (d *QuarkUCTV) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + return nil, errs.NotImplement +} + +type QuarkUCTVCommon struct { + AccessToken string +} + +var _ driver.Driver = (*QuarkUCTV)(nil) diff --git a/drivers/quark_uc_tv/meta.go b/drivers/quark_uc_tv/meta.go new file mode 100644 index 00000000000..cf7e478566e --- /dev/null +++ b/drivers/quark_uc_tv/meta.go @@ -0,0 +1,67 @@ +package quark_uc_tv + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // Usually one of two + driver.RootID + // define other + RefreshToken string `json:"refresh_token" required:"false" default:""` + // 必要且影响登录,由签名决定 + DeviceID string `json:"device_id" required:"false" default:""` + // 登陆所用的数据 无需手动填写 + QueryToken string `json:"query_token" required:"false" default:"" help:"don't edit'"` +} + +type Conf struct { + api string + clientID string + signKey string + appVer string + channel string + codeApi string +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &QuarkUCTV{ + config: driver.Config{ + Name: "QuarkTV", + OnlyLocal: false, + DefaultRoot: "0", + NoOverwriteUpload: true, + NoUpload: true, + }, + conf: Conf{ + api: "https://open-api-drive.quark.cn", + clientID: "d3194e61504e493eb6222857bccfed94", + signKey: "kw2dvtd7p4t3pjl2d9ed9yc8yej8kw2d", + appVer: "1.5.6", + channel: "CP", + codeApi: "http://api.extscreen.com/quarkdrive", + }, + } + }) + op.RegisterDriver(func() driver.Driver { + return &QuarkUCTV{ + config: driver.Config{ + Name: "UCTV", + OnlyLocal: false, + DefaultRoot: "0", + NoOverwriteUpload: true, + NoUpload: true, + }, + conf: Conf{ + api: "https://open-api-drive.uc.cn", + clientID: "5acf882d27b74502b7040b0c65519aa7", + signKey: "l3srvtd7p42l0d0x1u8d7yc8ye9kki4d", + appVer: "1.6.5", + channel: "UCTVOFFICIALWEB", + codeApi: "http://api.extscreen.com/ucdrive", + }, + } + }) +} diff --git a/drivers/quark_uc_tv/types.go b/drivers/quark_uc_tv/types.go new file mode 100644 index 00000000000..fb35b8b2d6d --- /dev/null +++ b/drivers/quark_uc_tv/types.go @@ -0,0 +1,102 @@ +package quark_uc_tv + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "time" +) + +type Resp struct { + CommonRsp + Errno int `json:"errno"` + ErrorInfo string `json:"error_info"` +} + +type CommonRsp struct { + Status int `json:"status"` + ReqID string `json:"req_id"` +} + +type RefreshTokenAuthResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Status int `json:"status"` + Errno int `json:"errno"` + ErrorInfo string `json:"error_info"` + ReqID string `json:"req_id"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int `json:"expires_in"` + Scope string `json:"scope"` + } `json:"data"` +} +type Files struct { + Fid string `json:"fid"` + ParentFid string `json:"parent_fid"` + Category int `json:"category"` + Filename string `json:"filename"` + Size int64 `json:"size"` + FileType string `json:"file_type"` + SubItems int `json:"sub_items,omitempty"` + Isdir int `json:"isdir"` + Duration int `json:"duration"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` + IsBackup int `json:"is_backup"` + ThumbnailURL string `json:"thumbnail_url,omitempty"` +} + +func (f *Files) GetSize() int64 { + return f.Size +} + +func (f *Files) GetName() string { + return f.Filename +} + +func (f *Files) ModTime() time.Time { + //return time.Unix(f.UpdatedAt, 0) + return time.Unix(0, f.UpdatedAt*int64(time.Millisecond)) +} + +func (f *Files) CreateTime() time.Time { + //return time.Unix(f.CreatedAt, 0) + return time.Unix(0, f.CreatedAt*int64(time.Millisecond)) +} + +func (f *Files) IsDir() bool { + return f.Isdir == 1 +} + +func (f *Files) GetHash() utils.HashInfo { + return utils.HashInfo{} +} + +func (f *Files) GetID() string { + return f.Fid +} + +func (f *Files) GetPath() string { + return "" +} + +var _ model.Obj = (*Files)(nil) + +type FilesData struct { + CommonRsp + Data struct { + TotalCount int64 `json:"total_count"` + Files []Files `json:"files"` + } `json:"data"` +} + +type FileLink struct { + CommonRsp + Data struct { + Fid string `json:"fid"` + FileName string `json:"file_name"` + Size int64 `json:"size"` + DownloadURL string `json:"download_url"` + } `json:"data"` +} diff --git a/drivers/quark_uc_tv/util.go b/drivers/quark_uc_tv/util.go new file mode 100644 index 00000000000..fefbb0361fb --- /dev/null +++ b/drivers/quark_uc_tv/util.go @@ -0,0 +1,211 @@ +package quark_uc_tv + +import ( + "context" + "crypto/md5" + "crypto/sha256" + "encoding/hex" + "errors" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "net/http" + "strconv" + "time" +) + +const ( + UserAgent = "Mozilla/5.0 (Linux; U; Android 13; zh-cn; M2004J7AC Build/UKQ1.231108.001) AppleWebKit/533.1 (KHTML, like Gecko) Mobile Safari/533.1" + DeviceBrand = "Xiaomi" + Platform = "tv" + DeviceName = "M2004J7AC" + DeviceModel = "M2004J7AC" + BuildDevice = "M2004J7AC" + BuildProduct = "M2004J7AC" + DeviceGpu = "Adreno (TM) 550" + ActivityRect = "{}" +) + +func (d *QuarkUCTV) request(ctx context.Context, pathname string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + u := d.conf.api + pathname + tm, token, reqID := d.generateReqSign(method, pathname, d.conf.signKey) + req := base.RestyClient.R() + req.SetContext(ctx) + req.SetHeaders(map[string]string{ + "Accept": "application/json, text/plain, */*", + "User-Agent": UserAgent, + "x-pan-tm": tm, + "x-pan-token": token, + "x-pan-client-id": d.conf.clientID, + }) + req.SetQueryParams(map[string]string{ + "req_id": reqID, + "access_token": d.QuarkUCTVCommon.AccessToken, + "app_ver": d.conf.appVer, + "device_id": d.Addition.DeviceID, + "device_brand": DeviceBrand, + "platform": Platform, + "device_name": DeviceName, + "device_model": DeviceModel, + "build_device": BuildDevice, + "build_product": BuildProduct, + "device_gpu": DeviceGpu, + "activity_rect": ActivityRect, + "channel": d.conf.channel, + }) + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + var e Resp + req.SetError(&e) + res, err := req.Execute(method, u) + if err != nil { + return nil, err + } + // 判断 是否需要 刷新 access_token + if e.Status == -1 && e.Errno == 10001 { + // token 过期 + err = d.getRefreshTokenByTV(ctx, d.Addition.RefreshToken, true) + if err != nil { + return nil, err + } + ctx1, cancelFunc := context.WithTimeout(ctx, 10*time.Second) + defer cancelFunc() + return d.request(ctx1, pathname, method, callback, resp) + } + + if e.Status >= 400 || e.Errno != 0 { + return nil, errors.New(e.ErrorInfo) + } + return res.Body(), nil +} + +func (d *QuarkUCTV) getLoginCode(ctx context.Context) (string, error) { + // 获取登录二维码 + pathname := "/oauth/authorize" + var resp struct { + CommonRsp + QrData string `json:"qr_data"` + QueryToken string `json:"query_token"` + } + _, err := d.request(ctx, pathname, "GET", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "auth_type": "code", + "client_id": d.conf.clientID, + "scope": "netdisk", + "qrcode": "1", + "qr_width": "460", + "qr_height": "460", + }) + }, &resp) + if err != nil { + return "", err + } + // 保存query_token 用于后续登录 + if resp.QueryToken != "" { + d.Addition.QueryToken = resp.QueryToken + op.MustSaveDriverStorage(d) + } + return resp.QrData, nil +} + +func (d *QuarkUCTV) getCode(ctx context.Context) (string, error) { + // 通过query token获取code + pathname := "/oauth/code" + var resp struct { + CommonRsp + Code string `json:"code"` + } + _, err := d.request(ctx, pathname, "GET", func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "client_id": d.conf.clientID, + "scope": "netdisk", + "query_token": d.Addition.QueryToken, + }) + }, &resp) + if err != nil { + return "", err + } + return resp.Code, nil +} + +func (d *QuarkUCTV) getRefreshTokenByTV(ctx context.Context, code string, isRefresh bool) error { + pathname := "/token" + _, _, reqID := d.generateReqSign("POST", pathname, d.conf.signKey) + u := d.conf.codeApi + pathname + var resp RefreshTokenAuthResp + body := map[string]string{ + "req_id": reqID, + "app_ver": d.conf.appVer, + "device_id": d.Addition.DeviceID, + "device_brand": DeviceBrand, + "platform": Platform, + "device_name": DeviceName, + "device_model": DeviceModel, + "build_device": BuildDevice, + "build_product": BuildProduct, + "device_gpu": DeviceGpu, + "activity_rect": ActivityRect, + "channel": d.conf.channel, + } + if isRefresh { + body["refresh_token"] = code + } else { + body["code"] = code + } + + _, err := base.RestyClient.R(). + SetHeader("Content-Type", "application/json"). + SetBody(body). + SetResult(&resp). + SetContext(ctx). + Post(u) + if err != nil { + return err + } + if resp.Code != 200 { + return errors.New(resp.Message) + } + if resp.Data.RefreshToken != "" { + d.Addition.RefreshToken = resp.Data.RefreshToken + op.MustSaveDriverStorage(d) + d.QuarkUCTVCommon.AccessToken = resp.Data.AccessToken + } else { + return errors.New("refresh token is empty") + } + return nil +} + +func (d *QuarkUCTV) isLogin(ctx context.Context) (bool, error) { + _, err := d.request(ctx, "/user", http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "method": "user_info", + }) + }, nil) + return err == nil, err +} + +func (d *QuarkUCTV) generateReqSign(method string, pathname string, key string) (string, string, string) { + //timestamp 13位时间戳 + timestamp := strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10) + deviceID := d.Addition.DeviceID + if deviceID == "" { + deviceID = utils.GetMD5EncodeStr(timestamp) + d.Addition.DeviceID = deviceID + op.MustSaveDriverStorage(d) + } + // 生成req_id + reqID := md5.Sum([]byte(deviceID + timestamp)) + reqIDHex := hex.EncodeToString(reqID[:]) + + // 生成x_pan_token + tokenData := method + "&" + pathname + "&" + timestamp + "&" + key + xPanToken := sha256.Sum256([]byte(tokenData)) + xPanTokenHex := hex.EncodeToString(xPanToken[:]) + + return timestamp, xPanTokenHex, reqIDHex +} diff --git a/drivers/quqi/driver.go b/drivers/quqi/driver.go new file mode 100644 index 00000000000..36758bd1d14 --- /dev/null +++ b/drivers/quqi/driver.go @@ -0,0 +1,452 @@ +package quqi + +import ( + "bytes" + "context" + "errors" + "io" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +type Quqi struct { + model.Storage + Addition + Cookie string // Cookie + GroupID string // 私人云群组ID + ClientID string // 随机生成客户端ID 经过测试,部分接口调用若不携带client id会出现错误 +} + +func (d *Quqi) Config() driver.Config { + return config +} + +func (d *Quqi) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Quqi) Init(ctx context.Context) error { + // 登录 + if err := d.login(); err != nil { + return err + } + + // 生成随机client id (与网页端生成逻辑一致) + d.ClientID = "quqipc_" + random.String(10) + + // 获取私人云ID (暂时仅获取私人云) + groupResp := &GroupRes{} + if _, err := d.request("group.quqi.com", "/v1/group/list", resty.MethodGet, nil, groupResp); err != nil { + return err + } + for _, groupInfo := range groupResp.Data { + if groupInfo == nil { + continue + } + if groupInfo.Type == 2 { + d.GroupID = strconv.Itoa(groupInfo.ID) + break + } + } + if d.GroupID == "" { + return errs.StorageNotFound + } + + return nil +} + +func (d *Quqi) Drop(ctx context.Context) error { + return nil +} + +func (d *Quqi) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + var ( + listResp = &ListRes{} + files []model.Obj + ) + + if _, err := d.request("", "/api/dir/ls", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": dir.GetID(), + "client_id": d.ClientID, + }) + }, listResp); err != nil { + return nil, err + } + + if listResp.Data == nil { + return nil, nil + } + + // dirs + for _, dirInfo := range listResp.Data.Dir { + if dirInfo == nil { + continue + } + files = append(files, &model.Object{ + ID: strconv.FormatInt(dirInfo.NodeID, 10), + Name: dirInfo.Name, + Modified: time.Unix(dirInfo.UpdateTime, 0), + Ctime: time.Unix(dirInfo.AddTime, 0), + IsFolder: true, + }) + } + + // files + for _, fileInfo := range listResp.Data.File { + if fileInfo == nil { + continue + } + if fileInfo.EXT != "" { + fileInfo.Name = strings.Join([]string{fileInfo.Name, fileInfo.EXT}, ".") + } + + files = append(files, &model.Object{ + ID: strconv.FormatInt(fileInfo.NodeID, 10), + Name: fileInfo.Name, + Size: fileInfo.Size, + Modified: time.Unix(fileInfo.UpdateTime, 0), + Ctime: time.Unix(fileInfo.AddTime, 0), + }) + } + + return files, nil +} + +func (d *Quqi) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if d.CDN { + link, err := d.linkFromCDN(file.GetID()) + if err != nil { + log.Warn(err) + } else { + return link, nil + } + } + + link, err := d.linkFromPreview(file.GetID()) + if err != nil { + log.Warn(err) + } else { + return link, nil + } + + link, err = d.linkFromDownload(file.GetID()) + if err != nil { + return nil, err + } + return link, nil +} + +func (d *Quqi) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + var ( + makeDirRes = &MakeDirRes{} + timeNow = time.Now() + ) + + if _, err := d.request("", "/api/dir/mkDir", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "parent_id": parentDir.GetID(), + "name": dirName, + "client_id": d.ClientID, + }) + }, makeDirRes); err != nil { + return nil, err + } + + return &model.Object{ + ID: strconv.FormatInt(makeDirRes.Data.NodeID, 10), + Name: dirName, + Modified: timeNow, + Ctime: timeNow, + IsFolder: true, + }, nil +} + +func (d *Quqi) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + var moveRes = &MoveRes{} + + if _, err := d.request("", "/api/dir/mvDir", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": dstDir.GetID(), + "source_quqi_id": d.GroupID, + "source_tree_id": "1", + "source_node_id": srcObj.GetID(), + "client_id": d.ClientID, + }) + }, moveRes); err != nil { + return nil, err + } + + return &model.Object{ + ID: strconv.FormatInt(moveRes.Data.NodeID, 10), + Name: moveRes.Data.NodeName, + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Quqi) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + var realName = newName + + if !srcObj.IsDir() { + srcExt, newExt := utils.Ext(srcObj.GetName()), utils.Ext(newName) + + // 曲奇网盘的文件名称由文件名和扩展名组成,若存在扩展名,则重命名时仅支持更改文件名,扩展名在曲奇服务端保留 + if srcExt != "" && srcExt == newExt { + parts := strings.Split(newName, ".") + if len(parts) > 1 { + realName = strings.Join(parts[:len(parts)-1], ".") + } + } + } + + if _, err := d.request("", "/api/dir/renameDir", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": srcObj.GetID(), + "rename": realName, + "client_id": d.ClientID, + }) + }, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: time.Now(), + Ctime: srcObj.CreateTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Quqi) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + // 无法从曲奇接口响应中直接获取复制后的文件信息 + if _, err := d.request("", "/api/node/copy", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": dstDir.GetID(), + "source_quqi_id": d.GroupID, + "source_tree_id": "1", + "source_node_id": srcObj.GetID(), + "client_id": d.ClientID, + }) + }, nil); err != nil { + return nil, err + } + + return nil, nil +} + +func (d *Quqi) Remove(ctx context.Context, obj model.Obj) error { + // 暂时不做直接删除,默认都放到回收站。直接删除方法:先调用删除接口放入回收站,在通过回收站接口删除文件 + if _, err := d.request("", "/api/node/del", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": obj.GetID(), + "client_id": d.ClientID, + }) + }, nil); err != nil { + return err + } + + return nil +} + +func (d *Quqi) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + // base info + sizeStr := strconv.FormatInt(stream.GetSize(), 10) + f, err := stream.CacheFullInTempFile() + if err != nil { + return nil, err + } + md5, err := utils.HashFile(utils.MD5, f) + if err != nil { + return nil, err + } + sha, err := utils.HashFile(utils.SHA256, f) + if err != nil { + return nil, err + } + // init upload + var uploadInitResp UploadInitResp + _, err = d.request("", "/api/upload/v1/file/init", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "parent_id": dstDir.GetID(), + "size": sizeStr, + "file_name": stream.GetName(), + "md5": md5, + "sha": sha, + "is_slice": "true", + "client_id": d.ClientID, + }) + }, &uploadInitResp) + if err != nil { + return nil, err + } + // check exist + // if the file already exists in Quqi server, there is no need to actually upload it + if uploadInitResp.Data.Exist { + // the file name returned by Quqi does not include the extension name + nodeName, nodeExt := uploadInitResp.Data.NodeName, utils.Ext(stream.GetName()) + if nodeExt != "" { + nodeName = nodeName + "." + nodeExt + } + return &model.Object{ + ID: strconv.FormatInt(uploadInitResp.Data.NodeID, 10), + Name: nodeName, + Size: stream.GetSize(), + Modified: stream.ModTime(), + Ctime: stream.CreateTime(), + }, nil + } + // listParts + _, err = d.request("upload.quqi.com:20807", "/upload/v1/listParts", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "token": uploadInitResp.Data.Token, + "task_id": uploadInitResp.Data.TaskID, + "client_id": d.ClientID, + }) + }, nil) + if err != nil { + return nil, err + } + // get temp key + var tempKeyResp TempKeyResp + _, err = d.request("upload.quqi.com:20807", "/upload/v1/tempKey", resty.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "token": uploadInitResp.Data.Token, + "task_id": uploadInitResp.Data.TaskID, + }) + }, &tempKeyResp) + if err != nil { + return nil, err + } + // upload + // u, err := url.Parse(fmt.Sprintf("https://%s.cos.ap-shanghai.myqcloud.com", uploadInitResp.Data.Bucket)) + // b := &cos.BaseURL{BucketURL: u} + // client := cos.NewClient(b, &http.Client{ + // Transport: &cos.CredentialTransport{ + // Credential: cos.NewTokenCredential(tempKeyResp.Data.Credentials.TmpSecretID, tempKeyResp.Data.Credentials.TmpSecretKey, tempKeyResp.Data.Credentials.SessionToken), + // }, + // }) + // partSize := int64(1024 * 1024 * 2) + // partCount := (stream.GetSize() + partSize - 1) / partSize + // for i := 1; i <= int(partCount); i++ { + // length := partSize + // if i == int(partCount) { + // length = stream.GetSize() - (int64(i)-1)*partSize + // } + // _, err := client.Object.UploadPart( + // ctx, uploadInitResp.Data.Key, uploadInitResp.Data.UploadID, i, io.LimitReader(f, partSize), &cos.ObjectUploadPartOptions{ + // ContentLength: length, + // }, + // ) + // if err != nil { + // return nil, err + // } + // } + + cfg := &aws.Config{ + Credentials: credentials.NewStaticCredentials(tempKeyResp.Data.Credentials.TmpSecretID, tempKeyResp.Data.Credentials.TmpSecretKey, tempKeyResp.Data.Credentials.SessionToken), + Region: aws.String("ap-shanghai"), + Endpoint: aws.String("cos.ap-shanghai.myqcloud.com"), + } + s, err := session.NewSession(cfg) + if err != nil { + return nil, err + } + uploader := s3manager.NewUploader(s) + buf := make([]byte, 1024*1024*2) + fup := &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{ + Reader: f, + Size: int64(len(buf)), + }, + UpdateProgress: up, + } + for partNumber := int64(1); ; partNumber++ { + n, err := io.ReadFull(fup, buf) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + if err == io.EOF { + break + } + return nil, err + } + reader := bytes.NewReader(buf[:n]) + _, err = uploader.S3.UploadPartWithContext(ctx, &s3.UploadPartInput{ + UploadId: &uploadInitResp.Data.UploadID, + Key: &uploadInitResp.Data.Key, + Bucket: &uploadInitResp.Data.Bucket, + PartNumber: aws.Int64(partNumber), + Body: struct { + *driver.RateLimitReader + io.Seeker + }{ + RateLimitReader: driver.NewLimitedUploadStream(ctx, reader), + Seeker: reader, + }, + }) + if err != nil { + return nil, err + } + } + // finish upload + var uploadFinishResp UploadFinishResp + _, err = d.request("", "/api/upload/v1/file/finish", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "token": uploadInitResp.Data.Token, + "task_id": uploadInitResp.Data.TaskID, + "client_id": d.ClientID, + }) + }, &uploadFinishResp) + if err != nil { + return nil, err + } + // the file name returned by Quqi does not include the extension name + nodeName, nodeExt := uploadFinishResp.Data.NodeName, utils.Ext(stream.GetName()) + if nodeExt != "" { + nodeName = nodeName + "." + nodeExt + } + return &model.Object{ + ID: strconv.FormatInt(uploadFinishResp.Data.NodeID, 10), + Name: nodeName, + Size: stream.GetSize(), + Modified: stream.ModTime(), + Ctime: stream.CreateTime(), + }, nil +} + +//func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Quqi)(nil) diff --git a/drivers/quqi/meta.go b/drivers/quqi/meta.go new file mode 100644 index 00000000000..aaaa0a19444 --- /dev/null +++ b/drivers/quqi/meta.go @@ -0,0 +1,28 @@ +package quqi + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Phone string `json:"phone"` + Password string `json:"password"` + Cookie string `json:"cookie" help:"Cookie can be used on multiple clients at the same time"` + CDN bool `json:"cdn" help:"If you enable this option, the download speed can be increased, but there will be some performance loss"` +} + +var config = driver.Config{ + Name: "Quqi", + OnlyLocal: true, + LocalSort: true, + //NoUpload: true, + DefaultRoot: "0", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Quqi{} + }) +} diff --git a/drivers/quqi/types.go b/drivers/quqi/types.go new file mode 100644 index 00000000000..cade93de885 --- /dev/null +++ b/drivers/quqi/types.go @@ -0,0 +1,197 @@ +package quqi + +type BaseReqQuery struct { + ID string `json:"quqiid"` +} + +type BaseReq struct { + GroupID string `json:"quqi_id"` +} + +type BaseRes struct { + //Data interface{} `json:"data"` + Code int `json:"err"` + Message string `json:"msg"` +} + +type GroupRes struct { + BaseRes + Data []*Group `json:"data"` +} + +type ListRes struct { + BaseRes + Data *List `json:"data"` +} + +type GetDocRes struct { + BaseRes + Data struct { + OriginPath string `json:"origin_path"` + } `json:"data"` +} + +type GetDownloadResp struct { + BaseRes + Data struct { + Url string `json:"url"` + } `json:"data"` +} + +type MakeDirRes struct { + BaseRes + Data struct { + IsRoot bool `json:"is_root"` + NodeID int64 `json:"node_id"` + ParentID int64 `json:"parent_id"` + } `json:"data"` +} + +type MoveRes struct { + BaseRes + Data struct { + NodeChildNum int64 `json:"node_child_num"` + NodeID int64 `json:"node_id"` + NodeName string `json:"node_name"` + ParentID int64 `json:"parent_id"` + GroupID int64 `json:"quqi_id"` + TreeID int64 `json:"tree_id"` + } `json:"data"` +} + +type RenameRes struct { + BaseRes + Data struct { + NodeID int64 `json:"node_id"` + GroupID int64 `json:"quqi_id"` + Rename string `json:"rename"` + TreeID int64 `json:"tree_id"` + UpdateTime int64 `json:"updatetime"` + } `json:"data"` +} + +type CopyRes struct { + BaseRes +} + +type RemoveRes struct { + BaseRes +} + +type Group struct { + ID int `json:"quqi_id"` + Type int `json:"type"` + Name string `json:"name"` + IsAdministrator int `json:"is_administrator"` + Role []int `json:"role"` + Avatar string `json:"avatar_url"` + IsStick int `json:"is_stick"` + Nickname string `json:"nickname"` + Status int `json:"status"` +} + +type List struct { + ListDir + Dir []*ListDir `json:"dir"` + File []*ListFile `json:"file"` +} + +type ListItem struct { + AddTime int64 `json:"add_time"` + IsDir int `json:"is_dir"` + IsExpand int `json:"is_expand"` + IsFinalize int `json:"is_finalize"` + LastEditorName string `json:"last_editor_name"` + Name string `json:"name"` + NodeID int64 `json:"nid"` + ParentID int64 `json:"parent_id"` + Permission int `json:"permission"` + TreeID int64 `json:"tid"` + UpdateCNT int64 `json:"update_cnt"` + UpdateTime int64 `json:"update_time"` +} + +type ListDir struct { + ListItem + ChildDocNum int64 `json:"child_doc_num"` + DirDetail string `json:"dir_detail"` + DirType int `json:"dir_type"` +} + +type ListFile struct { + ListItem + BroadDocType string `json:"broad_doc_type"` + CanDisplay bool `json:"can_display"` + Detail string `json:"detail"` + EXT string `json:"ext"` + Filetype string `json:"filetype"` + HasMobileThumbnail bool `json:"has_mobile_thumbnail"` + HasThumbnail bool `json:"has_thumbnail"` + Size int64 `json:"size"` + Version int `json:"version"` +} + +type UploadInitResp struct { + Data struct { + Bucket string `json:"bucket"` + Exist bool `json:"exist"` + Key string `json:"key"` + TaskID string `json:"task_id"` + Token string `json:"token"` + UploadID string `json:"upload_id"` + URL string `json:"url"` + NodeID int64 `json:"node_id"` + NodeName string `json:"node_name"` + ParentID int64 `json:"parent_id"` + } `json:"data"` + Err int `json:"err"` + Msg string `json:"msg"` +} + +type TempKeyResp struct { + Err int `json:"err"` + Msg string `json:"msg"` + Data struct { + ExpiredTime int `json:"expiredTime"` + Expiration string `json:"expiration"` + Credentials struct { + SessionToken string `json:"sessionToken"` + TmpSecretID string `json:"tmpSecretId"` + TmpSecretKey string `json:"tmpSecretKey"` + } `json:"credentials"` + RequestID string `json:"requestId"` + StartTime int `json:"startTime"` + } `json:"data"` +} + +type UploadFinishResp struct { + Data struct { + NodeID int64 `json:"node_id"` + NodeName string `json:"node_name"` + ParentID int64 `json:"parent_id"` + QuqiID int64 `json:"quqi_id"` + TreeID int64 `json:"tree_id"` + } `json:"data"` + Err int `json:"err"` + Msg string `json:"msg"` +} + +type UrlExchangeResp struct { + BaseRes + Data struct { + Name string `json:"name"` + Mime string `json:"mime"` + Size int64 `json:"size"` + DownloadType int `json:"download_type"` + ChannelType int `json:"channel_type"` + ChannelID int `json:"channel_id"` + Url string `json:"url"` + ExpiredTime int64 `json:"expired_time"` + IsEncrypted bool `json:"is_encrypted"` + EncryptedSize int64 `json:"encrypted_size"` + EncryptedAlg string `json:"encrypted_alg"` + EncryptedKey string `json:"encrypted_key"` + PassportID int64 `json:"passport_id"` + RequestExpiredTime int64 `json:"request_expired_time"` + } `json:"data"` +} diff --git a/drivers/quqi/util.go b/drivers/quqi/util.go new file mode 100644 index 00000000000..aa184d70fc4 --- /dev/null +++ b/drivers/quqi/util.go @@ -0,0 +1,299 @@ +package quqi + +import ( + "bufio" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/minio/sio" +) + +// do others that not defined in Driver interface +func (d *Quqi) request(host string, path string, method string, callback base.ReqCallback, resp interface{}) (*resty.Response, error) { + var ( + reqUrl = url.URL{ + Scheme: "https", + Host: "quqi.com", + Path: path, + } + req = base.RestyClient.R() + result BaseRes + ) + + if host != "" { + reqUrl.Host = host + } + req.SetHeaders(map[string]string{ + "Origin": "https://quqi.com", + "Cookie": d.Cookie, + }) + + if d.GroupID != "" { + req.SetQueryParam("quqiid", d.GroupID) + } + + if callback != nil { + callback(req) + } + + res, err := req.Execute(method, reqUrl.String()) + if err != nil { + return nil, err + } + // resty.Request.SetResult cannot parse result correctly sometimes + err = utils.Json.Unmarshal(res.Body(), &result) + if err != nil { + return nil, err + } + if result.Code != 0 { + return nil, errors.New(result.Message) + } + if resp != nil { + err = utils.Json.Unmarshal(res.Body(), resp) + if err != nil { + return nil, err + } + } + return res, nil +} + +func (d *Quqi) login() error { + if d.Addition.Cookie != "" { + d.Cookie = d.Addition.Cookie + } + if d.checkLogin() { + return nil + } + if d.Cookie != "" { + return errors.New("cookie is invalid") + } + if d.Phone == "" { + return errors.New("phone number is empty") + } + if d.Password == "" { + return errs.EmptyPassword + } + + resp, err := d.request("", "/auth/person/v2/login/password", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "phone": d.Phone, + "password": base64.StdEncoding.EncodeToString([]byte(d.Password)), + }) + }, nil) + if err != nil { + return err + } + + var cookies []string + for _, cookie := range resp.RawResponse.Cookies() { + cookies = append(cookies, fmt.Sprintf("%s=%s", cookie.Name, cookie.Value)) + } + d.Cookie = strings.Join(cookies, ";") + + return nil +} + +func (d *Quqi) checkLogin() bool { + if _, err := d.request("", "/auth/account/baseInfo", resty.MethodGet, nil, nil); err != nil { + return false + } + return true +} + +// decryptKey 获取密码 +func decryptKey(encodeKey string) []byte { + // 移除非法字符 + u := strings.ReplaceAll(encodeKey, "[^A-Za-z0-9+\\/]", "") + + // 计算输出字节数组的长度 + o := len(u) + a := 32 + + // 创建输出字节数组 + c := make([]byte, a) + + // 编码循环 + s := uint32(0) // 累加器 + f := 0 // 输出数组索引 + for l := 0; l < o; l++ { + r := l & 3 // 取模4,得到当前字符在四字节块中的位置 + i := u[l] // 当前字符的ASCII码 + + // 编码当前字符 + switch { + case i >= 65 && i < 91: // 大写字母 + s |= uint32(i-65) << uint32(6*(3-r)) + case i >= 97 && i < 123: // 小写字母 + s |= uint32(i-71) << uint32(6*(3-r)) + case i >= 48 && i < 58: // 数字 + s |= uint32(i+4) << uint32(6*(3-r)) + case i == 43: // 加号 + s |= uint32(62) << uint32(6*(3-r)) + case i == 47: // 斜杠 + s |= uint32(63) << uint32(6*(3-r)) + } + + // 如果累加器已经包含了四个字符,或者是最后一个字符,则写入输出数组 + if r == 3 || l == o-1 { + for e := 0; e < 3 && f < a; e, f = e+1, f+1 { + c[f] = byte(s >> (16 >> e & 24) & 255) + } + s = 0 + } + } + + return c +} + +func (d *Quqi) linkFromPreview(id string) (*model.Link, error) { + var getDocResp GetDocRes + if _, err := d.request("", "/api/doc/getDoc", resty.MethodPost, func(req *resty.Request) { + req.SetFormData(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": id, + "client_id": d.ClientID, + }) + }, &getDocResp); err != nil { + return nil, err + } + if getDocResp.Data.OriginPath == "" { + return nil, errors.New("cannot get link from preview") + } + return &model.Link{ + URL: getDocResp.Data.OriginPath, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Quqi) linkFromDownload(id string) (*model.Link, error) { + var getDownloadResp GetDownloadResp + if _, err := d.request("", "/api/doc/getDownload", resty.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "quqi_id": d.GroupID, + "tree_id": "1", + "node_id": id, + "url_type": "undefined", + "entry_type": "undefined", + "client_id": d.ClientID, + "no_redirect": "1", + }) + }, &getDownloadResp); err != nil { + return nil, err + } + if getDownloadResp.Data.Url == "" { + return nil, errors.New("cannot get link from download") + } + + return &model.Link{ + URL: getDownloadResp.Data.Url, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Quqi) linkFromCDN(id string) (*model.Link, error) { + downloadLink, err := d.linkFromDownload(id) + if err != nil { + return nil, err + } + + var urlExchangeResp UrlExchangeResp + if _, err = d.request("api.quqi.com", "/preview/downloadInfo/url/exchange", resty.MethodGet, func(req *resty.Request) { + req.SetQueryParam("url", downloadLink.URL) + }, &urlExchangeResp); err != nil { + return nil, err + } + if urlExchangeResp.Data.Url == "" { + return nil, errors.New("cannot get link from cdn") + } + + // 假设存在未加密的情况 + if !urlExchangeResp.Data.IsEncrypted { + return &model.Link{ + URL: urlExchangeResp.Data.Url, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }, nil + } + + // 根据sio(https://github.com/minio/sio/blob/master/DARE.md)描述及实际测试,得出以下结论: + // 1. 加密后大小(encrypted_size)-原始文件大小(size) = 加密包的头大小+身份验证标识 = (16+16) * N -> N为加密包的数量 + // 2. 原始文件大小(size)+64*1024-1 / (64*1024) = N -> 每个包的有效负载为64K + remoteClosers := utils.EmptyClosers() + payloadSize := int64(1 << 16) + expiration := time.Until(time.Unix(urlExchangeResp.Data.ExpiredTime, 0)) + resultRangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + encryptedOffset := httpRange.Start / payloadSize * (payloadSize + 32) + decryptedOffset := httpRange.Start % payloadSize + encryptedLength := (httpRange.Length+httpRange.Start+payloadSize-1)/payloadSize*(payloadSize+32) - encryptedOffset + if httpRange.Length < 0 { + encryptedLength = httpRange.Length + } else { + if httpRange.Length+httpRange.Start >= urlExchangeResp.Data.Size || encryptedLength+encryptedOffset >= urlExchangeResp.Data.EncryptedSize { + encryptedLength = -1 + } + } + //log.Debugf("size: %d\tencrypted_size: %d", urlExchangeResp.Data.Size, urlExchangeResp.Data.EncryptedSize) + //log.Debugf("http range offset: %d, length: %d", httpRange.Start, httpRange.Length) + //log.Debugf("encrypted offset: %d, length: %d, decrypted offset: %d", encryptedOffset, encryptedLength, decryptedOffset) + + rrc, err := stream.GetRangeReadCloserFromLink(urlExchangeResp.Data.EncryptedSize, &model.Link{ + URL: urlExchangeResp.Data.Url, + Header: http.Header{ + "Origin": []string{"https://quqi.com"}, + "Cookie": []string{d.Cookie}, + }, + }) + if err != nil { + return nil, err + } + + rc, err := rrc.RangeRead(ctx, http_range.Range{Start: encryptedOffset, Length: encryptedLength}) + remoteClosers.AddClosers(rrc.GetClosers()) + if err != nil { + return nil, err + } + + decryptReader, err := sio.DecryptReader(rc, sio.Config{ + MinVersion: sio.Version10, + MaxVersion: sio.Version20, + CipherSuites: []byte{sio.CHACHA20_POLY1305, sio.AES_256_GCM}, + Key: decryptKey(urlExchangeResp.Data.EncryptedKey), + SequenceNumber: uint32(httpRange.Start / payloadSize), + }) + if err != nil { + return nil, err + } + bufferReader := bufio.NewReader(decryptReader) + bufferReader.Discard(int(decryptedOffset)) + + return io.NopCloser(bufferReader), nil + } + + return &model.Link{ + RangeReadCloser: &model.RangeReadCloser{RangeReader: resultRangeReader, Closers: remoteClosers}, + Expiration: &expiration, + }, nil +} diff --git a/drivers/s3/doge.go b/drivers/s3/doge.go new file mode 100644 index 00000000000..12a584ca4f2 --- /dev/null +++ b/drivers/s3/doge.go @@ -0,0 +1,63 @@ +package s3 + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "strings" +) + +type TmpTokenResponse struct { + Code int `json:"code"` + Msg string `json:"msg"` + Data TmpTokenResponseData `json:"data,omitempty"` +} +type TmpTokenResponseData struct { + Credentials Credentials `json:"Credentials"` + ExpiredAt int `json:"ExpiredAt"` +} +type Credentials struct { + AccessKeyId string `json:"accessKeyId,omitempty"` + SecretAccessKey string `json:"secretAccessKey,omitempty"` + SessionToken string `json:"sessionToken,omitempty"` +} + +func getCredentials(AccessKey, SecretKey string) (rst Credentials, err error) { + apiPath := "/auth/tmp_token.json" + reqBody, err := json.Marshal(map[string]interface{}{"channel": "OSS_FULL", "scopes": []string{"*"}}) + if err != nil { + return rst, err + } + + signStr := apiPath + "\n" + string(reqBody) + hmacObj := hmac.New(sha1.New, []byte(SecretKey)) + hmacObj.Write([]byte(signStr)) + sign := hex.EncodeToString(hmacObj.Sum(nil)) + Authorization := "TOKEN " + AccessKey + ":" + sign + + req, err := http.NewRequest("POST", "https://api.dogecloud.com"+apiPath, strings.NewReader(string(reqBody))) + if err != nil { + return rst, err + } + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Authorization", Authorization) + client := http.Client{} + resp, err := client.Do(req) + if err != nil { + return rst, err + } + defer resp.Body.Close() + ret, err := io.ReadAll(resp.Body) + if err != nil { + return rst, err + } + var tmpTokenResp TmpTokenResponse + err = json.Unmarshal(ret, &tmpTokenResp) + if err != nil { + return rst, err + } + return tmpTokenResp.Data.Credentials, nil +} diff --git a/drivers/s3/driver.go b/drivers/s3/driver.go index dd643f5d76e..896f69b3028 100644 --- a/drivers/s3/driver.go +++ b/drivers/s3/driver.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "github.com/alist-org/alist/v3/internal/stream" "io" "net/url" stdpath "path" @@ -13,6 +12,10 @@ import ( "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/server/common" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -25,10 +28,40 @@ type S3 struct { Session *session.Session client *s3.S3 linkClient *s3.S3 + + config driver.Config + cron *cron.Cron +} + +var storageClassLookup = map[string]string{ + "standard": s3.ObjectStorageClassStandard, + "reduced_redundancy": s3.ObjectStorageClassReducedRedundancy, + "glacier": s3.ObjectStorageClassGlacier, + "standard_ia": s3.ObjectStorageClassStandardIa, + "onezone_ia": s3.ObjectStorageClassOnezoneIa, + "intelligent_tiering": s3.ObjectStorageClassIntelligentTiering, + "deep_archive": s3.ObjectStorageClassDeepArchive, + "outposts": s3.ObjectStorageClassOutposts, + "glacier_ir": s3.ObjectStorageClassGlacierIr, + "snow": s3.ObjectStorageClassSnow, + "express_onezone": s3.ObjectStorageClassExpressOnezone, +} + +func (d *S3) resolveStorageClass() *string { + value := strings.TrimSpace(d.StorageClass) + if value == "" { + return nil + } + normalized := strings.ToLower(strings.ReplaceAll(value, "-", "_")) + if v, ok := storageClassLookup[normalized]; ok { + return aws.String(v) + } + log.Warnf("s3: unknown storage class %q, using raw value", d.StorageClass) + return aws.String(value) } func (d *S3) Config() driver.Config { - return config + return d.config } func (d *S3) GetAddition() driver.Additional { @@ -36,9 +69,24 @@ func (d *S3) GetAddition() driver.Additional { } func (d *S3) Init(ctx context.Context) error { + if !strings.Contains(d.Storage.Addition, `"use_placeholder"`) { + d.UsePlaceholder = true + } if d.Region == "" { d.Region = "alist" } + if d.config.Name == "Doge" { + // 多吉云每次临时生成的秘钥有效期为 2h,所以这里设置为 118 分钟重新生成一次 + d.cron = cron.NewCron(time.Minute * 118) + d.cron.Do(func() { + err := d.initSession() + if err != nil { + log.Errorln("Doge init session error:", err) + } + d.client = d.getClient(false) + d.linkClient = d.getClient(true) + }) + } err := d.initSession() if err != nil { return err @@ -49,6 +97,9 @@ func (d *S3) Init(ctx context.Context) error { } func (d *S3) Drop(ctx context.Context) error { + if d.cron != nil { + d.cron.Stop() + } return nil } @@ -75,36 +126,48 @@ func (d *S3) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*mo input.ResponseContentDisposition = &disposition } req, _ := d.linkClient.GetObjectRequest(input) - var link string + var link model.Link var err error if d.CustomHost != "" { - err = req.Build() - link = req.HTTPRequest.URL.String() + if d.EnableCustomHostPresign { + link.URL, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + } else { + err = req.Build() + link.URL = req.HTTPRequest.URL.String() + } if d.RemoveBucket { - link = strings.Replace(link, "/"+d.Bucket, "", 1) + link.URL = strings.Replace(link.URL, "/"+d.Bucket, "", 1) } } else { - link, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + if common.ShouldProxy(d, filename) { + err = req.Sign() + link.URL = req.HTTPRequest.URL.String() + link.Header = req.HTTPRequest.Header + } else { + link.URL, err = req.Presign(time.Hour * time.Duration(d.SignURLExpire)) + } } if err != nil { return nil, err } - return &model.Link{ - URL: link, - }, nil + return &link, nil } func (d *S3) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - return d.Put(ctx, &model.Object{ - Path: stdpath.Join(parentDir.GetPath(), dirName), - }, &stream.FileStream{ - Obj: &model.Object{ - Name: getPlaceholderName(d.Placeholder), - Modified: time.Now(), - }, - Reader: io.NopCloser(bytes.NewReader([]byte{})), - Mimetype: "application/octet-stream", - }, func(int) {}) + dirPath := stdpath.Join(parentDir.GetPath(), dirName) + if d.UsePlaceholder { + return d.Put(ctx, &model.Object{ + Path: dirPath, + }, &stream.FileStream{ + Obj: &model.Object{ + Name: getPlaceholderName(d.Placeholder), + Modified: time.Now(), + }, + Reader: io.NopCloser(bytes.NewReader([]byte{})), + Mimetype: "application/octet-stream", + }, func(float64) {}) + } + return d.createDirMarker(ctx, dirPath) } func (d *S3) Move(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -134,22 +197,51 @@ func (d *S3) Remove(ctx context.Context, obj model.Obj) error { return d.removeFile(obj.GetPath()) } -func (d *S3) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *S3) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { uploader := s3manager.NewUploader(d.Session) - if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { - uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + if s.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = s.GetSize() / (s3manager.MaxUploadParts - 1) } - key := getKey(stdpath.Join(dstDir.GetPath(), stream.GetName()), false) - contentType := stream.GetMimetype() + key := getKey(stdpath.Join(dstDir.GetPath(), s.GetName()), false) + contentType := s.GetMimetype() log.Debugln("key:", key) + input := &s3manager.UploadInput{ + Bucket: &d.Bucket, + Key: &key, + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }), + ContentType: &contentType, + } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } + _, err := uploader.UploadWithContext(ctx, input) + return err +} + +func (d *S3) putEmptyObject(ctx context.Context, key string) error { + uploader := s3manager.NewUploader(d.Session) + contentType := "application/octet-stream" input := &s3manager.UploadInput{ Bucket: &d.Bucket, Key: &key, - Body: stream, + Body: driver.NewLimitedUploadStream(ctx, bytes.NewReader([]byte{})), ContentType: &contentType, } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } _, err := uploader.UploadWithContext(ctx, input) return err } -var _ driver.Driver = (*S3)(nil) +func (d *S3) createDirMarker(ctx context.Context, dirPath string) error { + return d.putEmptyObject(ctx, getKey(dirPath, true)) +} + +var ( + _ driver.Driver = (*S3)(nil) + _ driver.Other = (*S3)(nil) +) diff --git a/drivers/s3/meta.go b/drivers/s3/meta.go index 453f4db72e8..0c675e85646 100644 --- a/drivers/s3/meta.go +++ b/drivers/s3/meta.go @@ -14,23 +14,36 @@ type Addition struct { SecretAccessKey string `json:"secret_access_key" required:"true"` SessionToken string `json:"session_token"` CustomHost string `json:"custom_host"` + EnableCustomHostPresign bool `json:"enable_custom_host_presign"` SignURLExpire int `json:"sign_url_expire" type:"number" default:"4"` Placeholder string `json:"placeholder"` ForcePathStyle bool `json:"force_path_style"` ListObjectVersion string `json:"list_object_version" type:"select" options:"v1,v2" default:"v1"` + UsePlaceholder bool `json:"use_placeholder" default:"true" help:"Create hidden placeholder file (for example .alist) to keep empty folders."` RemoveBucket bool `json:"remove_bucket" help:"Remove bucket name from path when using custom host."` AddFilenameToDisposition bool `json:"add_filename_to_disposition" help:"Add filename to Content-Disposition header."` -} - -var config = driver.Config{ - Name: "S3", - DefaultRoot: "/", - LocalSort: true, - CheckStatus: true, + StorageClass string `json:"storage_class" type:"select" options:",standard,standard_ia,onezone_ia,intelligent_tiering,glacier,glacier_ir,deep_archive,archive" help:"Storage class for new objects. AWS and Tencent COS support different subsets (COS uses ARCHIVE/DEEP_ARCHIVE)."` } func init() { op.RegisterDriver(func() driver.Driver { - return &S3{} + return &S3{ + config: driver.Config{ + Name: "S3", + DefaultRoot: "/", + LocalSort: true, + CheckStatus: true, + }, + } + }) + op.RegisterDriver(func() driver.Driver { + return &S3{ + config: driver.Config{ + Name: "Doge", + DefaultRoot: "/", + LocalSort: true, + CheckStatus: true, + }, + } }) } diff --git a/drivers/s3/other.go b/drivers/s3/other.go new file mode 100644 index 00000000000..e299dae05d1 --- /dev/null +++ b/drivers/s3/other.go @@ -0,0 +1,286 @@ +package s3 + +import ( + "context" + "encoding/json" + "fmt" + "net/url" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" +) + +const ( + OtherMethodArchive = "archive" + OtherMethodArchiveStatus = "archive_status" + OtherMethodThaw = "thaw" + OtherMethodThawStatus = "thaw_status" +) + +type ArchiveRequest struct { + StorageClass string `json:"storage_class"` +} + +type ThawRequest struct { + Days int64 `json:"days"` + Tier string `json:"tier"` +} + +type ObjectDescriptor struct { + Path string `json:"path"` + Bucket string `json:"bucket"` + Key string `json:"key"` +} + +type ArchiveResponse struct { + Action string `json:"action"` + Object ObjectDescriptor `json:"object"` + StorageClass string `json:"storage_class"` + RequestID string `json:"request_id,omitempty"` + VersionID string `json:"version_id,omitempty"` + ETag string `json:"etag,omitempty"` + LastModified string `json:"last_modified,omitempty"` +} + +type ThawResponse struct { + Action string `json:"action"` + Object ObjectDescriptor `json:"object"` + RequestID string `json:"request_id,omitempty"` + Status *RestoreStatus `json:"status,omitempty"` +} + +type RestoreStatus struct { + Ongoing bool `json:"ongoing"` + Expiry string `json:"expiry,omitempty"` + Raw string `json:"raw"` +} + +func (d *S3) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + if args.Obj == nil { + return nil, fmt.Errorf("missing object reference") + } + if args.Obj.IsDir() { + return nil, errs.NotSupport + } + + switch strings.ToLower(strings.TrimSpace(args.Method)) { + case "archive": + return d.archive(ctx, args) + case "archive_status": + return d.archiveStatus(ctx, args) + case "thaw": + return d.thaw(ctx, args) + case "thaw_status": + return d.thawStatus(ctx, args) + default: + return nil, errs.NotSupport + } +} + +func (d *S3) archive(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + payload := ArchiveRequest{} + if err := DecodeOtherArgs(args.Data, &payload); err != nil { + return nil, fmt.Errorf("parse archive request: %w", err) + } + if payload.StorageClass == "" { + return nil, fmt.Errorf("storage_class is required") + } + storageClass := NormalizeStorageClass(payload.StorageClass) + input := &s3.CopyObjectInput{ + Bucket: &d.Bucket, + Key: &key, + CopySource: aws.String(url.PathEscape(d.Bucket + "/" + key)), + MetadataDirective: aws.String(s3.MetadataDirectiveCopy), + StorageClass: aws.String(storageClass), + } + copyReq, output := d.client.CopyObjectRequest(input) + copyReq.SetContext(ctx) + if err := copyReq.Send(); err != nil { + return nil, err + } + + resp := ArchiveResponse{ + Action: "archive", + Object: d.describeObject(args.Obj, key), + StorageClass: storageClass, + RequestID: copyReq.RequestID, + } + if output.VersionId != nil { + resp.VersionID = aws.StringValue(output.VersionId) + } + if result := output.CopyObjectResult; result != nil { + resp.ETag = aws.StringValue(result.ETag) + if result.LastModified != nil { + resp.LastModified = result.LastModified.UTC().Format(time.RFC3339) + } + } + if status, err := d.describeObjectStatus(ctx, key); err == nil { + if status.StorageClass != "" { + resp.StorageClass = status.StorageClass + } + } + return resp, nil +} + +func (d *S3) archiveStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + status, err := d.describeObjectStatus(ctx, key) + if err != nil { + return nil, err + } + return ArchiveResponse{ + Action: "archive_status", + Object: d.describeObject(args.Obj, key), + StorageClass: status.StorageClass, + }, nil +} + +func (d *S3) thaw(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + payload := ThawRequest{Days: 1} + if err := DecodeOtherArgs(args.Data, &payload); err != nil { + return nil, fmt.Errorf("parse thaw request: %w", err) + } + if payload.Days <= 0 { + payload.Days = 1 + } + restoreRequest := &s3.RestoreRequest{ + Days: aws.Int64(payload.Days), + } + if tier := NormalizeRestoreTier(payload.Tier); tier != "" { + restoreRequest.GlacierJobParameters = &s3.GlacierJobParameters{Tier: aws.String(tier)} + } + input := &s3.RestoreObjectInput{ + Bucket: &d.Bucket, + Key: &key, + RestoreRequest: restoreRequest, + } + restoreReq, _ := d.client.RestoreObjectRequest(input) + restoreReq.SetContext(ctx) + if err := restoreReq.Send(); err != nil { + return nil, err + } + status, _ := d.describeObjectStatus(ctx, key) + resp := ThawResponse{ + Action: "thaw", + Object: d.describeObject(args.Obj, key), + RequestID: restoreReq.RequestID, + } + if status != nil { + resp.Status = status.Restore + } + return resp, nil +} + +func (d *S3) thawStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + key := getKey(args.Obj.GetPath(), false) + status, err := d.describeObjectStatus(ctx, key) + if err != nil { + return nil, err + } + return ThawResponse{ + Action: "thaw_status", + Object: d.describeObject(args.Obj, key), + Status: status.Restore, + }, nil +} + +func (d *S3) describeObject(obj model.Obj, key string) ObjectDescriptor { + return ObjectDescriptor{ + Path: obj.GetPath(), + Bucket: d.Bucket, + Key: key, + } +} + +type objectStatus struct { + StorageClass string + Restore *RestoreStatus +} + +func (d *S3) describeObjectStatus(ctx context.Context, key string) (*objectStatus, error) { + head, err := d.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{Bucket: &d.Bucket, Key: &key}) + if err != nil { + return nil, err + } + status := &objectStatus{ + StorageClass: aws.StringValue(head.StorageClass), + Restore: parseRestoreHeader(head.Restore), + } + return status, nil +} + +func parseRestoreHeader(header *string) *RestoreStatus { + if header == nil { + return nil + } + value := strings.TrimSpace(*header) + if value == "" { + return nil + } + status := &RestoreStatus{Raw: value} + parts := strings.Split(value, ",") + for _, part := range parts { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if strings.HasPrefix(part, "ongoing-request=") { + status.Ongoing = strings.Contains(part, "\"true\"") + } + if strings.HasPrefix(part, "expiry-date=") { + expiry := strings.Trim(part[len("expiry-date="):], "\"") + if expiry != "" { + if t, err := time.Parse(time.RFC1123, expiry); err == nil { + status.Expiry = t.UTC().Format(time.RFC3339) + } else { + status.Expiry = expiry + } + } + } + } + return status +} + +func DecodeOtherArgs(data interface{}, target interface{}) error { + if data == nil { + return nil + } + raw, err := json.Marshal(data) + if err != nil { + return err + } + return json.Unmarshal(raw, target) +} + +func NormalizeStorageClass(value string) string { + normalized := strings.ToLower(strings.TrimSpace(strings.ReplaceAll(value, "-", "_"))) + if normalized == "" { + return value + } + if v, ok := storageClassLookup[normalized]; ok { + return v + } + return value +} + +func NormalizeRestoreTier(value string) string { + normalized := strings.ToLower(strings.TrimSpace(value)) + switch normalized { + case "", "default": + return "" + case "bulk": + return s3.TierBulk + case "standard": + return s3.TierStandard + case "expedited": + return s3.TierExpedited + default: + return value + } +} diff --git a/drivers/s3/util.go b/drivers/s3/util.go index 5578176a7a3..863b88bcab8 100644 --- a/drivers/s3/util.go +++ b/drivers/s3/util.go @@ -4,9 +4,11 @@ import ( "context" "errors" "net/http" + "net/url" "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -21,13 +23,21 @@ import ( // do others that not defined in Driver interface func (d *S3) initSession() error { + var err error + accessKeyID, secretAccessKey, sessionToken := d.AccessKeyID, d.SecretAccessKey, d.SessionToken + if d.config.Name == "Doge" { + credentialsTmp, err := getCredentials(d.AccessKeyID, d.SecretAccessKey) + if err != nil { + return err + } + accessKeyID, secretAccessKey, sessionToken = credentialsTmp.AccessKeyId, credentialsTmp.SecretAccessKey, credentialsTmp.SessionToken + } cfg := &aws.Config{ - Credentials: credentials.NewStaticCredentials(d.AccessKeyID, d.SecretAccessKey, d.SessionToken), + Credentials: credentials.NewStaticCredentials(accessKeyID, secretAccessKey, sessionToken), Region: &d.Region, Endpoint: &d.Endpoint, S3ForcePathStyle: aws.Bool(d.ForcePathStyle), } - var err error d.Session, err = session.NewSession(cfg) return err } @@ -96,17 +106,20 @@ func (d *S3) listV1(prefix string, args model.ListArgs) ([]model.Obj, error) { files = append(files, &file) } for _, object := range listObjectsResult.Contents { + if strings.HasSuffix(*object.Key, "/") { + continue + } name := path.Base(*object.Key) if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } - file := model.Object{ + file := &model.Object{ //Id: *object.Key, Name: name, Size: *object.Size, Modified: *object.LastModified, } - files = append(files, &file) + files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass))) } if listObjectsResult.IsTruncated == nil { return nil, errors.New("IsTruncated nil") @@ -155,13 +168,13 @@ func (d *S3) listV2(prefix string, args model.ListArgs) ([]model.Obj, error) { if !args.S3ShowPlaceholder && (name == getPlaceholderName(d.Placeholder) || name == d.Placeholder) { continue } - file := model.Object{ + file := &model.Object{ //Id: *object.Key, Name: name, Size: *object.Size, Modified: *object.LastModified, } - files = append(files, &file) + files = append(files, model.WrapObjStorageClass(file, aws.StringValue(object.StorageClass))) } if !aws.BoolValue(listObjectsResult.IsTruncated) { break @@ -190,18 +203,24 @@ func (d *S3) copyFile(ctx context.Context, src string, dst string) error { dstKey := getKey(dst, false) input := &s3.CopyObjectInput{ Bucket: &d.Bucket, - CopySource: aws.String("/" + d.Bucket + "/" + srcKey), + CopySource: aws.String(url.PathEscape(d.Bucket + "/" + srcKey)), Key: &dstKey, } + if storageClass := d.resolveStorageClass(); storageClass != nil { + input.StorageClass = storageClass + } _, err := d.client.CopyObject(input) return err } func (d *S3) copyDir(ctx context.Context, src string, dst string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true}) + objs, err := op.List(ctx, d, src, model.ListArgs{S3ShowPlaceholder: true, Refresh: true}) if err != nil { return err } + if len(objs) == 0 && !d.UsePlaceholder { + return d.createDirMarker(ctx, dst) + } for _, obj := range objs { cSrc := path.Join(src, obj.GetName()) cDst := path.Join(dst, obj.GetName()) @@ -218,8 +237,13 @@ func (d *S3) copyDir(ctx context.Context, src string, dst string) error { } func (d *S3) removeDir(ctx context.Context, src string) error { - objs, err := op.List(ctx, d, src, model.ListArgs{}) + d.cleanupDirArtifacts(src) + + objs, err := op.List(ctx, d, src, model.ListArgs{Refresh: true}) if err != nil { + if errs.IsObjectNotFound(err) { + return nil + } return err } for _, obj := range objs { @@ -233,9 +257,14 @@ func (d *S3) removeDir(ctx context.Context, src string) error { return err } } + d.cleanupDirArtifacts(src) + return nil +} + +func (d *S3) cleanupDirArtifacts(src string) { _ = d.removeFile(path.Join(src, getPlaceholderName(d.Placeholder))) _ = d.removeFile(path.Join(src, d.Placeholder)) - return nil + _ = d.removeFile(getKey(src, true)) } func (d *S3) removeFile(src string) error { diff --git a/drivers/seafile/driver.go b/drivers/seafile/driver.go index 49cf3386058..239f57dd949 100644 --- a/drivers/seafile/driver.go +++ b/drivers/seafile/driver.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "net/http" - "path/filepath" "strings" "time" @@ -19,6 +18,7 @@ type Seafile struct { Addition authorization string + libraryMap map[string]*LibraryInfo } func (d *Seafile) Config() driver.Config { @@ -31,6 +31,8 @@ func (d *Seafile) GetAddition() driver.Additional { func (d *Seafile) Init(ctx context.Context) error { d.Address = strings.TrimSuffix(d.Address, "/") + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + d.libraryMap = make(map[string]*LibraryInfo) return d.getToken() } @@ -38,10 +40,37 @@ func (d *Seafile) Drop(ctx context.Context) error { return nil } -func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { +func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) (result []model.Obj, err error) { path := dir.GetPath() + if path == d.RootFolderPath { + libraries, err := d.listLibraries() + if err != nil { + return nil, err + } + if path == "/" && d.RepoId == "" { + return utils.SliceConvert(libraries, func(f LibraryItemResp) (model.Obj, error) { + return &model.Object{ + Name: f.Name, + Modified: time.Unix(f.Modified, 0), + Size: f.Size, + IsFolder: true, + }, nil + }) + } + } + var repo *LibraryInfo + repo, path, err = d.getRepoAndPath(path) + if err != nil { + return nil, err + } + if repo.Encrypted { + err = d.decryptLibrary(repo) + if err != nil { + return nil, err + } + } var resp []RepoDirItemResp - _, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", d.Addition.RepoId), func(req *resty.Request) { + _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/dir/", repo.Id), func(req *resty.Request) { req.SetResult(&resp).SetQueryParams(map[string]string{ "p": path, }) @@ -63,9 +92,13 @@ func (d *Seafile) List(ctx context.Context, dir model.Obj, args model.ListArgs) } func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { - res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(file.GetPath()) + if err != nil { + return nil, err + } + res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": file.GetPath(), + "p": path, "reuse": "1", }) }) @@ -78,9 +111,14 @@ func (d *Seafile) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(parentDir.GetPath()) + if err != nil { + return err + } + path, _ = utils.JoinBasePath(path, dirName) + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/dir/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": filepath.Join(parentDir.GetPath(), dirName), + "p": path, }).SetFormData(map[string]string{ "operation": "mkdir", }) @@ -89,22 +127,34 @@ func (d *Seafile) MakeDir(ctx context.Context, parentDir model.Obj, dirName stri } func (d *Seafile) Move(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(srcObj.GetPath()) + if err != nil { + return err + } + dstRepo, dstPath, err := d.getRepoAndPath(dstDir.GetPath()) + if err != nil { + return err + } + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": srcObj.GetPath(), + "p": path, }).SetFormData(map[string]string{ "operation": "move", - "dst_repo": d.Addition.RepoId, - "dst_dir": dstDir.GetPath(), + "dst_repo": dstRepo.Id, + "dst_dir": dstPath, }) }, true) return err } func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) error { - _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(srcObj.GetPath()) + if err != nil { + return err + } + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": srcObj.GetPath(), + "p": path, }).SetFormData(map[string]string{ "operation": "rename", "newname": newName, @@ -114,31 +164,47 @@ func (d *Seafile) Rename(ctx context.Context, srcObj model.Obj, newName string) } func (d *Seafile) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { - _, err := d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(srcObj.GetPath()) + if err != nil { + return err + } + dstRepo, dstPath, err := d.getRepoAndPath(dstDir.GetPath()) + if err != nil { + return err + } + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": srcObj.GetPath(), + "p": path, }).SetFormData(map[string]string{ "operation": "copy", - "dst_repo": d.Addition.RepoId, - "dst_dir": dstDir.GetPath(), + "dst_repo": dstRepo.Id, + "dst_dir": dstPath, }) }) return err } func (d *Seafile) Remove(ctx context.Context, obj model.Obj) error { - _, err := d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", d.Addition.RepoId), func(req *resty.Request) { + repo, path, err := d.getRepoAndPath(obj.GetPath()) + if err != nil { + return err + } + _, err = d.request(http.MethodDelete, fmt.Sprintf("/api2/repos/%s/file/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": obj.GetPath(), + "p": path, }) }) return err } -func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", d.Addition.RepoId), func(req *resty.Request) { +func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { + repo, path, err := d.getRepoAndPath(dstDir.GetPath()) + if err != nil { + return err + } + res, err := d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/upload-link/", repo.Id), func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "p": dstDir.GetPath(), + "p": path, }) }) if err != nil { @@ -148,11 +214,16 @@ func (d *Seafile) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt u := string(res) u = u[1 : len(u)-1] // remove quotes _, err = d.request(http.MethodPost, u, func(req *resty.Request) { - req.SetFileReader("file", stream.GetName(), stream). + r := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + req.SetFileReader("file", s.GetName(), r). SetFormData(map[string]string{ - "parent_dir": dstDir.GetPath(), + "parent_dir": path, "replace": "1", - }) + }). + SetContext(ctx) }) return err } diff --git a/drivers/seafile/meta.go b/drivers/seafile/meta.go index e333fd905bb..fd5255f592b 100644 --- a/drivers/seafile/meta.go +++ b/drivers/seafile/meta.go @@ -9,9 +9,11 @@ type Addition struct { driver.RootPath Address string `json:"address" required:"true"` - UserName string `json:"username" required:"true"` - Password string `json:"password" required:"true"` - RepoId string `json:"repoId" required:"true"` + UserName string `json:"username" required:"false"` + Password string `json:"password" required:"false"` + Token string `json:"token" required:"false"` + RepoId string `json:"repoId" required:"false"` + RepoPwd string `json:"repoPwd" required:"false"` } var config = driver.Config{ diff --git a/drivers/seafile/types.go b/drivers/seafile/types.go index 5c5b528d31f..47cb322df4a 100644 --- a/drivers/seafile/types.go +++ b/drivers/seafile/types.go @@ -1,14 +1,44 @@ package seafile +import "time" + type AuthTokenResp struct { Token string `json:"token"` } -type RepoDirItemResp struct { +type RepoItemResp struct { Id string `json:"id"` - Type string `json:"type"` // dir, file + Type string `json:"type"` // repo, dir, file Name string `json:"name"` Size int64 `json:"size"` Modified int64 `json:"mtime"` Permission string `json:"permission"` } + +type LibraryItemResp struct { + RepoItemResp + OwnerContactEmail string `json:"owner_contact_email"` + OwnerName string `json:"owner_name"` + Owner string `json:"owner"` + ModifierEmail string `json:"modifier_email"` + ModifierContactEmail string `json:"modifier_contact_email"` + ModifierName string `json:"modifier_name"` + Virtual bool `json:"virtual"` + MtimeRelative string `json:"mtime_relative"` + Encrypted bool `json:"encrypted"` + Version int `json:"version"` + HeadCommitId string `json:"head_commit_id"` + Root string `json:"root"` + Salt string `json:"salt"` + SizeFormatted string `json:"size_formatted"` +} + +type RepoDirItemResp struct { + RepoItemResp +} + +type LibraryInfo struct { + LibraryItemResp + decryptedTime time.Time + decryptedSuccess bool +} \ No newline at end of file diff --git a/drivers/seafile/util.go b/drivers/seafile/util.go index 23255545c3a..89b7b0fc39f 100644 --- a/drivers/seafile/util.go +++ b/drivers/seafile/util.go @@ -1,14 +1,23 @@ package seafile import ( + "errors" "fmt" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/pkg/utils" + "net/http" "strings" + "time" "github.com/alist-org/alist/v3/drivers/base" "github.com/go-resty/resty/v2" ) func (d *Seafile) getToken() error { + if d.Token != "" { + d.authorization = fmt.Sprintf("Token %s", d.Token) + return nil + } var authResp AuthTokenResp res, err := base.RestyClient.R(). SetResult(&authResp). @@ -60,3 +69,110 @@ func (d *Seafile) request(method string, pathname string, callback base.ReqCallb } return res.Body(), nil } + +func (d *Seafile) getRepoAndPath(fullPath string) (repo *LibraryInfo, path string, err error) { + libraryMap := d.libraryMap + repoId := d.Addition.RepoId + if repoId != "" { + if len(repoId) == 36 /* uuid */ { + for _, library := range libraryMap { + if library.Id == repoId { + return library, fullPath, nil + } + } + } + } else { + var repoName string + str := fullPath[1:] + pos := strings.IndexRune(str, '/') + if pos == -1 { + repoName = str + } else { + repoName = str[:pos] + } + path = utils.FixAndCleanPath(fullPath[1+len(repoName):]) + if library, ok := libraryMap[repoName]; ok { + return library, path, nil + } + } + return nil, "", errs.ObjectNotFound +} + +func (d *Seafile) listLibraries() (resp []LibraryItemResp, err error) { + repoId := d.Addition.RepoId + if repoId == "" { + _, err = d.request(http.MethodGet, "/api2/repos/", func(req *resty.Request) { + req.SetResult(&resp) + }) + } else { + var oneResp LibraryItemResp + _, err = d.request(http.MethodGet, fmt.Sprintf("/api2/repos/%s/", repoId), func(req *resty.Request) { + req.SetResult(&oneResp) + }) + if err == nil { + resp = append(resp, oneResp) + } + } + if err != nil { + return nil, err + } + libraryMap := make(map[string]*LibraryInfo) + var putLibraryMap func(library LibraryItemResp, index int) + putLibraryMap = func(library LibraryItemResp, index int) { + name := library.Name + if index > 0 { + name = fmt.Sprintf("%s (%d)", name, index) + } + if _, exist := libraryMap[name]; exist { + putLibraryMap(library, index+1) + } else { + libraryInfo := LibraryInfo{} + data, _ := utils.Json.Marshal(library) + _ = utils.Json.Unmarshal(data, &libraryInfo) + libraryMap[name] = &libraryInfo + } + } + for _, library := range resp { + putLibraryMap(library, 0) + } + d.libraryMap = libraryMap + return resp, nil +} + +var repoPwdNotConfigured = errors.New("library password not configured") +var repoPwdIncorrect = errors.New("library password is incorrect") + +func (d *Seafile) decryptLibrary(repo *LibraryInfo) (err error) { + if !repo.Encrypted { + return nil + } + if d.RepoPwd == "" { + return repoPwdNotConfigured + } + now := time.Now() + decryptedTime := repo.decryptedTime + if repo.decryptedSuccess { + if now.Sub(decryptedTime).Minutes() <= 30 { + return nil + } + } else { + if now.Sub(decryptedTime).Seconds() <= 10 { + return repoPwdIncorrect + } + } + var resp string + _, err = d.request(http.MethodPost, fmt.Sprintf("/api2/repos/%s/", repo.Id), func(req *resty.Request) { + req.SetResult(&resp).SetFormData(map[string]string{ + "password": d.RepoPwd, + }) + }) + repo.decryptedTime = time.Now() + if err != nil || !strings.Contains(resp, "success") { + repo.decryptedSuccess = false + return err + } + repo.decryptedSuccess = true + return nil +} + + diff --git a/drivers/sftp/driver.go b/drivers/sftp/driver.go index 77f5198457c..7498ce39f66 100644 --- a/drivers/sftp/driver.go +++ b/drivers/sftp/driver.go @@ -16,7 +16,8 @@ import ( type SFTP struct { model.Storage Addition - client *sftp.Client + client *sftp.Client + clientConnectionError error } func (d *SFTP) Config() driver.Config { @@ -39,6 +40,9 @@ func (d *SFTP) Drop(ctx context.Context) error { } func (d *SFTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + if err := d.clientReconnectOnConnectionError(); err != nil { + return nil, err + } log.Debugf("[sftp] list dir: %s", dir.GetPath()) files, err := d.client.ReadDir(dir.GetPath()) if err != nil { @@ -51,6 +55,9 @@ func (d *SFTP) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] } func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if err := d.clientReconnectOnConnectionError(); err != nil { + return nil, err + } remoteFile, err := d.client.Open(file.GetPath()) if err != nil { return nil, err @@ -62,14 +69,23 @@ func (d *SFTP) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* } func (d *SFTP) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } return d.client.MkdirAll(path.Join(parentDir.GetPath(), dirName)) } func (d *SFTP) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } return d.client.Rename(srcObj.GetPath(), path.Join(dstDir.GetPath(), srcObj.GetName())) } func (d *SFTP) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } return d.client.Rename(srcObj.GetPath(), path.Join(path.Dir(srcObj.GetPath()), newName)) } @@ -78,10 +94,16 @@ func (d *SFTP) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { } func (d *SFTP) Remove(ctx context.Context, obj model.Obj) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } return d.remove(obj.GetPath()) } func (d *SFTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if err := d.clientReconnectOnConnectionError(); err != nil { + return err + } dstFile, err := d.client.Create(path.Join(dstDir.GetPath(), stream.GetName())) if err != nil { return err @@ -89,7 +111,7 @@ func (d *SFTP) Put(ctx context.Context, dstDir model.Obj, stream model.FileStrea defer func() { _ = dstFile.Close() }() - err = utils.CopyWithCtx(ctx, dstFile, stream, stream.GetSize(), up) + err = utils.CopyWithCtx(ctx, dstFile, driver.NewLimitedUploadStream(ctx, stream), stream.GetSize(), up) return err } diff --git a/drivers/sftp/meta.go b/drivers/sftp/meta.go index d77398f333f..9b1665679cd 100644 --- a/drivers/sftp/meta.go +++ b/drivers/sftp/meta.go @@ -10,7 +10,9 @@ type Addition struct { Username string `json:"username" required:"true"` PrivateKey string `json:"private_key" type:"text"` Password string `json:"password"` + Passphrase string `json:"passphrase"` driver.RootPath + IgnoreSymlinkError bool `json:"ignore_symlink_error" default:"false" info:"Ignore symlink error"` } var config = driver.Config{ diff --git a/drivers/sftp/types.go b/drivers/sftp/types.go index 70a03b983ab..493e884c151 100644 --- a/drivers/sftp/types.go +++ b/drivers/sftp/types.go @@ -30,6 +30,14 @@ func (d *SFTP) fileToObj(f os.FileInfo, dir string) (model.Obj, error) { } _f, err := d.client.Stat(target) if err != nil { + if d.IgnoreSymlinkError { + return &model.Object{ + Name: f.Name(), + Size: f.Size(), + Modified: f.ModTime(), + IsFolder: f.IsDir(), + }, nil + } return nil, err } // set basic info diff --git a/drivers/sftp/util.go b/drivers/sftp/util.go index 3deb8dcf94b..53f9c379e04 100644 --- a/drivers/sftp/util.go +++ b/drivers/sftp/util.go @@ -4,6 +4,7 @@ import ( "path" "github.com/pkg/sftp" + log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" ) @@ -11,8 +12,14 @@ import ( func (d *SFTP) initClient() error { var auth ssh.AuthMethod - if d.PrivateKey != "" { - signer, err := ssh.ParsePrivateKey([]byte(d.PrivateKey)) + if len(d.PrivateKey) > 0 { + var err error + var signer ssh.Signer + if len(d.Passphrase) > 0 { + signer, err = ssh.ParsePrivateKeyWithPassphrase([]byte(d.PrivateKey), []byte(d.Passphrase)) + } else { + signer, err = ssh.ParsePrivateKey([]byte(d.PrivateKey)) + } if err != nil { return err } @@ -30,6 +37,23 @@ func (d *SFTP) initClient() error { return err } d.client, err = sftp.NewClient(conn) + if err == nil { + d.clientConnectionError = nil + go func(d *SFTP) { + d.clientConnectionError = d.client.Wait() + }(d) + } + return err +} + +func (d *SFTP) clientReconnectOnConnectionError() error { + err := d.clientConnectionError + if err == nil { + return nil + } + log.Debugf("[sftp] discarding closed sftp connection: %v", err) + _ = d.client.Close() + err = d.initClient() return err } diff --git a/drivers/smb/driver.go b/drivers/smb/driver.go index 9632f24e0eb..c292e92e03a 100644 --- a/drivers/smb/driver.go +++ b/drivers/smb/driver.go @@ -186,7 +186,7 @@ func (d *SMB) Put(ctx context.Context, dstDir model.Obj, stream model.FileStream _ = d.fs.Remove(fullPath) } }() - err = utils.CopyWithCtx(ctx, out, stream, stream.GetSize(), up) + err = utils.CopyWithCtx(ctx, out, driver.NewLimitedUploadStream(ctx, stream), stream.GetSize(), up) if err != nil { return err } diff --git a/drivers/smb/util.go b/drivers/smb/util.go index f4605536da7..d9fbf6c5a5a 100644 --- a/drivers/smb/util.go +++ b/drivers/smb/util.go @@ -1,7 +1,7 @@ package smb import ( - "io" + "github.com/alist-org/alist/v3/pkg/utils" "io/fs" "net" "os" @@ -74,7 +74,7 @@ func (d *SMB) CopyFile(src, dst string) error { } defer dstfd.Close() - if _, err = io.Copy(dstfd, srcfd); err != nil { + if _, err = utils.CopyWithBuffer(dstfd, srcfd); err != nil { return err } if srcinfo, err = d.fs.Stat(src); err != nil { diff --git a/drivers/streamtape/driver.go b/drivers/streamtape/driver.go new file mode 100644 index 00000000000..9ce32e25b69 --- /dev/null +++ b/drivers/streamtape/driver.go @@ -0,0 +1,543 @@ +package streamtape + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + log "github.com/sirupsen/logrus" +) + +type Streamtape struct { + model.Storage + Addition +} + +var waitMoreSecondsRe = regexp.MustCompile(`wait\s+(\d+)\s+more\s+seconds?`) + +func (d *Streamtape) Config() driver.Config { + return config +} + +func (d *Streamtape) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Streamtape) Init(ctx context.Context) error { + if strings.TrimSpace(d.APILogin) == "" || strings.TrimSpace(d.APIKey) == "" { + return errors.New("api_login and api_key are required") + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + + var account accountInfo + if err := d.callAPI(ctx, "/account/info", nil, &account); err != nil { + return err + } + + op.MustSaveDriverStorage(d) + return nil +} + +func (d *Streamtape) Drop(ctx context.Context) error { + return nil +} + +func (d *Streamtape) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + folderID := d.RootFolderID + if dir.GetID() != "" { + folderID = folderIDFromObjID(dir.GetID()) + } + + params := map[string]string{} + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + + var result listFolderResult + if err := d.callAPI(ctx, "/file/listfolder", params, &result); err != nil { + return nil, err + } + + objects := make([]model.Obj, 0, len(result.Folders)+len(result.Files)) + for _, f := range result.Folders { + objects = append(objects, &model.Object{ + ID: encodeFolderID(f.ID), + Name: f.Name, + IsFolder: true, + }) + } + for _, f := range result.Files { + objects = append(objects, buildFileObj(f)) + } + return objects, nil +} + +func (d *Streamtape) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := fileIDFromObjID(file.GetID()) + if fileID == "" { + return nil, errors.New("empty file id") + } + + var ticket dlTicketResult + if err := d.callAPI(ctx, "/file/dlticket", map[string]string{"file": fileID}, &ticket); err != nil { + return nil, err + } + + var dl dlResult + waitSeconds := ticket.WaitTime + if waitSeconds > 0 { + timer := time.NewTimer(time.Duration(waitSeconds+1) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + + var err error + for i := 0; i < 3; i++ { + err = d.callAPI(ctx, "/file/dl", map[string]string{ + "file": fileID, + "ticket": ticket.Ticket, + }, &dl) + if err == nil { + break + } + waitSeconds = extractWaitSecondsFromErr(err) + if waitSeconds <= 0 { + return nil, err + } + timer := time.NewTimer(time.Duration(waitSeconds+1) * time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return nil, ctx.Err() + case <-timer.C: + } + } + if err != nil { + return nil, err + } + + finalURL := ensureStreamQuery(dl.URL) + log.Infof("streamtape direct link file=%s url=%s", fileID, finalURL) + link := &model.Link{ + URL: finalURL, + Header: http.Header{ + "Referer": []string{"https://streamtape.com/"}, + "Origin": []string{"https://streamtape.com"}, + }, + } + d.applyRangeStrategy(link, file.GetSize()) + return link, nil +} + +func extractWaitSecondsFromErr(err error) int { + if err == nil { + return 0 + } + matches := waitMoreSecondsRe.FindStringSubmatch(strings.ToLower(err.Error())) + if len(matches) < 2 { + return 0 + } + seconds, convErr := strconv.Atoi(matches[1]) + if convErr != nil || seconds < 0 { + return 0 + } + return seconds +} + +func ensureStreamQuery(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + q := u.Query() + if q.Get("stream") == "" { + q.Set("stream", "1") + u.RawQuery = q.Encode() + } + return u.String() +} + +func (d *Streamtape) applyRangeStrategy(link *model.Link, size int64) { + if !d.EnableRangeControl || size <= 0 { + return + } + + mode := strings.ToLower(strings.TrimSpace(d.RangeMode)) + if mode == "" { + mode = "chunk" + } + + switch mode { + case "full": + // Keep single full-tail behavior while still using ranged requests. + link.Concurrency = 1 + link.PartSize = int(size) + case "percent": + percent := d.RangePercent + if percent <= 0 { + percent = 15 + } + if percent > 100 { + percent = 100 + } + partSize := size * int64(percent) / 100 + if partSize < 1*1024*1024 { + partSize = 1 * 1024 * 1024 + } + if partSize > size { + partSize = size + } + link.Concurrency = 1 + link.PartSize = int(partSize) + default: + chunkMB := d.RangeChunkMB + if chunkMB <= 0 { + chunkMB = 8 + } + partSize := int64(chunkMB) * 1024 * 1024 + if partSize > size { + partSize = size + } + concurrency := d.RangeConcurrency + if concurrency <= 0 { + concurrency = 4 + } + link.Concurrency = concurrency + link.PartSize = int(partSize) + } +} + +func (d *Streamtape) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + pid := d.RootFolderID + if parentDir.GetID() != "" { + pid = folderIDFromObjID(parentDir.GetID()) + } + + params := map[string]string{"name": dirName} + if pid != "" && pid != "0" { + params["pid"] = pid + } + + var result createFolderResult + if err := d.callAPI(ctx, "/file/createfolder", params, &result); err != nil { + return nil, err + } + + return &model.Object{ + ID: encodeFolderID(result.FolderID), + Name: dirName, + IsFolder: true, + }, nil +} + +func (d *Streamtape) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if srcObj.IsDir() { + return nil, errs.NotImplement + } + fileID := fileIDFromObjID(srcObj.GetID()) + if fileID == "" { + return nil, errors.New("empty file id") + } + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + if folderID == "" || folderID == "0" { + return nil, fmt.Errorf("streamtape move to root is not supported by API") + } + + if err := d.callAPI(ctx, "/file/move", map[string]string{ + "file": fileID, + "folder": folderID, + }, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: srcObj.GetName(), + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: false, + }, nil +} + +func (d *Streamtape) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + endpoint := "/file/rename" + params := map[string]string{"name": newName} + if srcObj.IsDir() { + endpoint = "/file/renamefolder" + params["folder"] = folderIDFromObjID(srcObj.GetID()) + } else { + params["file"] = fileIDFromObjID(srcObj.GetID()) + } + + if err := d.callAPI(ctx, endpoint, params, nil); err != nil { + return nil, err + } + + return &model.Object{ + ID: srcObj.GetID(), + Name: newName, + Size: srcObj.GetSize(), + Modified: srcObj.ModTime(), + IsFolder: srcObj.IsDir(), + }, nil +} + +func (d *Streamtape) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Remove(ctx context.Context, obj model.Obj) error { + endpoint := "/file/delete" + params := map[string]string{} + if obj.IsDir() { + endpoint = "/file/deletefolder" + params["folder"] = folderIDFromObjID(obj.GetID()) + } else { + params["file"] = fileIDFromObjID(obj.GetID()) + } + return d.callAPI(ctx, endpoint, params, nil) +} + +func (d *Streamtape) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + params := map[string]string{} + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + if d.Sha256 != "" { + params["sha256"] = d.Sha256 + } + + var uploadURL uploadURLResult + if err := d.callAPI(ctx, "/file/ul", params, &uploadURL); err != nil { + return nil, err + } + + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) + + res, err := base.RestyClient.R(). + SetContext(ctx). + SetFileReader("file1", file.GetName(), reader). + Post(uploadURL.URL) + if err != nil { + return nil, err + } + if res.StatusCode() >= http.StatusBadRequest { + return nil, fmt.Errorf("streamtape upload failed: http %d", res.StatusCode()) + } + + uploadedID := extractFileIDFromUploadBody(res.Body()) + if uploadedID == "" { + list, listErr := d.List(ctx, &model.Object{ID: encodeFolderID(folderID), IsFolder: true}, model.ListArgs{}) + if listErr == nil { + for _, obj := range list { + if obj.IsDir() { + continue + } + if obj.GetName() == file.GetName() && (file.GetSize() <= 0 || obj.GetSize() == file.GetSize()) { + return obj, nil + } + } + } + return &model.Object{ + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil + } + + return &model.Object{ + ID: encodeFileID(uploadedID), + Name: file.GetName(), + Size: file.GetSize(), + IsFolder: false, + }, nil +} + +// PutURL initiates a remote upload from an external URL +func (d *Streamtape) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) { + folderID := d.RootFolderID + if dstDir.GetID() != "" { + folderID = folderIDFromObjID(dstDir.GetID()) + } + + params := map[string]string{ + "url": url, + } + if folderID != "" && folderID != "0" { + params["folder"] = folderID + } + if name != "" { + params["name"] = name + } + + var result remoteDlAddResult + if err := d.callAPI(ctx, "/remotedl/add", params, &result); err != nil { + return nil, err + } + + return &model.Object{ + ID: encodeRemoteUploadID(result.ID), + Name: name, + IsFolder: false, + }, nil +} + +func (d *Streamtape) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + return nil, errs.NotImplement +} + +func (d *Streamtape) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + switch strings.ToLower(args.Method) { + case "remotedl_status": + return d.remoteDlStatus(ctx, args) + case "remotedl_remove": + return d.remoteDlRemove(ctx, args) + case "file_info": + return d.fileInfo(ctx, args) + case "thumbnail": + return d.thumbnail(ctx, args) + case "conversion_status": + return d.conversionStatus(ctx, args) + default: + return nil, errs.NotSupport + } +} + +func (d *Streamtape) extractRemoteUploadID(args model.OtherArgs) (string, error) { + uploadID := remoteUploadIDFromObjID(args.Obj.GetID()) + if uploadID == "" { + if data, ok := args.Data.(map[string]interface{}); ok { + if id, ok := data["id"].(string); ok { + uploadID = id + } + } + } + if uploadID == "" { + return "", fmt.Errorf("remote upload ID required") + } + return uploadID, nil +} + +func (d *Streamtape) remoteDlStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID, err := d.extractRemoteUploadID(args) + if err != nil { + return nil, err + } + + var result remoteDlStatusResult + if err := d.callAPI(ctx, "/remotedl/status", map[string]string{"id": uploadID}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) remoteDlRemove(ctx context.Context, args model.OtherArgs) (interface{}, error) { + uploadID, err := d.extractRemoteUploadID(args) + if err != nil { + return nil, err + } + + if err := d.callAPI(ctx, "/remotedl/remove", map[string]string{"id": uploadID}, nil); err != nil { + return nil, err + } + return true, nil +} + +func (d *Streamtape) fileInfo(ctx context.Context, args model.OtherArgs) (interface{}, error) { + var fileIDs string + if data, ok := args.Data.(map[string]interface{}); ok { + if ids, ok := data["file_ids"].(string); ok { + fileIDs = ids + } + } + if fileIDs == "" { + fileIDs = fileIDFromObjID(args.Obj.GetID()) + } + if fileIDs == "" { + return nil, fmt.Errorf("file IDs required") + } + + var result fileInfoResult + if err := d.callAPI(ctx, "/file/info", map[string]string{"file": fileIDs}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) thumbnail(ctx context.Context, args model.OtherArgs) (interface{}, error) { + fileID := fileIDFromObjID(args.Obj.GetID()) + if fileID == "" { + return nil, fmt.Errorf("file ID required") + } + + var result string + if err := d.callAPI(ctx, "/file/getsplash", map[string]string{"file": fileID}, &result); err != nil { + return nil, err + } + return result, nil +} + +func (d *Streamtape) conversionStatus(ctx context.Context, args model.OtherArgs) (interface{}, error) { + isFailed := false + if data, ok := args.Data.(map[string]interface{}); ok { + if t, ok := data["type"].(string); ok && t == "failed" { + isFailed = true + } + } + + endpoint := "/file/runningconverts" + if isFailed { + endpoint = "/file/failedconverts" + } + + var result conversionResult + if err := d.callAPI(ctx, endpoint, nil, &result); err != nil { + return nil, err + } + return result, nil +} + +var _ driver.Driver = (*Streamtape)(nil) diff --git a/drivers/streamtape/meta.go b/drivers/streamtape/meta.go new file mode 100644 index 00000000000..55e97894dbe --- /dev/null +++ b/drivers/streamtape/meta.go @@ -0,0 +1,39 @@ +package streamtape + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + APILogin string `json:"api_login" required:"true" help:"API Login from Streamtape account settings"` + APIKey string `json:"api_key" required:"true" help:"API Key from Streamtape account settings"` + RangeMode string `json:"range_mode" type:"select" options:"chunk,full,percent" default:"chunk" help:"Range strategy for preview: chunk=bounded ranges, full=single full-tail range, percent=part size by file percentage"` + RangeChunkMB int `json:"range_chunk_mb" type:"number" default:"8" help:"Chunk mode part size in MB"` + RangeConcurrency int `json:"range_concurrency" type:"number" default:"4" help:"Chunk mode concurrent upstream requests"` + RangePercent int `json:"range_percent" type:"number" default:"15" help:"Percent mode part size percentage (1-100)"` + EnableRangeControl bool `json:"enable_range_control" default:"true" help:"Enable driver-level range shaping for smoother streaming"` + Sha256 string `json:"sha256" help:"Expected SHA256 hash for upload verification (optional)"` +} + +var config = driver.Config{ + Name: "Streamtape", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: true, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "warning|Moving files to root folder is not supported by Streamtape API", + NoOverwriteUpload: false, + ProxyRangeOption: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Streamtape{} + }) +} diff --git a/drivers/streamtape/types.go b/drivers/streamtape/types.go new file mode 100644 index 00000000000..347e89d8a67 --- /dev/null +++ b/drivers/streamtape/types.go @@ -0,0 +1,97 @@ +package streamtape + +import "encoding/json" + +type apiResponse struct { + Status int `json:"status"` + Msg string `json:"msg"` + Result json.RawMessage `json:"result"` +} + +type accountInfo struct { + APIID string `json:"apiid"` + Email string `json:"email"` + SignupAt string `json:"signup_at"` +} + +type listFolderResult struct { + Folders []folderItem `json:"folders"` + Files []fileItem `json:"files"` +} + +type folderItem struct { + ID string `json:"id"` + Name string `json:"name"` +} + +type fileItem struct { + Name string `json:"name"` + Size int64 `json:"size"` + Link string `json:"link"` + CreatedAt int64 `json:"created_at"` + Downloads int64 `json:"downloads"` + LinkID string `json:"linkid"` + Convert string `json:"convert"` +} + +type dlTicketResult struct { + Ticket string `json:"ticket"` + WaitTime int `json:"wait_time"` +} + +type dlResult struct { + Name string `json:"name"` + Size int64 `json:"size"` + URL string `json:"url"` +} + +type createFolderResult struct { + FolderID string `json:"folderid"` +} + +type uploadURLResult struct { + URL string `json:"url"` +} + +type remoteDlAddResult struct { + ID string `json:"id"` + FolderID string `json:"folderid"` +} + +type remoteDlStatusResult map[string]remoteDlStatusItem + +type remoteDlStatusItem struct { + ID string `json:"id"` + RemoteURL string `json:"remoteurl"` + Status string `json:"status"` + BytesLoaded interface{} `json:"bytes_loaded"` + BytesTotal interface{} `json:"bytes_total"` + FolderID string `json:"folderid"` + Added string `json:"added"` + LastUpdate string `json:"last_update"` + ExtID bool `json:"extid"` + URL bool `json:"url"` +} + +type fileInfoResult map[string]fileInfoItem + +type fileInfoItem struct { + ID string `json:"id"` + Name string `json:"name"` + Size int64 `json:"size"` + Type string `json:"type"` + Converted bool `json:"converted"` + Status int `json:"status"` +} + +type conversionResult []conversionItem + +type conversionItem struct { + Name string `json:"name"` + FolderID string `json:"folderid"` + Status string `json:"status"` + Progress int `json:"progress"` + Retries int `json:"retries"` + Link string `json:"link"` + LinkID string `json:"linkid"` +} diff --git a/drivers/streamtape/util.go b/drivers/streamtape/util.go new file mode 100644 index 00000000000..51ad73d59bb --- /dev/null +++ b/drivers/streamtape/util.go @@ -0,0 +1,163 @@ +package streamtape + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" +) + +const apiBase = "https://api.streamtape.com" + +func (d *Streamtape) callAPI(ctx context.Context, endpoint string, params map[string]string, out any) error { + query := map[string]string{ + "login": d.APILogin, + "key": d.APIKey, + } + for k, v := range params { + if strings.TrimSpace(v) == "" { + continue + } + query[k] = v + } + + var resp apiResponse + r, err := base.RestyClient.R(). + SetContext(ctx). + SetQueryParams(query). + SetResult(&resp). + Get(apiBase + endpoint) + if err != nil { + return err + } + if r.StatusCode() != http.StatusOK { + return fmt.Errorf("streamtape http error: %d", r.StatusCode()) + } + if resp.Status != 200 { + return fmt.Errorf("streamtape api error: status=%d msg=%s", resp.Status, resp.Msg) + } + if out == nil || len(resp.Result) == 0 || string(resp.Result) == "null" { + return nil + } + if err := json.Unmarshal(resp.Result, out); err != nil { + return fmt.Errorf("decode streamtape result failed: %w", err) + } + return nil +} + +func folderIDFromObjID(id string) string { + if id == "" || id == "0" || id == "/" { + return "0" + } + if strings.HasPrefix(id, "d:") { + return strings.TrimPrefix(id, "d:") + } + return id +} + +func fileIDFromObjID(id string) string { + if strings.HasPrefix(id, "f:") { + return strings.TrimPrefix(id, "f:") + } + return id +} + +func encodeFolderID(id string) string { + if id == "" || id == "0" || id == "/" { + return "d:0" + } + return "d:" + id +} + +func encodeFileID(id string) string { + if strings.HasPrefix(id, "f:") { + return id + } + return "f:" + id +} + +func extractFileIDFromLink(link string) string { + if link == "" { + return "" + } + u, err := url.Parse(link) + if err != nil { + return "" + } + parts := strings.Split(strings.Trim(path.Clean(u.Path), "/"), "/") + for i := 0; i < len(parts)-1; i++ { + if parts[i] == "v" { + return parts[i+1] + } + } + return "" +} + +func buildFileObj(f fileItem) model.Obj { + id := f.LinkID + if id == "" { + id = extractFileIDFromLink(f.Link) + } + mod := time.Now() + if f.CreatedAt > 0 { + mod = time.Unix(f.CreatedAt, 0) + } + return &model.Object{ + ID: encodeFileID(id), + Name: f.Name, + Size: f.Size, + Modified: mod, + IsFolder: false, + } +} + +func extractFileIDFromUploadBody(body []byte) string { + if len(body) == 0 { + return "" + } + + var resp apiResponse + if err := json.Unmarshal(body, &resp); err != nil { + return "" + } + if resp.Status != 200 || len(resp.Result) == 0 { + return "" + } + + var result map[string]any + if err := json.Unmarshal(resp.Result, &result); err != nil { + return "" + } + for _, key := range []string{"file", "fileid", "id", "linkid"} { + if v, ok := result[key]; ok { + if s, ok := v.(string); ok && s != "" { + return s + } + } + } + return "" +} + +const remoteUploadPrefix = "ru:" + +func encodeRemoteUploadID(id string) string { + return remoteUploadPrefix + id +} + +func remoteUploadIDFromObjID(id string) string { + if strings.HasPrefix(id, remoteUploadPrefix) { + return strings.TrimPrefix(id, remoteUploadPrefix) + } + return "" +} + +func isRemoteUploadID(id string) bool { + return strings.HasPrefix(id, remoteUploadPrefix) +} diff --git a/drivers/strm/driver.go b/drivers/strm/driver.go new file mode 100644 index 00000000000..175a558dfcc --- /dev/null +++ b/drivers/strm/driver.go @@ -0,0 +1,283 @@ +package strm + +import ( + "context" + "errors" + stdpath "path" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + log "github.com/sirupsen/logrus" +) + +type Strm struct { + model.Storage + Addition + + aliases map[string][]string + autoFlatten bool + singleRootKey string + + mediaExtSet map[string]struct{} + downloadExtSet map[string]struct{} + normalizedMode string + normalizedPrefix string +} + +func (d *Strm) Config() driver.Config { + return config +} + +func (d *Strm) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Strm) Init(ctx context.Context) error { + if strings.TrimSpace(d.Paths) == "" { + return errors.New("paths is required") + } + if d.SaveStrmToLocal && strings.TrimSpace(d.SaveStrmLocalPath) == "" { + return errors.New("SaveStrmLocalPath is required") + } + + d.aliases = parseAliases(d.Paths) + if len(d.aliases) == 0 { + return errors.New("no valid path mapping found") + } + + d.autoFlatten = len(d.aliases) == 1 + d.singleRootKey = "" + if d.autoFlatten { + for k := range d.aliases { + d.singleRootKey = k + } + } + + d.mediaExtSet = parseExtSet(defaultIfEmpty(d.FilterFileTypes, defaultMediaExt)) + d.downloadExtSet = parseExtSet(defaultIfEmpty(d.DownloadFileTypes, defaultDownloadExt)) + d.normalizedPrefix = normalizePrefix(defaultIfEmpty(d.PathPrefix, "/d")) + d.normalizedMode = normalizeSaveMode(d.SaveLocalMode) + + if d.Version != 5 { + d.FilterFileTypes = mergeDefaultExtCSV(d.FilterFileTypes, defaultMediaExt) + d.DownloadFileTypes = mergeDefaultExtCSV(d.DownloadFileTypes, defaultDownloadExt) + d.PathPrefix = "/d" + d.Version = 5 + } + if d.SaveLocalMode == "" { + d.SaveLocalMode = SaveLocalInsertMode + } + if d.SignExpireHours < 0 { + d.SignExpireHours = 0 + } + if d.RotateSignNow { + d.RotateSignNow = false + op.MustSaveDriverStorage(d) + if d.SaveStrmToLocal && strings.TrimSpace(d.SaveStrmLocalPath) != "" { + go func() { + log.Infof("strm: start rotating signs for [%s]", d.MountPath) + d.rotateAllLocal(context.Background()) + log.Infof("strm: finished rotating signs for [%s]", d.MountPath) + }() + } + } + return nil +} + +func (d *Strm) Drop(ctx context.Context) error { + d.aliases = nil + d.mediaExtSet = nil + d.downloadExtSet = nil + return nil +} + +func (Addition) GetRootPath() string { + return "/" +} + +func (d *Strm) Get(ctx context.Context, path string) (model.Obj, error) { + path = cleanPath(path) + root, sub := d.splitVirtualPath(path) + targets, ok := d.aliases[root] + if !ok { + return nil, errs.ObjectNotFound + } + + for _, targetRoot := range targets { + realPath := stdpath.Join(targetRoot, sub) + obj, err := fs.Get(ctx, realPath, &fs.GetArgs{NoLog: true}) + if err != nil { + continue + } + if obj.IsDir() { + return wrapObj(path, obj, 0), nil + } + return wrapObj(realPath, obj, obj.GetSize()), nil + } + + if strings.HasSuffix(strings.ToLower(path), ".strm") { + return nil, errs.NotSupport + } + return nil, errs.ObjectNotFound +} + +func (d *Strm) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + virtualDir := cleanPath(dir.GetPath()) + if virtualDir == "/" && !d.autoFlatten { + objs := d.listVirtualRoots() + d.syncLocalDir(ctx, virtualDir, objs) + return objs, nil + } + + root, sub := d.splitVirtualPath(virtualDir) + targets, ok := d.aliases[root] + if !ok { + return nil, errs.ObjectNotFound + } + + out := make([]model.Obj, 0) + for _, targetRoot := range targets { + realDir := stdpath.Join(targetRoot, sub) + objs, err := fs.List(ctx, realDir, &fs.ListArgs{NoLog: true, Refresh: args.Refresh}) + if err != nil { + continue + } + out = append(out, d.mapListedObjects(ctx, realDir, objs)...) + } + + d.syncLocalDir(ctx, virtualDir, out) + return out, nil +} + +func (d *Strm) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.GetID() == "strm" { + line := d.buildStrmLine(ctx, file.GetPath()) + return &model.Link{MFile: model.NewNopMFile(strings.NewReader(line + "\n"))}, nil + } + return d.linkRealFile(ctx, file.GetPath(), args) +} + +func (d *Strm) listVirtualRoots() []model.Obj { + objs := make([]model.Obj, 0, len(d.aliases)) + for k := range d.aliases { + objs = append(objs, &model.Object{ + Path: "/" + k, + Name: k, + IsFolder: true, + Modified: d.Modified, + }) + } + return objs +} + +func (d *Strm) rotateAllLocal(ctx context.Context) { + for alias, roots := range d.aliases { + virtualRoot := "/" + if !d.autoFlatten { + virtualRoot = "/" + alias + } + for _, realRoot := range roots { + d.walkAndSync(ctx, virtualRoot, realRoot) + } + } +} + +func (d *Strm) walkAndSync(ctx context.Context, virtualDir, realDir string) { + objs, err := fs.List(ctx, realDir, &fs.ListArgs{NoLog: true, Refresh: true}) + if err != nil { + log.Warnf("strm: rotate list failed %s: %v", realDir, err) + return + } + mapped := d.mapListedObjects(ctx, realDir, objs) + d.syncLocalDirWithMode(ctx, virtualDir, mapped, SaveLocalUpdateMode) + for _, obj := range objs { + if !obj.IsDir() { + continue + } + childVirtual := stdpath.Join(virtualDir, obj.GetName()) + childReal := stdpath.Join(realDir, obj.GetName()) + d.walkAndSync(ctx, childVirtual, childReal) + } +} + +func (d *Strm) mapListedObjects(ctx context.Context, realDir string, listed []model.Obj) []model.Obj { + ret := make([]model.Obj, 0, len(listed)) + for _, obj := range listed { + if obj.IsDir() { + ret = append(ret, &model.Object{ + Name: obj.GetName(), + Path: "", + IsFolder: true, + Modified: obj.ModTime(), + }) + continue + } + + realPath := stdpath.Join(realDir, obj.GetName()) + ext := fileExt(obj.GetName()) + + if _, ok := d.downloadExtSet[ext]; ok { + ret = append(ret, d.cloneWithPath(obj, realPath, obj.GetName(), "", obj.GetSize())) + continue + } + if _, ok := d.mediaExtSet[ext]; ok { + strmName := strings.TrimSuffix(obj.GetName(), stdpath.Ext(obj.GetName())) + ".strm" + size := int64(len(d.buildStrmLine(ctx, realPath)) + 1) + ret = append(ret, d.cloneWithPath(obj, realPath, strmName, "strm", size)) + } + } + return ret +} + +func (d *Strm) cloneWithPath(src model.Obj, realPath, name, id string, size int64) model.Obj { + baseObj := model.Object{ + ID: id, + Path: realPath, + Name: name, + Size: size, + Modified: src.ModTime(), + IsFolder: src.IsDir(), + } + thumb, ok := model.GetThumb(src) + if !ok { + return &baseObj + } + return &model.ObjThumb{Object: baseObj, Thumbnail: model.Thumbnail{Thumbnail: thumb}} +} + +func (d *Strm) splitVirtualPath(path string) (string, string) { + if d.autoFlatten { + return d.singleRootKey, path + } + trimmed := strings.TrimPrefix(path, "/") + parts := strings.SplitN(trimmed, "/", 2) + if len(parts) == 1 { + return parts[0], "" + } + return parts[0], parts[1] +} + +func cleanPath(path string) string { + if path == "" { + return "/" + } + return filepath.ToSlash(stdpath.Clean("/" + strings.TrimPrefix(path, "/"))) +} + +func wrapObj(path string, src model.Obj, size int64) model.Obj { + return &model.Object{ + Path: path, + Name: src.GetName(), + Size: size, + Modified: src.ModTime(), + IsFolder: src.IsDir(), + HashInfo: src.GetHash(), + } +} + +var _ driver.Driver = (*Strm)(nil) diff --git a/drivers/strm/hook.go b/drivers/strm/hook.go new file mode 100644 index 00000000000..6fad6993c67 --- /dev/null +++ b/drivers/strm/hook.go @@ -0,0 +1,3 @@ +package strm + +// Local sync is triggered during STRM directory listing. diff --git a/drivers/strm/meta.go b/drivers/strm/meta.go new file mode 100644 index 00000000000..803ed035762 --- /dev/null +++ b/drivers/strm/meta.go @@ -0,0 +1,44 @@ +package strm + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +const ( + SaveLocalInsertMode = "insert" + SaveLocalUpdateMode = "update" + SaveLocalSyncMode = "sync" +) + +type Addition struct { + Paths string `json:"paths" required:"true" type:"text"` + SiteUrl string `json:"siteUrl" type:"text" required:"false" help:"The prefix URL of generated strm file"` + PathPrefix string `json:"PathPrefix" type:"text" required:"false" default:"/d" help:"Path prefix in strm content"` + DownloadFileTypes string `json:"downloadFileTypes" type:"text" default:"ass,srt,vtt,sub,strm" required:"false" help:"Extensions to download as local files"` + FilterFileTypes string `json:"filterFileTypes" type:"text" default:"mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" required:"false" help:"Extensions to expose as .strm"` + EncodePath bool `json:"encodePath" default:"true" required:"true" help:"Encode path in strm content"` + WithoutUrl bool `json:"withoutUrl" default:"false" help:"Generate path-only strm content"` + WithSign bool `json:"withSign" default:"false" help:"Append sign query to generated URL"` + SignExpireHours int `json:"SignExpireHours" type:"number" default:"0" help:"Driver-level sign expiration in hours. 0 uses global link_expiration"` + RotateSignNow bool `json:"RotateSignNow" type:"bool" default:"false" help:"Set true and save to rotate signs now (rewrite local STRM), then auto reset to false"` + SaveStrmToLocal bool `json:"SaveStrmToLocal" default:"false" help:"Save generated files to local disk"` + SaveStrmLocalPath string `json:"SaveStrmLocalPath" type:"text" help:"Local path for generated files"` + SaveLocalMode string `json:"SaveLocalMode" type:"select" help:"Local save mode" options:"insert,update,sync" default:"insert"` + Version int +} + +var config = driver.Config{ + Name: "Strm", + LocalSort: true, + OnlyProxy: true, + NoCache: true, + NoUpload: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Strm{Addition: Addition{EncodePath: true}} + }) +} diff --git a/drivers/strm/util.go b/drivers/strm/util.go new file mode 100644 index 00000000000..2d640bc32fb --- /dev/null +++ b/drivers/strm/util.go @@ -0,0 +1,317 @@ +package strm + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + stdpath "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + pkgerr "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +const ( + defaultMediaExt = "mp4,mkv,flv,avi,wmv,ts,rmvb,webm,mp3,flac,aac,wav,ogg,m4a,wma,alac" + defaultDownloadExt = "ass,srt,vtt,sub,strm" +) + +func parseAliases(raw string) map[string][]string { + aliases := map[string][]string{} + for _, line := range strings.Split(raw, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + name, target := parseAliasLine(line) + aliases[name] = append(aliases[name], cleanPath(target)) + } + return aliases +} + +func parseAliasLine(line string) (string, string) { + if strings.Contains(line, ":") { + parts := strings.SplitN(line, ":", 2) + if !strings.Contains(parts[0], "/") { + return parts[0], parts[1] + } + } + return stdpath.Base(line), line +} + +func parseExtSet(csv string) map[string]struct{} { + ret := map[string]struct{}{} + for _, part := range strings.Split(csv, ",") { + ext := normalizeExt(part) + if ext != "" { + ret[ext] = struct{}{} + } + } + return ret +} + +func mergeDefaultExtCSV(csv, defaults string) string { + base := parseExtSet(csv) + for ext := range parseExtSet(defaults) { + base[ext] = struct{}{} + } + keys := make([]string, 0, len(base)) + for k := range base { + keys = append(keys, k) + } + sort.Strings(keys) + return strings.Join(keys, ",") +} + +func normalizeExt(ext string) string { + ext = strings.ToLower(strings.TrimSpace(ext)) + ext = strings.TrimPrefix(ext, ".") + return ext +} + +func fileExt(name string) string { + return normalizeExt(stdpath.Ext(name)) +} + +func defaultIfEmpty(v, fallback string) string { + if strings.TrimSpace(v) == "" { + return fallback + } + return v +} + +func normalizeSaveMode(mode string) string { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "sync": + return SaveLocalSyncMode + case "update": + return SaveLocalUpdateMode + case "insert", "missing": + return SaveLocalInsertMode + default: + return SaveLocalInsertMode + } +} + +func normalizePrefix(prefix string) string { + prefix = strings.TrimSpace(prefix) + if prefix == "" { + return "/d" + } + if !strings.HasPrefix(prefix, "/") { + prefix = "/" + prefix + } + return prefix +} + +func (d *Strm) buildStrmLine(ctx context.Context, realPath string) string { + pathPart := realPath + if d.EncodePath { + pathPart = utils.EncodePath(pathPart, true) + } + if d.WithSign { + sep := "?" + if strings.Contains(pathPart, "?") { + sep = "&" + } + pathPart += sep + "sign=" + d.generateSign(realPath) + } + joined := stdpath.Join(d.normalizedPrefix, pathPart) + if !strings.HasPrefix(joined, "/") { + joined = "/" + joined + } + if d.WithoutUrl { + return joined + } + baseURL := strings.TrimSpace(d.SiteUrl) + if baseURL == "" { + if c, ok := ctx.(*gin.Context); ok { + baseURL = common.GetApiUrl(c.Request) + } else { + baseURL = common.GetApiUrl(nil) + } + } + baseURL = strings.TrimSuffix(baseURL, "/") + return baseURL + joined +} + +func (d *Strm) linkRealFile(ctx context.Context, realPath string, args model.LinkArgs) (*model.Link, error) { + storage, actualPath, err := op.GetStorageAndActualPath(realPath) + if err != nil { + return nil, err + } + if !args.Redirect { + link, _, linkErr := op.Link(ctx, storage, actualPath, args) + return link, linkErr + } + obj, err := fs.Get(ctx, realPath, &fs.GetArgs{NoLog: true}) + if err != nil { + return nil, err + } + if common.ShouldProxy(storage, obj.GetName()) { + api := common.GetApiUrl(args.HttpReq) + if api == "" { + api = strings.TrimSuffix(strings.TrimSpace(d.SiteUrl), "/") + } + if api == "" { + api = common.GetApiUrl(nil) + } + return &model.Link{URL: fmt.Sprintf("%s/p%s?sign=%s", api, utils.EncodePath(realPath, true), d.generateSign(realPath))}, nil + } + link, _, linkErr := op.Link(ctx, storage, actualPath, args) + return link, linkErr +} + +func (d *Strm) syncLocalDir(ctx context.Context, virtualDir string, objs []model.Obj) { + d.syncLocalDirWithMode(ctx, virtualDir, objs, d.normalizedMode) +} + +func (d *Strm) syncLocalDirWithMode(ctx context.Context, virtualDir string, objs []model.Obj, mode string) { + if !d.SaveStrmToLocal || strings.TrimSpace(d.SaveStrmLocalPath) == "" { + return + } + baseDir := filepath.Clean(d.SaveStrmLocalPath) + localDir := baseDir + if virtualDir != "/" { + localDir = filepath.Join(baseDir, filepath.FromSlash(strings.TrimPrefix(virtualDir, "/"))) + } + if err := os.MkdirAll(localDir, 0o755); err != nil { + log.Warnf("strm: mkdir failed %s: %v", localDir, err) + return + } + + expected := map[string]bool{} + for _, obj := range objs { + name := obj.GetName() + expected[name] = obj.IsDir() + localPath := filepath.Join(localDir, name) + if obj.IsDir() { + _ = os.MkdirAll(localPath, 0o755) + continue + } + payload, err := d.localPayload(ctx, obj) + if err != nil { + log.Warnf("strm: build local payload failed %s: %v", localPath, err) + continue + } + if err = d.writeLocal(localPath, payload, mode); err != nil { + log.Warnf("strm: write local failed %s: %v", localPath, err) + } + } + + if mode == SaveLocalSyncMode { + d.syncDeleteExtras(localDir, expected) + } +} + +func (d *Strm) localPayload(ctx context.Context, obj model.Obj) ([]byte, error) { + if obj.GetID() == "strm" { + return []byte(d.buildStrmLine(ctx, obj.GetPath()) + "\n"), nil + } + link, err := d.linkRealFile(ctx, obj.GetPath(), model.LinkArgs{Redirect: true}) + if err != nil { + return nil, err + } + return readLinkBytes(ctx, link) +} + +func readLinkBytes(ctx context.Context, link *model.Link) ([]byte, error) { + if link.MFile != nil { + defer link.MFile.Close() + return io.ReadAll(link.MFile) + } + if link.RangeReadCloser != nil { + rc, err := link.RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) + if err == nil && rc != nil { + defer rc.Close() + return io.ReadAll(rc) + } + } + if link.URL == "" { + return nil, fmt.Errorf("empty link") + } + url := link.URL + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + api := common.GetApiUrl(nil) + if api == "" { + return nil, fmt.Errorf("relative url without site url: %s", url) + } + url = strings.TrimSuffix(api, "/") + url + } + res, err := base.RestyClient.R().SetContext(ctx).SetDoNotParseResponse(true).Get(url) + if err != nil { + return nil, err + } + defer res.RawBody().Close() + if res.StatusCode() >= http.StatusBadRequest { + return nil, fmt.Errorf("read url failed: status=%d", res.StatusCode()) + } + return io.ReadAll(res.RawBody()) +} + +func (d *Strm) writeLocal(path string, payload []byte, mode string) error { + if mode == SaveLocalInsertMode && utils.Exists(path) { + return nil + } + if st, err := os.Stat(path); err == nil && st.IsDir() { + if mode != SaveLocalSyncMode { + return nil + } + if err = os.RemoveAll(path); err != nil { + return err + } + } + if mode != SaveLocalInsertMode { + if old, err := os.ReadFile(path); err == nil { + if bytes.Equal(old, payload) { + return nil + } + } + } + f, err := utils.CreateNestedFile(path) + if err != nil { + return err + } + defer f.Close() + _, err = f.Write(payload) + return err +} + +func (d *Strm) syncDeleteExtras(localDir string, expected map[string]bool) { + entries, err := os.ReadDir(localDir) + if err != nil { + if pkgerr.Cause(err) != os.ErrNotExist { + log.Warnf("strm: read local dir failed %s: %v", localDir, err) + } + return + } + for _, e := range entries { + expectDir, ok := expected[e.Name()] + full := filepath.Join(localDir, e.Name()) + if !ok || expectDir != e.IsDir() { + _ = os.RemoveAll(full) + } + } +} + +func (d *Strm) generateSign(path string) string { + if d.SignExpireHours > 0 { + return sign.WithDuration(path, time.Duration(d.SignExpireHours)*time.Hour) + } + return sign.Sign(path) +} diff --git a/drivers/teambition/driver.go b/drivers/teambition/driver.go index d4fcc401bad..b37c324b288 100644 --- a/drivers/teambition/driver.go +++ b/drivers/teambition/driver.go @@ -3,12 +3,12 @@ package teambition import ( "context" "errors" + "github.com/alist-org/alist/v3/pkg/utils" "net/http" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" ) @@ -128,15 +128,27 @@ func (d *Teambition) Put(ctx context.Context, dstDir model.Obj, stream model.Fil if d.UseS3UploadMethod { return d.newUpload(ctx, dstDir, stream, up) } - res, err := d.request("/api/v2/users/me", http.MethodGet, nil, nil) - if err != nil { - return err + var ( + token string + err error + ) + if d.isInternational() { + res, err := d.request("/projects", http.MethodGet, nil, nil) + if err != nil { + return err + } + token = getBetweenStr(string(res), "strikerAuth":"", "","phoneForLogin") + } else { + res, err := d.request("/api/v2/users/me", http.MethodGet, nil, nil) + if err != nil { + return err + } + token = utils.Json.Get(res, "strikerAuth").ToString() } - token := utils.Json.Get(res, "strikerAuth").ToString() var newFile *FileUpload if stream.GetSize() <= 20971520 { // post upload - newFile, err = d.upload(ctx, stream, token) + newFile, err = d.upload(ctx, stream, token, up) } else { // chunk upload //err = base.ErrNotImplement diff --git a/drivers/teambition/help.go b/drivers/teambition/help.go new file mode 100644 index 00000000000..8581c3e827c --- /dev/null +++ b/drivers/teambition/help.go @@ -0,0 +1,18 @@ +package teambition + +import "strings" + +func getBetweenStr(str, start, end string) string { + n := strings.Index(str, start) + if n == -1 { + return "" + } + n = n + len(start) + str = string([]byte(str)[n:]) + m := strings.Index(str, end) + if m == -1 { + return "" + } + str = string([]byte(str)[:m]) + return str +} diff --git a/drivers/teambition/util.go b/drivers/teambition/util.go index 04f222de95f..01c12cb17a1 100644 --- a/drivers/teambition/util.go +++ b/drivers/teambition/util.go @@ -1,6 +1,7 @@ package teambition import ( + "bytes" "context" "errors" "fmt" @@ -120,25 +121,31 @@ func (d *Teambition) getFiles(parentId string) ([]model.Obj, error) { return files, nil } -func (d *Teambition) upload(ctx context.Context, file model.FileStreamer, token string) (*FileUpload, error) { +func (d *Teambition) upload(ctx context.Context, file model.FileStreamer, token string, up driver.UpdateProgress) (*FileUpload, error) { prefix := "tcs" if d.isInternational() { prefix = "us-tcs" } + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }) var newFile FileUpload - _, err := base.RestyClient.R(). + res, err := base.RestyClient.R(). SetContext(ctx). SetResult(&newFile).SetHeader("Authorization", token). SetMultipartFormData(map[string]string{ - "name": file.GetName(), - "type": file.GetMimetype(), - "size": strconv.FormatInt(file.GetSize(), 10), - //"lastModifiedDate": "", - }).SetMultipartField("file", file.GetName(), file.GetMimetype(), file). + "name": file.GetName(), + "type": file.GetMimetype(), + "size": strconv.FormatInt(file.GetSize(), 10), + "lastModifiedDate": time.Now().Format("Mon Jan 02 2006 15:04:05 GMT+0800 (中国标准时间)"), + }). + SetMultipartField("file", file.GetName(), file.GetMimetype(), reader). Post(fmt.Sprintf("https://%s.teambition.net/upload", prefix)) if err != nil { return nil, err } + log.Debugf("[teambition] upload response: %s", res.String()) return &newFile, nil } @@ -182,14 +189,13 @@ func (d *Teambition) chunkUpload(ctx context.Context, file model.FileStreamer, t "Authorization": token, "Content-Type": "application/octet-stream", "Referer": referer, - }).SetBody(chunkData).Post(u) + }). + SetBody(driver.NewLimitedUploadStream(ctx, bytes.NewReader(chunkData))). + Post(u) if err != nil { return nil, err } - if err != nil { - return nil, err - } - up(i * 100 / newChunk.Chunks) + up(float64(i) * 100 / float64(newChunk.Chunks)) } _, err = base.RestyClient.R().SetHeader("Authorization", token).Post( fmt.Sprintf("https://%s.teambition.net/upload/chunk/%s", @@ -243,12 +249,18 @@ func (d *Teambition) newUpload(ctx context.Context, dstDir model.Obj, stream mod return err } uploader := s3manager.NewUploader(ss) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } input := &s3manager.UploadInput{ Bucket: &uploadToken.Upload.Bucket, Key: &uploadToken.Upload.Key, ContentDisposition: &uploadToken.Upload.ContentDisposition, ContentType: &uploadToken.Upload.ContentType, - Body: stream, + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: stream, + UpdateProgress: up, + }), } _, err = uploader.UploadWithContext(ctx, input) if err != nil { diff --git a/drivers/template/driver.go b/drivers/template/driver.go index 439f57f35f9..ff3648db3b0 100644 --- a/drivers/template/driver.go +++ b/drivers/template/driver.go @@ -66,11 +66,33 @@ func (d *Template) Remove(ctx context.Context, obj model.Obj) error { return errs.NotImplement } -func (d *Template) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (d *Template) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { // TODO upload file, optional return nil, errs.NotImplement } +func (d *Template) GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) { + // TODO get archive file meta-info, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Template) ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) { + // TODO list args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Template) Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) { + // TODO return link of file args.InnerPath in the archive obj, return errs.NotImplement to use an internal archive tool, optional + return nil, errs.NotImplement +} + +func (d *Template) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) { + // TODO extract args.InnerPath path in the archive srcObj to the dstDir location, optional + // a folder with the same name as the archive file needs to be created to store the extracted results if args.PutIntoNewDir + // return errs.NotImplement to use an internal archive tool + return nil, errs.NotImplement +} + //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/terabox/driver.go b/drivers/terabox/driver.go index b5287f5a7f8..82962b8148a 100644 --- a/drivers/terabox/driver.go +++ b/drivers/terabox/driver.go @@ -10,7 +10,6 @@ import ( "math" stdpath "path" "strconv" - "strings" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/pkg/utils" @@ -23,7 +22,9 @@ import ( type Terabox struct { model.Storage Addition - JsToken string + JsToken string + url_domain_prefix string + base_url string } func (d *Terabox) Config() driver.Config { @@ -36,6 +37,8 @@ func (d *Terabox) GetAddition() driver.Additional { func (d *Terabox) Init(ctx context.Context) error { var resp CheckLoginResp + d.base_url = "https://www.terabox.com" + d.url_domain_prefix = "jp" _, err := d.get("/api/check/login", nil, &resp) if err != nil { return err @@ -71,7 +74,16 @@ func (d *Terabox) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Terabox) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - _, err := d.create(stdpath.Join(parentDir.GetPath(), dirName), 0, 1, "", "") + params := map[string]string{ + "a": "commit", + } + data := map[string]string{ + "path": stdpath.Join(parentDir.GetPath(), dirName), + "isdir": "1", + "block_list": "[]", + } + res, err := d.post_form("/api/create", params, data, nil) + log.Debugln(string(res)) return err } @@ -117,63 +129,61 @@ func (d *Terabox) Remove(ctx context.Context, obj model.Obj) error { } func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - tempFile, err := stream.CacheFullInTempFile() + resp, err := base.RestyClient.R(). + SetContext(ctx). + Get("https://" + d.url_domain_prefix + "-data.terabox.com/rest/2.0/pcs/file?method=locateupload") if err != nil { return err } - var Default int64 = 4 * 1024 * 1024 - defaultByteData := make([]byte, Default) - count := int(math.Ceil(float64(stream.GetSize()) / float64(Default))) - // cal md5 - h1 := md5.New() - h2 := md5.New() - block_list := make([]string, 0) - left := stream.GetSize() - for i := 0; i < count; i++ { - byteSize := Default - var byteData []byte - if left < Default { - byteSize = left - byteData = make([]byte, byteSize) - } else { - byteData = defaultByteData - } - left -= byteSize - _, err = io.ReadFull(tempFile, byteData) - if err != nil { - return err - } - h1.Write(byteData) - h2.Write(byteData) - block_list = append(block_list, fmt.Sprintf("\"%s\"", hex.EncodeToString(h2.Sum(nil)))) - h2.Reset() - } - - _, err = tempFile.Seek(0, io.SeekStart) + var locateupload_resp LocateUploadResp + err = utils.Json.Unmarshal(resp.Body(), &locateupload_resp) if err != nil { + log.Debugln(resp) return err } + log.Debugln(locateupload_resp) + // precreate file rawPath := stdpath.Join(dstDir.GetPath(), stream.GetName()) path := encodeURIComponent(rawPath) - block_list_str := fmt.Sprintf("[%s]", strings.Join(block_list, ",")) - data := fmt.Sprintf("path=%s&size=%d&isdir=0&autoinit=1&block_list=%s", - path, stream.GetSize(), - block_list_str) - params := map[string]string{} + + var precreateBlockListStr string + if stream.GetSize() > initialChunkSize { + precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761","a5fc157d78e6ad1c7e114b056c92821e"]` + } else { + precreateBlockListStr = `["5910a591dd8fc18c32a8f3df4fdc1761"]` + } + + data := map[string]string{ + "path": rawPath, + "autoinit": "1", + "target_path": dstDir.GetPath(), + "block_list": precreateBlockListStr, + "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), + "file_limit_switch_v34": "true", + } var precreateResp PrecreateResp - _, err = d.post("/api/precreate", params, data, &precreateResp) + log.Debugln(data) + res, err := d.post_form("/api/precreate", nil, data, &precreateResp) if err != nil { return err } log.Debugf("%+v", precreateResp) if precreateResp.Errno != 0 { + log.Debugln(string(res)) return fmt.Errorf("[terabox] failed to precreate file, errno: %d", precreateResp.Errno) } if precreateResp.ReturnType == 2 { return nil } - params = map[string]string{ + + // upload chunks + tempFile, err := stream.CacheFullInTempFile() + if err != nil { + return err + } + + params := map[string]string{ "method": "upload", "path": path, "uploadid": precreateResp.Uploadid, @@ -182,42 +192,82 @@ func (d *Terabox) Put(ctx context.Context, dstDir model.Obj, stream model.FileSt "channel": "dubox", "clienttype": "0", } - left = stream.GetSize() - for i, partseq := range precreateResp.BlockList { + + streamSize := stream.GetSize() + chunkSize := calculateChunkSize(streamSize) + chunkByteData := make([]byte, chunkSize) + count := int(math.Ceil(float64(streamSize) / float64(chunkSize))) + left := streamSize + uploadBlockList := make([]string, 0, count) + h := md5.New() + for partseq := 0; partseq < count; partseq++ { if utils.IsCanceled(ctx) { return ctx.Err() } - byteSize := Default + byteSize := chunkSize var byteData []byte - if left < Default { + if left >= chunkSize { + byteData = chunkByteData + } else { byteSize = left byteData = make([]byte, byteSize) - } else { - byteData = defaultByteData } left -= byteSize _, err = io.ReadFull(tempFile, byteData) if err != nil { return err } - u := "https://c-jp.terabox.com/rest/2.0/pcs/superfile2" + + // calculate md5 + h.Write(byteData) + uploadBlockList = append(uploadBlockList, hex.EncodeToString(h.Sum(nil))) + h.Reset() + + u := "https://" + locateupload_resp.Host + "/rest/2.0/pcs/superfile2" params["partseq"] = strconv.Itoa(partseq) res, err := base.RestyClient.R(). SetContext(ctx). SetQueryParams(params). - SetFileReader("file", stream.GetName(), bytes.NewReader(byteData)). + SetFileReader("file", stream.GetName(), driver.NewLimitedUploadStream(ctx, bytes.NewReader(byteData))). SetHeader("Cookie", d.Cookie). Post(u) if err != nil { return err } log.Debugln(res.String()) - if len(precreateResp.BlockList) > 0 { - up(i * 100 / len(precreateResp.BlockList)) + if count > 0 { + up(float64(partseq) * 100 / float64(count)) } } - _, err = d.create(rawPath, stream.GetSize(), 0, precreateResp.Uploadid, block_list_str) - return err + + // create file + params = map[string]string{ + "isdir": "0", + "rtype": "1", + } + + uploadBlockListStr, err := utils.Json.MarshalToString(uploadBlockList) + if err != nil { + return err + } + data = map[string]string{ + "path": rawPath, + "size": strconv.FormatInt(stream.GetSize(), 10), + "uploadid": precreateResp.Uploadid, + "target_path": dstDir.GetPath(), + "block_list": uploadBlockListStr, + "local_mtime": strconv.FormatInt(stream.ModTime().Unix(), 10), + } + var createResp CreateResp + res, err = d.post_form("/api/create", params, data, &createResp) + log.Debugln(string(res)) + if err != nil { + return err + } + if createResp.Errno != 0 { + return fmt.Errorf("[terabox] failed to create file, errno: %d", createResp.Errno) + } + return nil } var _ driver.Driver = (*Terabox)(nil) diff --git a/drivers/terabox/types.go b/drivers/terabox/types.go index 890d53056ea..f4d50ddef37 100644 --- a/drivers/terabox/types.go +++ b/drivers/terabox/types.go @@ -95,3 +95,11 @@ type PrecreateResp struct { type CheckLoginResp struct { Errno int `json:"errno"` } + +type LocateUploadResp struct { + Host string `json:"host"` +} + +type CreateResp struct { + Errno int `json:"errno"` +} diff --git a/drivers/terabox/util.go b/drivers/terabox/util.go index 0a4e7879379..058eecd6085 100644 --- a/drivers/terabox/util.go +++ b/drivers/terabox/util.go @@ -14,6 +14,12 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" +) + +const ( + initialChunkSize int64 = 4 << 20 // 4MB + initialSizeThreshold int64 = 4 << 30 // 4GB ) func getStrBetween(raw, start, end string) string { @@ -28,11 +34,11 @@ func getStrBetween(raw, start, end string) string { } func (d *Terabox) resetJsToken() error { - u := "https://www.terabox.com/main" + u := d.base_url res, err := base.RestyClient.R().SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", - "Referer": "https://www.terabox.com/", + "Referer": d.base_url, "User-Agent": base.UserAgent, "X-Requested-With": "XMLHttpRequest", }).Get(u) @@ -48,12 +54,12 @@ func (d *Terabox) resetJsToken() error { return nil } -func (d *Terabox) request(furl string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { +func (d *Terabox) request(rurl string, method string, callback base.ReqCallback, resp interface{}, noRetry ...bool) ([]byte, error) { req := base.RestyClient.R() req.SetHeaders(map[string]string{ "Cookie": d.Cookie, "Accept": "application/json, text/plain, */*", - "Referer": "https://www.terabox.com/", + "Referer": d.base_url, "User-Agent": base.UserAgent, "X-Requested-With": "XMLHttpRequest", }) @@ -70,7 +76,7 @@ func (d *Terabox) request(furl string, method string, callback base.ReqCallback, if resp != nil { req.SetResult(resp) } - res, err := req.Execute(method, furl) + res, err := req.Execute(method, d.base_url+rurl) if err != nil { return nil, err } @@ -82,14 +88,24 @@ func (d *Terabox) request(furl string, method string, callback base.ReqCallback, return nil, err } if !utils.IsBool(noRetry...) { - return d.request(furl, method, callback, resp, true) + return d.request(rurl, method, callback, resp, true) + } + } else if errno == -6 { + header := res.Header() + log.Debugln(header) + urlDomainPrefix := header.Get("Url-Domain-Prefix") + if len(urlDomainPrefix) > 0 { + d.url_domain_prefix = urlDomainPrefix + d.base_url = "https://" + d.url_domain_prefix + ".terabox.com" + log.Debugln("Redirect base_url to", d.base_url) + return d.request(rurl, method, callback, resp, noRetry...) } } return res.Body(), nil } func (d *Terabox) get(pathname string, params map[string]string, resp interface{}) ([]byte, error) { - return d.request("https://www.terabox.com"+pathname, http.MethodGet, func(req *resty.Request) { + return d.request(pathname, http.MethodGet, func(req *resty.Request) { if params != nil { req.SetQueryParams(params) } @@ -97,7 +113,7 @@ func (d *Terabox) get(pathname string, params map[string]string, resp interface{ } func (d *Terabox) post(pathname string, params map[string]string, data interface{}, resp interface{}) ([]byte, error) { - return d.request("https://www.terabox.com"+pathname, http.MethodPost, func(req *resty.Request) { + return d.request(pathname, http.MethodPost, func(req *resty.Request) { if params != nil { req.SetQueryParams(params) } @@ -105,6 +121,15 @@ func (d *Terabox) post(pathname string, params map[string]string, data interface }, resp) } +func (d *Terabox) post_form(pathname string, params map[string]string, data map[string]string, resp interface{}) ([]byte, error) { + return d.request(pathname, http.MethodPost, func(req *resty.Request) { + if params != nil { + req.SetQueryParams(params) + } + req.SetFormData(data) + }, resp) +} + func (d *Terabox) getFiles(dir string) ([]File, error) { page := 1 num := 100 @@ -237,17 +262,24 @@ func (d *Terabox) manage(opera string, filelist interface{}) ([]byte, error) { return d.post("/api/filemanager", params, data, nil) } -func (d *Terabox) create(path string, size int64, isdir int, uploadid, block_list string) ([]byte, error) { - params := map[string]string{} - data := fmt.Sprintf("path=%s&size=%d&isdir=%d", encodeURIComponent(path), size, isdir) - if uploadid != "" { - data += fmt.Sprintf("&uploadid=%s&block_list=%s", uploadid, block_list) - } - return d.post("/api/create", params, data, nil) -} - func encodeURIComponent(str string) string { r := url.QueryEscape(str) r = strings.ReplaceAll(r, "+", "%20") return r } + +func calculateChunkSize(streamSize int64) int64 { + chunkSize := initialChunkSize + sizeThreshold := initialSizeThreshold + + if streamSize < chunkSize { + return streamSize + } + + for streamSize > sizeThreshold { + chunkSize <<= 1 + sizeThreshold <<= 1 + } + + return chunkSize +} diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index cac6733f01b..1d2f2a81f65 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strconv" "strings" "github.com/alist-org/alist/v3/drivers/base" @@ -11,6 +12,7 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" "github.com/aws/aws-sdk-go/aws" @@ -43,26 +45,29 @@ func (x *Thunder) Init(ctx context.Context) (err error) { Common: &Common{ client: base.NewRestyClient(), Algorithms: []string{ - "HPxr4BVygTQVtQkIMwQH33ywbgYG5l4JoR", - "GzhNkZ8pOBsCY+7", - "v+l0ImTpG7c7/", - "e5ztohgVXNP", - "t", - "EbXUWyVVqQbQX39Mbjn2geok3/0WEkAVxeqhtx857++kjJiRheP8l77gO", - "o7dvYgbRMOpHXxCs", - "6MW8TD8DphmakaxCqVrfv7NReRRN7ck3KLnXBculD58MvxjFRqT+", - "kmo0HxCKVfmxoZswLB4bVA/dwqbVAYghSb", - "j", - "4scKJNdd7F27Hv7tbt", + "9uJNVj/wLmdwKrJaVj/omlQ", + "Oz64Lp0GigmChHMf/6TNfxx7O9PyopcczMsnf", + "Eb+L7Ce+Ej48u", + "jKY0", + "ASr0zCl6v8W4aidjPK5KHd1Lq3t+vBFf41dqv5+fnOd", + "wQlozdg6r1qxh0eRmt3QgNXOvSZO6q/GXK", + "gmirk+ciAvIgA/cxUUCema47jr/YToixTT+Q6O", + "5IiCoM9B1/788ntB", + "P07JH0h6qoM6TSUAK2aL9T5s2QBVeY9JWvalf", + "+oK0AN", }, - DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), + DeviceID: func() string { + if len(x.DeviceID) != 32 { + return utils.GetMD5EncodeStr(x.DeviceID) + } + return x.DeviceID + }(), ClientID: "Xp6vsxz_7IYVw2BB", ClientSecret: "Xp6vsy4tN9toTVdMSpomVdXpRmES", - ClientVersion: "7.51.0.8196", + ClientVersion: "8.31.0.9726", PackageName: "com.xunlei.downloadprovider", - UserAgent: "ANDROID-com.xunlei.downloadprovider/7.51.0.8196 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", + UserAgent: "ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)", DownloadUserAgent: "Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)", - refreshCTokenCk: func(token string) { x.CaptchaToken = token op.MustSaveDriverStorage(x) @@ -78,6 +83,8 @@ func (x *Thunder) Init(ctx context.Context) (err error) { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) op.MustSaveDriverStorage(x) } + // 清空 信任密钥 + x.Addition.CreditKey = "" } x.SetTokenResp(token) return err @@ -91,6 +98,17 @@ func (x *Thunder) Init(ctx context.Context) (err error) { x.SetCaptchaToken(ctoekn) } + if x.Addition.CreditKey != "" { + x.SetCreditKey(x.Addition.CreditKey) + } + + if x.Addition.DeviceID != "" { + x.Common.DeviceID = x.Addition.DeviceID + } else { + x.Addition.DeviceID = x.Common.DeviceID + op.MustSaveDriverStorage(x) + } + // 防止重复登录 identity := x.GetIdentity() if x.identity != identity || !x.IsLogin() { @@ -100,6 +118,8 @@ func (x *Thunder) Init(ctx context.Context) (err error) { if err != nil { return err } + // 清空 信任密钥 + x.Addition.CreditKey = "" x.SetTokenResp(token) } return nil @@ -159,6 +179,17 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) { x.SetCaptchaToken(x.CaptchaToken) } + if x.ExpertAddition.CreditKey != "" { + x.SetCreditKey(x.ExpertAddition.CreditKey) + } + + if x.ExpertAddition.DeviceID != "" { + x.Common.DeviceID = x.ExpertAddition.DeviceID + } else { + x.ExpertAddition.DeviceID = x.Common.DeviceID + op.MustSaveDriverStorage(x) + } + // 签名方法 if x.SignType == "captcha_sign" { x.Common.Timestamp = x.Timestamp @@ -192,6 +223,8 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) { if err != nil { return err } + // 清空 信任密钥 + x.ExpertAddition.CreditKey = "" x.SetTokenResp(token) x.SetRefreshTokenFunc(func() error { token, err := x.XunLeiCommon.RefreshToken(x.TokenResp.RefreshToken) @@ -200,6 +233,8 @@ func (x *ThunderExpert) Init(ctx context.Context) (err error) { if err != nil { x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) } + // 清空 信任密钥 + x.ExpertAddition.CreditKey = "" } x.SetTokenResp(token) op.MustSaveDriverStorage(x) @@ -231,7 +266,8 @@ func (x *ThunderExpert) SetTokenResp(token *TokenResp) { type XunLeiCommon struct { *Common - *TokenResp // 登录信息 + *TokenResp // 登录信息 + *CoreLoginResp // core登录信息 refreshTokenFunc func() error } @@ -331,29 +367,24 @@ func (xc *XunLeiCommon) Remove(ctx context.Context, obj model.Obj) error { return err } -func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - hi := stream.GetHash() - gcid := hi.GetHash(hash_extend.GCID) +func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + gcid := file.GetHash().GetHash(hash_extend.GCID) + var err error if len(gcid) < hash_extend.GCID.Width { - tFile, err := stream.CacheFullInTempFile() - if err != nil { - return err - } - - gcid, err = utils.HashFile(hash_extend.GCID, tFile, stream.GetSize()) + _, gcid, err = stream.CacheFullInTempFileAndHash(file, hash_extend.GCID, file.GetSize()) if err != nil { return err } } var resp UploadTaskResponse - _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { r.SetContext(ctx) r.SetBody(&base.Json{ "kind": FILE, "parent_id": dstDir.GetID(), - "name": stream.GetName(), - "size": stream.GetSize(), + "name": file.GetName(), + "size": file.GetSize(), "hash": gcid, "upload_type": UPLOAD_TYPE_RESUMABLE, }) @@ -373,11 +404,18 @@ func (xc *XunLeiCommon) Put(ctx context.Context, dstDir model.Obj, stream model. if err != nil { return err } - _, err = s3manager.NewUploader(s).UploadWithContext(ctx, &s3manager.UploadInput{ + uploader := s3manager.NewUploader(s) + if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) + } + _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ Bucket: aws.String(param.Bucket), Key: aws.String(param.Key), Expires: aws.Time(param.Expiration), - Body: stream, + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }), }) return err } @@ -429,6 +467,10 @@ func (xc *XunLeiCommon) SetTokenResp(tr *TokenResp) { xc.TokenResp = tr } +func (xc *XunLeiCommon) SetCoreTokenResp(tr *CoreLoginResp) { + xc.CoreLoginResp = tr +} + // 携带Authorization和CaptchaToken的请求 func (xc *XunLeiCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { data, err := xc.Common.Request(url, method, func(req *resty.Request) { @@ -457,7 +499,7 @@ func (xc *XunLeiCommon) Request(url string, method string, callback base.ReqCall } return nil, err case 9: // 验证码token过期 - if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.UserID); err != nil { + if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.TokenResp.UserID); err != nil { return nil, err } default: @@ -489,20 +531,25 @@ func (xc *XunLeiCommon) RefreshToken(refreshToken string) (*TokenResp, error) { // 登录 func (xc *XunLeiCommon) Login(username, password string) (*TokenResp, error) { - url := XLUSER_API_URL + "/auth/signin" - err := xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username) + //v3 login拿到 sessionID + sessionID, err := xc.CoreLogin(username, password) if err != nil { return nil, err } + //v1 login拿到令牌 + url := XLUSER_API_URL + "/auth/signin/token" + if err = xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username); err != nil { + return nil, err + } var resp TokenResp _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetPathParam("client_id", xc.ClientID) req.SetBody(&SignInRequest{ - CaptchaToken: xc.GetCaptchaToken(), ClientID: xc.ClientID, ClientSecret: xc.ClientSecret, - Username: username, - Password: password, + Provider: SignProvider, + SigninToken: sessionID, }) }, &resp) if err != nil { @@ -518,3 +565,108 @@ func (xc *XunLeiCommon) IsLogin() bool { _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) return err == nil } + +// 离线下载文件 +func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + var resp OfflineDownloadResp + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FILE, + "name": fileName, + "parent_id": parentDir.GetID(), + "upload_type": UPLOAD_TYPE_URL, + "url": base.Json{ + "url": fileUrl, + }, + }) + }, &resp) + + if err != nil { + return nil, err + } + + return &resp.Task, err +} + +/* +获取离线下载任务列表 +*/ +func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) ([]OfflineTask, error) { + res := make([]OfflineTask, 0) + + var resp OfflineListResp + _, err := xc.Request(TASK_API_URL, http.MethodGet, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(map[string]string{ + "type": "offline", + "limit": "10000", + "page_token": nextPageToken, + }) + }, &resp) + + if err != nil { + return nil, fmt.Errorf("failed to get offline list: %w", err) + } + res = append(res, resp.Tasks...) + return res, nil +} + +func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + _, err := xc.Request(TASK_API_URL, http.MethodDelete, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(map[string]string{ + "task_ids": strings.Join(taskIDs, ","), + "delete_files": strconv.FormatBool(deleteFiles), + }) + }, nil) + if err != nil { + return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) + } + return nil +} + +func (xc *XunLeiCommon) CoreLogin(username string, password string) (sessionID string, err error) { + url := XLUSER_API_BASE_URL + "/xluser.core.login/v3/login" + var resp CoreLoginResp + res, err := xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetHeader("User-Agent", "android-ok-http-client/xl-acc-sdk/version-5.0.12.512000") + req.SetBody(&CoreLoginRequest{ + ProtocolVersion: "301", + SequenceNo: "1000012", + PlatformVersion: "10", + IsCompressed: "0", + Appid: APPID, + ClientVersion: "8.31.0.9726", + PeerID: "00000000000000000000000000000000", + AppName: "ANDROID-com.xunlei.downloadprovider", + SdkVersion: "512000", + Devicesign: generateDeviceSign(xc.DeviceID, xc.PackageName), + NetWorkType: "WIFI", + ProviderName: "NONE", + DeviceModel: "M2004J7AC", + DeviceName: "Xiaomi_M2004j7ac", + OSVersion: "12", + Creditkey: xc.GetCreditKey(), + Hl: "zh-CN", + UserName: username, + PassWord: password, + VerifyKey: "", + VerifyCode: "", + IsMd5Pwd: "0", + }) + }, nil) + if err != nil { + return "", err + } + + if err = utils.Json.Unmarshal(res, &resp); err != nil { + return "", err + } + + xc.SetCoreTokenResp(&resp) + + sessionID = resp.SessionID + + return sessionID, nil +} diff --git a/drivers/thunder/meta.go b/drivers/thunder/meta.go index 12b01cbaa16..5e6e251387e 100644 --- a/drivers/thunder/meta.go +++ b/drivers/thunder/meta.go @@ -23,23 +23,25 @@ type ExpertAddition struct { RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` // 签名方法1 - Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"HPxr4BVygTQVtQkIMwQH33ywbgYG5l4JoR,GzhNkZ8pOBsCY+7,v+l0ImTpG7c7/,e5ztohgVXNP,t,EbXUWyVVqQbQX39Mbjn2geok3/0WEkAVxeqhtx857++kjJiRheP8l77gO,o7dvYgbRMOpHXxCs,6MW8TD8DphmakaxCqVrfv7NReRRN7ck3KLnXBculD58MvxjFRqT+,kmo0HxCKVfmxoZswLB4bVA/dwqbVAYghSb,j,4scKJNdd7F27Hv7tbt"` + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"9uJNVj/wLmdwKrJaVj/omlQ,Oz64Lp0GigmChHMf/6TNfxx7O9PyopcczMsnf,Eb+L7Ce+Ej48u,jKY0,ASr0zCl6v8W4aidjPK5KHd1Lq3t+vBFf41dqv5+fnOd,wQlozdg6r1qxh0eRmt3QgNXOvSZO6q/GXK,gmirk+ciAvIgA/cxUUCema47jr/YToixTT+Q6O,5IiCoM9B1/788ntB,P07JH0h6qoM6TSUAK2aL9T5s2QBVeY9JWvalf,+oK0AN"` // 签名方法2 CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` // 验证码 CaptchaToken string `json:"captcha_token"` + // 信任密钥 + CreditKey string `json:"credit_key" help:"credit key,used for login"` // 必要且影响登录,由签名决定 - DeviceID string `json:"device_id" required:"true" default:"9aa5c268e7bcfc197a9ad88e2fb330e5"` + DeviceID string `json:"device_id" default:""` ClientID string `json:"client_id" required:"true" default:"Xp6vsxz_7IYVw2BB"` ClientSecret string `json:"client_secret" required:"true" default:"Xp6vsy4tN9toTVdMSpomVdXpRmES"` - ClientVersion string `json:"client_version" required:"true" default:"7.51.0.8196"` + ClientVersion string `json:"client_version" required:"true" default:"8.31.0.9726"` PackageName string `json:"package_name" required:"true" default:"com.xunlei.downloadprovider"` //不影响登录,影响下载速度 - UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.xunlei.downloadprovider/7.51.0.8196 netWorkType/4G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/220200 Oauth2Client/0.9 (Linux 4_14_186-perf-gdcf98eab238b) (JAVA 0)"` + UserAgent string `json:"user_agent" required:"true" default:"ANDROID-com.xunlei.downloadprovider/8.31.0.9726 netWorkType/5G appid/40 deviceName/Xiaomi_M2004j7ac deviceModel/M2004J7AC OSVersion/12 protocolVersion/301 platformVersion/10 sdkVersion/512000 Oauth2Client/0.9 (Linux 4_14_186-perf-gddfs8vbb238b) (JAVA 0)"` DownloadUserAgent string `json:"download_user_agent" required:"true" default:"Dalvik/2.1.0 (Linux; U; Android 12; M2004J7AC Build/SP1A.210812.016)"` //优先使用视频链接代替下载链接 @@ -74,6 +76,10 @@ type Addition struct { Username string `json:"username" required:"true"` Password string `json:"password" required:"true"` CaptchaToken string `json:"captcha_token"` + // 信任密钥 + CreditKey string `json:"credit_key" help:"credit key,used for login"` + // 登录设备ID + DeviceID string `json:"device_id" default:""` } // 登录特征,用于判断是否重新登录 diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index 7c223673448..1fe8432c2bb 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -18,6 +18,10 @@ type ErrResp struct { } func (e *ErrResp) IsError() bool { + if e.ErrorMsg == "success" { + return false + } + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" } @@ -61,13 +65,79 @@ func (t *TokenResp) Token() string { } type SignInRequest struct { - CaptchaToken string `json:"captcha_token"` - ClientID string `json:"client_id"` ClientSecret string `json:"client_secret"` - Username string `json:"username"` - Password string `json:"password"` + Provider string `json:"provider"` + SigninToken string `json:"signin_token"` +} + +type CoreLoginRequest struct { + ProtocolVersion string `json:"protocolVersion"` + SequenceNo string `json:"sequenceNo"` + PlatformVersion string `json:"platformVersion"` + IsCompressed string `json:"isCompressed"` + Appid string `json:"appid"` + ClientVersion string `json:"clientVersion"` + PeerID string `json:"peerID"` + AppName string `json:"appName"` + SdkVersion string `json:"sdkVersion"` + Devicesign string `json:"devicesign"` + NetWorkType string `json:"netWorkType"` + ProviderName string `json:"providerName"` + DeviceModel string `json:"deviceModel"` + DeviceName string `json:"deviceName"` + OSVersion string `json:"OSVersion"` + Creditkey string `json:"creditkey"` + Hl string `json:"hl"` + UserName string `json:"userName"` + PassWord string `json:"passWord"` + VerifyKey string `json:"verifyKey"` + VerifyCode string `json:"verifyCode"` + IsMd5Pwd string `json:"isMd5Pwd"` +} + +type CoreLoginResp struct { + Account string `json:"account"` + Creditkey string `json:"creditkey"` + /* Error string `json:"error"` + ErrorCode string `json:"errorCode"` + ErrorDescription string `json:"error_description"`*/ + ExpiresIn int `json:"expires_in"` + IsCompressed string `json:"isCompressed"` + IsSetPassWord string `json:"isSetPassWord"` + KeepAliveMinPeriod string `json:"keepAliveMinPeriod"` + KeepAlivePeriod string `json:"keepAlivePeriod"` + LoginKey string `json:"loginKey"` + NickName string `json:"nickName"` + PlatformVersion string `json:"platformVersion"` + ProtocolVersion string `json:"protocolVersion"` + SecureKey string `json:"secureKey"` + SequenceNo string `json:"sequenceNo"` + SessionID string `json:"sessionID"` + Timestamp string `json:"timestamp"` + UserID string `json:"userID"` + UserName string `json:"userName"` + UserNewNo string `json:"userNewNo"` + Version string `json:"version"` + /* VipList []struct { + ExpireDate string `json:"expireDate"` + IsAutoDeduct string `json:"isAutoDeduct"` + IsVip string `json:"isVip"` + IsYear string `json:"isYear"` + PayID string `json:"payId"` + PayName string `json:"payName"` + Register string `json:"register"` + Vasid string `json:"vasid"` + VasType string `json:"vasType"` + VipDayGrow string `json:"vipDayGrow"` + VipGrow string `json:"vipGrow"` + VipLevel string `json:"vipLevel"` + Icon struct { + General string `json:"general"` + Small string `json:"small"` + } `json:"icon"` + } `json:"vipList"`*/ } /* @@ -204,3 +274,76 @@ type UploadTaskResponse struct { File Files `json:"file"` } + +// 添加离线下载响应 +type OfflineDownloadResp struct { + File *string `json:"file"` + Task OfflineTask `json:"task"` + UploadType string `json:"upload_type"` + URL struct { + Kind string `json:"kind"` + } `json:"url"` +} + +// 离线下载列表 +type OfflineListResp struct { + ExpiresIn int64 `json:"expires_in"` + NextPageToken string `json:"next_page_token"` + Tasks []OfflineTask `json:"tasks"` +} + +// offlineTask +type OfflineTask struct { + Callback string `json:"callback"` + CreatedTime string `json:"created_time"` + FileID string `json:"file_id"` + FileName string `json:"file_name"` + FileSize string `json:"file_size"` + IconLink string `json:"icon_link"` + ID string `json:"id"` + Kind string `json:"kind"` + Message string `json:"message"` + Name string `json:"name"` + Params Params `json:"params"` + Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING + Progress int64 `json:"progress"` + Space string `json:"space"` + StatusSize int64 `json:"status_size"` + Statuses []string `json:"statuses"` + ThirdTaskID string `json:"third_task_id"` + Type string `json:"type"` + UpdatedTime string `json:"updated_time"` + UserID string `json:"user_id"` +} + +type Params struct { + FolderType string `json:"folder_type"` + PredictSpeed string `json:"predict_speed"` + PredictType string `json:"predict_type"` +} + +// LoginReviewResp 登录验证响应 +type LoginReviewResp struct { + Creditkey string `json:"creditkey"` + Error string `json:"error"` + ErrorCode string `json:"errorCode"` + ErrorDesc string `json:"errorDesc"` + ErrorDescURL string `json:"errorDescUrl"` + ErrorIsRetry int `json:"errorIsRetry"` + ErrorDescription string `json:"error_description"` + IsCompressed string `json:"isCompressed"` + PlatformVersion string `json:"platformVersion"` + ProtocolVersion string `json:"protocolVersion"` + Reviewurl string `json:"reviewurl"` + SequenceNo string `json:"sequenceNo"` + UserID string `json:"userID"` + VerifyType string `json:"verifyType"` +} + +// ReviewData 验证数据 +type ReviewData struct { + Creditkey string `json:"creditkey"` + Reviewurl string `json:"reviewurl"` + Deviceid string `json:"deviceid"` + Devicesign string `json:"devicesign"` +} diff --git a/drivers/thunder/util.go b/drivers/thunder/util.go index f6dec3260cf..b7afe56debc 100644 --- a/drivers/thunder/util.go +++ b/drivers/thunder/util.go @@ -1,8 +1,10 @@ package thunder import ( + "crypto/md5" "crypto/sha1" "encoding/hex" + "encoding/json" "fmt" "io" "net/http" @@ -15,9 +17,11 @@ import ( ) const ( - API_URL = "https://api-pan.xunlei.com/drive/v1" - FILE_API_URL = API_URL + "/files" - XLUSER_API_URL = "https://xluser-ssl.xunlei.com/v1" + API_URL = "https://api-pan.xunlei.com/drive/v1" + FILE_API_URL = API_URL + "/files" + TASK_API_URL = API_URL + "/tasks" + XLUSER_API_BASE_URL = "https://xluser-ssl.xunlei.com" + XLUSER_API_URL = XLUSER_API_BASE_URL + "/v1" ) const ( @@ -33,6 +37,12 @@ const ( UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" ) +const ( + SignProvider = "access_end_point_token" + APPID = "40" + APPKey = "34a062aaa22f906fca4fefe9fb3a3021" +) + func GetAction(method string, url string) string { urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] return method + ":" + urlpath @@ -43,6 +53,8 @@ type Common struct { captchaToken string + creditKey string + // 签名相关,二选一 Algorithms []string Timestamp, CaptchaSign string @@ -68,6 +80,13 @@ func (c *Common) GetCaptchaToken() string { return c.captchaToken } +func (c *Common) SetCreditKey(creditKey string) { + c.creditKey = creditKey +} +func (c *Common) GetCreditKey() string { + return c.creditKey +} + // 刷新验证码token(登录后) func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { metas := map[string]string{ @@ -169,12 +188,53 @@ func (c *Common) Request(url, method string, callback base.ReqCallback, resp int var erron ErrResp utils.Json.Unmarshal(res.Body(), &erron) if erron.IsError() { + // review_panel 表示需要短信验证码进行验证 + if erron.ErrorMsg == "review_panel" { + return nil, c.getReviewData(res) + } + return nil, &erron } return res.Body(), nil } +// 获取验证所需内容 +func (c *Common) getReviewData(res *resty.Response) error { + var reviewResp LoginReviewResp + var reviewData ReviewData + + if err := utils.Json.Unmarshal(res.Body(), &reviewResp); err != nil { + return err + } + + deviceSign := generateDeviceSign(c.DeviceID, c.PackageName) + + reviewData = ReviewData{ + Creditkey: reviewResp.Creditkey, + Reviewurl: reviewResp.Reviewurl + "&deviceid=" + deviceSign, + Deviceid: deviceSign, + Devicesign: deviceSign, + } + + // 将reviewData转为JSON字符串 + reviewDataJSON, _ := json.MarshalIndent(reviewData, "", " ") + //reviewDataJSON, _ := json.Marshal(reviewData) + + return fmt.Errorf(` +
+ 🔒 本次登录需要验证
+ This login requires verification + +

下面是验证所需要的数据,具体使用方法请参照对应的驱动文档
+ Below are the relevant verification data. For specific usage methods, please refer to the corresponding driver documentation.

+
+
%s
+
+
`, string(reviewDataJSON)) +} + // 计算文件Gcid func getGcid(r io.Reader, size int64) (string, error) { calcBlockSize := func(j int64) int64 { @@ -190,7 +250,7 @@ func getGcid(r io.Reader, size int64) (string, error) { readSize := calcBlockSize(size) for { hash2.Reset() - if n, err := io.CopyN(hash2, r, readSize); err != nil && n == 0 { + if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { if err != io.EOF { return "", err } @@ -200,3 +260,24 @@ func getGcid(r io.Reader, size int64) (string, error) { } return hex.EncodeToString(hash1.Sum(nil)), nil } + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, APPID, APPKey) + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} diff --git a/drivers/thunder_browser/driver.go b/drivers/thunder_browser/driver.go new file mode 100644 index 00000000000..0b38d07714f --- /dev/null +++ b/drivers/thunder_browser/driver.go @@ -0,0 +1,698 @@ +package thunder_browser + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + streamPkg "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/go-resty/resty/v2" +) + +type ThunderBrowser struct { + *XunLeiBrowserCommon + model.Storage + Addition + + identity string +} + +func (x *ThunderBrowser) Config() driver.Config { + return config +} + +func (x *ThunderBrowser) GetAddition() driver.Additional { + return &x.Addition +} + +func (x *ThunderBrowser) Init(ctx context.Context) (err error) { + + spaceTokenFunc := func() error { + // 如果用户未设置 "超级保险柜" 密码 则直接返回 + if x.SafePassword == "" { + return nil + } + // 通过 GetSafeAccessToken 获取 + token, err := x.GetSafeAccessToken(x.SafePassword) + x.SetSpaceTokenResp(token) + return err + } + + // 初始化所需参数 + if x.XunLeiBrowserCommon == nil { + x.XunLeiBrowserCommon = &XunLeiBrowserCommon{ + Common: &Common{ + client: base.NewRestyClient(), + Algorithms: Algorithms, + DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), + ClientID: ClientID, + ClientSecret: ClientSecret, + ClientVersion: ClientVersion, + PackageName: PackageName, + UserAgent: BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), PackageName, SdkVersion, ClientVersion, PackageName), + DownloadUserAgent: DownloadUserAgent, + UseVideoUrl: x.UseVideoUrl, + RemoveWay: x.Addition.RemoveWay, + refreshCTokenCk: func(token string) { + x.CaptchaToken = token + op.MustSaveDriverStorage(x) + }, + }, + refreshTokenFunc: func() error { + // 通过RefreshToken刷新 + token, err := x.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + // 重新登录 + token, err = x.Login(x.Username, x.Password) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + op.MustSaveDriverStorage(x) + } + } + x.SetTokenResp(token) + return err + }, + } + } + + // 自定义验证码token + ctoekn := strings.TrimSpace(x.CaptchaToken) + if ctoekn != "" { + x.SetCaptchaToken(ctoekn) + } + if x.DeviceID == "" { + x.SetDeviceID(utils.GetMD5EncodeStr(x.Username + x.Password)) + } + x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl + x.Addition.RootFolderID = x.RootFolderID + // 防止重复登录 + identity := x.GetIdentity() + if x.identity != identity || !x.IsLogin() { + x.identity = identity + // 登录 + token, err := x.Login(x.Username, x.Password) + if err != nil { + return err + } + x.SetTokenResp(token) + } + + // 获取 spaceToken + err = spaceTokenFunc() + if err != nil { + return err + } + + return nil +} + +func (x *ThunderBrowser) Drop(ctx context.Context) error { + return nil +} + +type ThunderBrowserExpert struct { + *XunLeiBrowserCommon + model.Storage + ExpertAddition + + identity string +} + +func (x *ThunderBrowserExpert) Config() driver.Config { + return configExpert +} + +func (x *ThunderBrowserExpert) GetAddition() driver.Additional { + return &x.ExpertAddition +} + +func (x *ThunderBrowserExpert) Init(ctx context.Context) (err error) { + + spaceTokenFunc := func() error { + // 如果用户未设置 "超级保险柜" 密码 则直接返回 + if x.SafePassword == "" { + return nil + } + // 通过 GetSafeAccessToken 获取 + token, err := x.GetSafeAccessToken(x.SafePassword) + x.SetSpaceTokenResp(token) + return err + } + + // 防止重复登录 + identity := x.GetIdentity() + if identity != x.identity || !x.IsLogin() { + x.identity = identity + x.XunLeiBrowserCommon = &XunLeiBrowserCommon{ + Common: &Common{ + client: base.NewRestyClient(), + DeviceID: func() string { + if len(x.DeviceID) != 32 { + if x.LoginType == "user" { + return utils.GetMD5EncodeStr(x.Username + x.Password) + } + return utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken) + } + return x.DeviceID + }(), + ClientID: x.ClientID, + ClientSecret: x.ClientSecret, + ClientVersion: x.ClientVersion, + PackageName: x.PackageName, + UserAgent: func() string { + if x.ExpertAddition.UserAgent != "" { + return x.ExpertAddition.UserAgent + } + if x.LoginType == "user" { + return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), x.PackageName, SdkVersion, x.ClientVersion, x.PackageName) + } + return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken), x.PackageName, SdkVersion, x.ClientVersion, x.PackageName) + }(), + DownloadUserAgent: func() string { + if x.ExpertAddition.DownloadUserAgent != "" { + return x.ExpertAddition.DownloadUserAgent + } + return DownloadUserAgent + }(), + UseVideoUrl: x.UseVideoUrl, + RemoveWay: x.ExpertAddition.RemoveWay, + refreshCTokenCk: func(token string) { + x.CaptchaToken = token + op.MustSaveDriverStorage(x) + }, + }, + } + + if x.ExpertAddition.CaptchaToken != "" { + x.SetCaptchaToken(x.ExpertAddition.CaptchaToken) + op.MustSaveDriverStorage(x) + } + if x.Common.DeviceID != "" { + x.ExpertAddition.DeviceID = x.Common.DeviceID + op.MustSaveDriverStorage(x) + } + if x.Common.UserAgent != "" { + x.ExpertAddition.UserAgent = x.Common.UserAgent + op.MustSaveDriverStorage(x) + } + if x.Common.DownloadUserAgent != "" { + x.ExpertAddition.DownloadUserAgent = x.Common.DownloadUserAgent + op.MustSaveDriverStorage(x) + } + x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl + x.ExpertAddition.RootFolderID = x.RootFolderID + // 签名方法 + if x.SignType == "captcha_sign" { + x.Common.Timestamp = x.Timestamp + x.Common.CaptchaSign = x.CaptchaSign + } else { + x.Common.Algorithms = strings.Split(x.Algorithms, ",") + } + + // 登录方式 + if x.LoginType == "refresh_token" { + // 通过RefreshToken登录 + token, err := x.XunLeiBrowserCommon.RefreshToken(x.ExpertAddition.RefreshToken) + if err != nil { + return err + } + x.SetTokenResp(token) + + // 刷新token方法 + x.SetRefreshTokenFunc(func() error { + token, err := x.XunLeiBrowserCommon.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + } + x.SetTokenResp(token) + op.MustSaveDriverStorage(x) + return err + }) + + err = spaceTokenFunc() + if err != nil { + return err + } + + } else { + // 通过用户密码登录 + token, err := x.Login(x.Username, x.Password) + if err != nil { + return err + } + x.SetTokenResp(token) + x.SetRefreshTokenFunc(func() error { + token, err := x.XunLeiBrowserCommon.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + token, err = x.Login(x.Username, x.Password) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + } + } + x.SetTokenResp(token) + op.MustSaveDriverStorage(x) + return err + }) + + err = spaceTokenFunc() + if err != nil { + return err + } + } + } else { + // 仅修改验证码token + if x.CaptchaToken != "" { + x.SetCaptchaToken(x.CaptchaToken) + } + + err = spaceTokenFunc() + if err != nil { + return err + } + + x.XunLeiBrowserCommon.UserAgent = x.UserAgent + x.XunLeiBrowserCommon.DownloadUserAgent = x.DownloadUserAgent + x.XunLeiBrowserCommon.UseVideoUrl = x.UseVideoUrl + x.ExpertAddition.RootFolderID = x.RootFolderID + } + + return nil +} + +func (x *ThunderBrowserExpert) Drop(ctx context.Context) error { + return nil +} + +func (x *ThunderBrowserExpert) SetTokenResp(token *TokenResp) { + x.XunLeiBrowserCommon.SetTokenResp(token) + if token != nil { + x.ExpertAddition.RefreshToken = token.RefreshToken + } +} + +type XunLeiBrowserCommon struct { + *Common + *TokenResp // 登录信息 + + refreshTokenFunc func() error +} + +func (xc *XunLeiBrowserCommon) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return xc.getFiles(ctx, dir, args.ReqPath) +} + +func (xc *XunLeiBrowserCommon) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var lFile Files + + params := map[string]string{ + "_magic": "2021", + "space": file.(*Files).GetSpace(), + "thumbnail_size": "SIZE_LARGE", + "with": "url", + } + + _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", file.GetID()) + r.SetQueryParams(params) + //r.SetQueryParam("space", "") + }, &lFile) + if err != nil { + return nil, err + } + link := &model.Link{ + URL: lFile.WebContentLink, + Header: http.Header{ + "User-Agent": {xc.DownloadUserAgent}, + }, + } + + if xc.UseVideoUrl { + for _, media := range lFile.Medias { + if media.Link.URL != "" { + link.URL = media.Link.URL + break + } + } + } + return link, nil +} + +func (xc *XunLeiBrowserCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + js := base.Json{ + "kind": FOLDER, + "name": dirName, + "parent_id": parentDir.GetID(), + "space": parentDir.(*Files).GetSpace(), + } + + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err +} + +func (xc *XunLeiBrowserCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + + params := map[string]string{ + "_from": srcObj.(*Files).GetSpace(), + } + js := base.Json{ + "to": base.Json{"parent_id": dstDir.GetID(), "space": dstDir.(*Files).GetSpace()}, + "space": srcObj.(*Files).GetSpace(), + "ids": []string{srcObj.GetID()}, + } + + _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + r.SetQueryParams(params) + }, nil) + return err +} + +func (xc *XunLeiBrowserCommon) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + + params := map[string]string{ + "space": srcObj.(*Files).GetSpace(), + } + + _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", srcObj.GetID()) + r.SetBody(&base.Json{"name": newName}) + r.SetQueryParams(params) + }, nil) + return err +} + +func (xc *XunLeiBrowserCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + + params := map[string]string{ + "_from": srcObj.(*Files).GetSpace(), + } + js := base.Json{ + "to": base.Json{"parent_id": dstDir.GetID(), "space": dstDir.(*Files).GetSpace()}, + "space": srcObj.(*Files).GetSpace(), + "ids": []string{srcObj.GetID()}, + } + + _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + r.SetQueryParams(params) + }, nil) + return err +} + +func (xc *XunLeiBrowserCommon) Remove(ctx context.Context, obj model.Obj) error { + + js := base.Json{ + "ids": []string{obj.GetID()}, + "space": obj.(*Files).GetSpace(), + } + // 先判断是否是特殊情况 + if obj.(*Files).GetSpace() == ThunderDriveSpace { + _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", obj.GetID()) + r.SetBody("{}") + }, nil) + return err + } else if obj.(*Files).GetSpace() == ThunderBrowserDriveSafeSpace || obj.(*Files).GetSpace() == ThunderDriveSafeSpace { + _, err := xc.Request(FILE_API_URL+":batchDelete", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err + } + + // 根据用户选择的删除方式进行删除 + if xc.RemoveWay == "delete" { + _, err := xc.Request(FILE_API_URL+":batchDelete", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err + } else { + _, err := xc.Request(FILE_API_URL+":batchTrash", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, nil) + return err + } +} + +func (xc *XunLeiBrowserCommon) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + gcid := stream.GetHash().GetHash(hash_extend.GCID) + var err error + if len(gcid) < hash_extend.GCID.Width { + _, gcid, err = streamPkg.CacheFullInTempFileAndHash(stream, hash_extend.GCID, stream.GetSize()) + if err != nil { + return err + } + } + + js := base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": stream.GetName(), + "size": stream.GetSize(), + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + "space": dstDir.(*Files).GetSpace(), + } + + var resp UploadTaskResponse + _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&js) + }, &resp) + if err != nil { + return err + } + + param := resp.Resumable.Params + if resp.UploadType == UPLOAD_TYPE_RESUMABLE { + param.Endpoint = strings.TrimLeft(param.Endpoint, param.Bucket+".") + s, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(param.AccessKeyID, param.AccessKeySecret, param.SecurityToken), + Region: aws.String("xunlei"), + Endpoint: aws.String(param.Endpoint), + }) + if err != nil { + return err + } + uploader := s3manager.NewUploader(s) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } + _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(param.Bucket), + Key: aws.String(param.Key), + Expires: aws.Time(param.Expiration), + Body: driver.NewLimitedUploadStream(ctx, io.TeeReader(stream, driver.NewProgress(stream.GetSize(), up))), + }) + return err + } + return nil +} + +func (xc *XunLeiBrowserCommon) getFiles(ctx context.Context, dir model.Obj, path string) ([]model.Obj, error) { + files := make([]model.Obj, 0) + var pageToken string + for { + var fileList FileList + folderSpace := "" + switch dirF := dir.(type) { + case *Files: + folderSpace = dirF.GetSpace() + default: + // 处理 根目录的情况 + folderSpace = ThunderBrowserDriveSpace + } + params := map[string]string{ + "parent_id": dir.GetID(), + "page_token": pageToken, + "space": folderSpace, + "filters": `{"trashed":{"eq":false}}`, + "with": "url", + "with_audit": "true", + "thumbnail_size": "SIZE_LARGE", + } + + _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + r.SetQueryParams(params) + }, &fileList) + if err != nil { + return nil, err + } + + for i := range fileList.Files { + // 解决 "迅雷云盘" 重复出现问题————迅雷后端发送错误 + if fileList.Files[i].FolderType == ThunderDriveFolderType && fileList.Files[i].ID == "" && fileList.Files[i].Space == "" && dir.GetID() != "" { + continue + } + files = append(files, &fileList.Files[i]) + } + + if fileList.NextPageToken == "" { + break + } + pageToken = fileList.NextPageToken + } + return files, nil +} + +// SetRefreshTokenFunc 设置刷新Token的方法 +func (xc *XunLeiBrowserCommon) SetRefreshTokenFunc(fn func() error) { + xc.refreshTokenFunc = fn +} + +// SetTokenResp 设置Token +func (xc *XunLeiBrowserCommon) SetTokenResp(tr *TokenResp) { + xc.TokenResp = tr +} + +// SetSpaceTokenResp 设置Token +func (xc *XunLeiBrowserCommon) SetSpaceTokenResp(spaceToken string) { + xc.TokenResp.Token = spaceToken +} + +// Request 携带Authorization和CaptchaToken的请求 +func (xc *XunLeiBrowserCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + data, err := xc.Common.Request(url, method, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Authorization": xc.GetToken(), + "X-Captcha-Token": xc.GetCaptchaToken(), + "X-Space-Authorization": xc.GetSpaceToken(), + }) + if callback != nil { + callback(req) + } + }, resp) + + errResp, ok := err.(*ErrResp) + if !ok { + return nil, err + } + + switch errResp.ErrorCode { + case 0: + return data, nil + case 4122, 4121, 10, 16: + if xc.refreshTokenFunc != nil { + if err = xc.refreshTokenFunc(); err == nil { + break + } + } + return nil, err + case 9: + // space_token 获取失败 + if errResp.ErrorMsg == "space_token_invalid" { + if token, err := xc.GetSafeAccessToken(xc.Token); err != nil { + return nil, err + } else { + xc.SetSpaceTokenResp(token) + } + + } + if errResp.ErrorMsg == "captcha_invalid" { + // 验证码token过期 + if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.UserID); err != nil { + return nil, err + } + } + return nil, err + default: + return nil, err + } + return xc.Request(url, method, callback, resp) +} + +// RefreshToken 刷新Token +func (xc *XunLeiBrowserCommon) RefreshToken(refreshToken string) (*TokenResp, error) { + var resp TokenResp + _, err := xc.Common.Request(XLUSER_API_URL+"/auth/token", http.MethodPost, func(req *resty.Request) { + req.SetBody(&base.Json{ + "grant_type": "refresh_token", + "refresh_token": refreshToken, + "client_id": xc.ClientID, + "client_secret": xc.ClientSecret, + }) + }, &resp) + if err != nil { + return nil, err + } + + if resp.RefreshToken == "" { + return nil, errors.New("refresh token is empty") + } + return &resp, nil +} + +// GetSafeAccessToken 获取 超级保险柜 AccessToken +func (xc *XunLeiBrowserCommon) GetSafeAccessToken(safePassword string) (string, error) { + var resp TokenResp + _, err := xc.Request(XLUSER_API_URL+"/password/check", http.MethodPost, func(req *resty.Request) { + req.SetBody(&base.Json{ + "scene": "box", + "password": EncryptPassword(safePassword), + }) + }, &resp) + if err != nil { + return "", err + } + + if resp.Token == "" { + return "", errors.New("SafePassword is incorrect ") + } + return resp.Token, nil +} + +// Login 登录 +func (xc *XunLeiBrowserCommon) Login(username, password string) (*TokenResp, error) { + url := XLUSER_API_URL + "/auth/signin" + err := xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username) + if err != nil { + return nil, err + } + + var resp TokenResp + _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(&SignInRequest{ + CaptchaToken: xc.GetCaptchaToken(), + ClientID: xc.ClientID, + ClientSecret: xc.ClientSecret, + Username: username, + Password: password, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (xc *XunLeiBrowserCommon) IsLogin() bool { + if xc.TokenResp == nil { + return false + } + _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) + return err == nil +} diff --git a/drivers/thunder_browser/meta.go b/drivers/thunder_browser/meta.go new file mode 100644 index 00000000000..f535ea6cd47 --- /dev/null +++ b/drivers/thunder_browser/meta.go @@ -0,0 +1,108 @@ +package thunder_browser + +import ( + "crypto/md5" + "encoding/hex" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// ExpertAddition 高级设置 +type ExpertAddition struct { + driver.RootID + + LoginType string `json:"login_type" type:"select" options:"user,refresh_token" default:"user"` + SignType string `json:"sign_type" type:"select" options:"algorithms,captcha_sign" default:"algorithms"` + + // 登录方式1 + Username string `json:"username" required:"true" help:"login type is user,this is required"` + Password string `json:"password" required:"true" help:"login type is user,this is required"` + // 登录方式2 + RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` + + SafePassword string `json:"safe_password" required:"true" help:"super safe password"` // 超级保险箱密码 + + // 签名方法1 + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"uWRwO7gPfdPB/0NfPtfQO+71,F93x+qPluYy6jdgNpq+lwdH1ap6WOM+nfz8/V,0HbpxvpXFsBK5CoTKam,dQhzbhzFRcawnsZqRETT9AuPAJ+wTQso82mRv,SAH98AmLZLRa6DB2u68sGhyiDh15guJpXhBzI,unqfo7Z64Rie9RNHMOB,7yxUdFADp3DOBvXdz0DPuKNVT35wqa5z0DEyEvf,RBG,ThTWPG5eC0UBqlbQ+04nZAptqGCdpv9o55A"` + // 签名方法2 + CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` + Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` + + // 验证码 + CaptchaToken string `json:"captcha_token"` + + // 必要且影响登录,由签名决定 + DeviceID string `json:"device_id" required:"false" default:""` + ClientID string `json:"client_id" required:"true" default:"ZUBzD9J_XPXfn7f7"` + ClientSecret string `json:"client_secret" required:"true" default:"yESVmHecEe6F0aou69vl-g"` + ClientVersion string `json:"client_version" required:"true" default:"1.10.0.2633"` + PackageName string `json:"package_name" required:"true" default:"com.xunlei.browser"` + + // 不影响登录,影响下载速度 + UserAgent string `json:"user_agent" required:"false" default:""` + DownloadUserAgent string `json:"download_user_agent" required:"false" default:""` + + // 优先使用视频链接代替下载链接 + UseVideoUrl bool `json:"use_video_url"` + // 移除方式 + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` +} + +// GetIdentity 登录特征,用于判断是否重新登录 +func (i *ExpertAddition) GetIdentity() string { + hash := md5.New() + if i.LoginType == "refresh_token" { + hash.Write([]byte(i.RefreshToken)) + } else { + hash.Write([]byte(i.Username + i.Password)) + } + + if i.SignType == "captcha_sign" { + hash.Write([]byte(i.CaptchaSign + i.Timestamp)) + } else { + hash.Write([]byte(i.Algorithms)) + } + + hash.Write([]byte(i.DeviceID)) + hash.Write([]byte(i.ClientID)) + hash.Write([]byte(i.ClientSecret)) + hash.Write([]byte(i.ClientVersion)) + hash.Write([]byte(i.PackageName)) + return hex.EncodeToString(hash.Sum(nil)) +} + +type Addition struct { + driver.RootID + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + SafePassword string `json:"safe_password" required:"true"` // 超级保险箱密码 + CaptchaToken string `json:"captcha_token"` + UseVideoUrl bool `json:"use_video_url" default:"false"` + RemoveWay string `json:"remove_way" required:"true" type:"select" options:"trash,delete"` +} + +// GetIdentity 登录特征,用于判断是否重新登录 +func (i *Addition) GetIdentity() string { + return utils.GetMD5EncodeStr(i.Username + i.Password) +} + +var config = driver.Config{ + Name: "ThunderBrowser", + LocalSort: true, +} + +var configExpert = driver.Config{ + Name: "ThunderBrowserExpert", + LocalSort: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ThunderBrowser{} + }) + op.RegisterDriver(func() driver.Driver { + return &ThunderBrowserExpert{} + }) +} diff --git a/drivers/thunder_browser/types.go b/drivers/thunder_browser/types.go new file mode 100644 index 00000000000..b3e21d2bc08 --- /dev/null +++ b/drivers/thunder_browser/types.go @@ -0,0 +1,236 @@ +package thunder_browser + +import ( + "fmt" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" +) + +type ErrResp struct { + ErrorCode int64 `json:"error_code"` + ErrorMsg string `json:"error"` + ErrorDescription string `json:"error_description"` + // ErrorDetails interface{} `json:"error_details"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) +} + +/* +* 验证码Token +**/ +type CaptchaTokenRequest struct { + Action string `json:"action"` + CaptchaToken string `json:"captcha_token"` + ClientID string `json:"client_id"` + DeviceID string `json:"device_id"` + Meta map[string]string `json:"meta"` + RedirectUri string `json:"redirect_uri"` +} + +type CaptchaTokenResponse struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Url string `json:"url"` +} + +/* +* 登录 +**/ +type TokenResp struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int64 `json:"expires_in"` + + Sub string `json:"sub"` + UserID string `json:"user_id"` + + Token string `json:"token"` // "超级保险箱" 访问Token +} + +func (t *TokenResp) GetToken() string { + return fmt.Sprint(t.TokenType, " ", t.AccessToken) +} + +// GetSpaceToken 获取"超级保险箱" 访问Token +func (t *TokenResp) GetSpaceToken() string { + return t.Token +} + +type SignInRequest struct { + CaptchaToken string `json:"captcha_token"` + + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + + Username string `json:"username"` + Password string `json:"password"` +} + +/* +* 文件 +**/ +type FileList struct { + Kind string `json:"kind"` + NextPageToken string `json:"next_page_token"` + Files []Files `json:"files"` + Version string `json:"version"` + VersionOutdated bool `json:"version_outdated"` + FolderType int8 +} + +type Link struct { + URL string `json:"url"` + Token string `json:"token"` + Expire time.Time `json:"expire"` + Type string `json:"type"` +} + +var _ model.Obj = (*Files)(nil) + +type Files struct { + Kind string `json:"kind"` + ID string `json:"id"` + ParentID string `json:"parent_id"` + Name string `json:"name"` + //UserID string `json:"user_id"` + Size string `json:"size"` + //Revision string `json:"revision"` + //FileExtension string `json:"file_extension"` + //MimeType string `json:"mime_type"` + //Starred bool `json:"starred"` + WebContentLink string `json:"web_content_link"` + CreatedTime CustomTime `json:"created_time"` + ModifiedTime CustomTime `json:"modified_time"` + IconLink string `json:"icon_link"` + ThumbnailLink string `json:"thumbnail_link"` + Md5Checksum string `json:"md5_checksum"` + Hash string `json:"hash"` + // Links map[string]Link `json:"links"` + // Phase string `json:"phase"` + // Audit struct { + // Status string `json:"status"` + // Message string `json:"message"` + // Title string `json:"title"` + // } `json:"audit"` + Medias []struct { + //Category string `json:"category"` + //IconLink string `json:"icon_link"` + //IsDefault bool `json:"is_default"` + //IsOrigin bool `json:"is_origin"` + //IsVisible bool `json:"is_visible"` + Link Link `json:"link"` + //MediaID string `json:"media_id"` + //MediaName string `json:"media_name"` + //NeedMoreQuota bool `json:"need_more_quota"` + //Priority int `json:"priority"` + //RedirectLink string `json:"redirect_link"` + //ResolutionName string `json:"resolution_name"` + // Video struct { + // AudioCodec string `json:"audio_codec"` + // BitRate int `json:"bit_rate"` + // Duration int `json:"duration"` + // FrameRate int `json:"frame_rate"` + // Height int `json:"height"` + // VideoCodec string `json:"video_codec"` + // VideoType string `json:"video_type"` + // Width int `json:"width"` + // } `json:"video"` + // VipTypes []string `json:"vip_types"` + } `json:"medias"` + Trashed bool `json:"trashed"` + DeleteTime string `json:"delete_time"` + OriginalURL string `json:"original_url"` + //Params struct{} `json:"params"` + //OriginalFileIndex int `json:"original_file_index"` + Space string `json:"space"` + //Apps []interface{} `json:"apps"` + //Writable bool `json:"writable"` + FolderType string `json:"folder_type"` + //Collection interface{} `json:"collection"` + SortName string `json:"sort_name"` + UserModifiedTime CustomTime `json:"user_modified_time"` + //SpellName []interface{} `json:"spell_name"` + //FileCategory string `json:"file_category"` + //Tags []interface{} `json:"tags"` + //ReferenceEvents []interface{} `json:"reference_events"` + //ReferenceResource interface{} `json:"reference_resource"` + //Params0 struct { + // PlatformIcon string `json:"platform_icon"` + // SmallThumbnail string `json:"small_thumbnail"` + //} `json:"params,omitempty"` +} + +func (c *Files) GetHash() utils.HashInfo { + return utils.NewHashInfo(hash_extend.GCID, c.Hash) +} + +func (c *Files) GetSize() int64 { size, _ := strconv.ParseInt(c.Size, 10, 64); return size } +func (c *Files) GetName() string { return c.Name } +func (c *Files) CreateTime() time.Time { return c.CreatedTime.Time } +func (c *Files) ModTime() time.Time { return c.ModifiedTime.Time } +func (c *Files) IsDir() bool { return c.Kind == FOLDER } +func (c *Files) GetID() string { return c.ID } +func (c *Files) GetPath() string { + return "" +} +func (c *Files) Thumb() string { return c.ThumbnailLink } + +func (c *Files) GetSpace() string { + if c.Space != "" { + return c.Space + } else { + // "迅雷云盘" 文件夹内 Space 为空 + return "" + } +} + +/* +* 上传 +**/ +type UploadTaskResponse struct { + UploadType string `json:"upload_type"` + + /*//UPLOAD_TYPE_FORM + Form struct { + //Headers struct{} `json:"headers"` + Kind string `json:"kind"` + Method string `json:"method"` + MultiParts struct { + OSSAccessKeyID string `json:"OSSAccessKeyId"` + Signature string `json:"Signature"` + Callback string `json:"callback"` + Key string `json:"key"` + Policy string `json:"policy"` + XUserData string `json:"x:user_data"` + } `json:"multi_parts"` + URL string `json:"url"` + } `json:"form"`*/ + + //UPLOAD_TYPE_RESUMABLE + Resumable struct { + Kind string `json:"kind"` + Params struct { + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + Bucket string `json:"bucket"` + Endpoint string `json:"endpoint"` + Expiration time.Time `json:"expiration"` + Key string `json:"key"` + SecurityToken string `json:"security_token"` + } `json:"params"` + Provider string `json:"provider"` + } `json:"resumable"` + + File Files `json:"file"` +} diff --git a/drivers/thunder_browser/util.go b/drivers/thunder_browser/util.go new file mode 100644 index 00000000000..befd1a904c8 --- /dev/null +++ b/drivers/thunder_browser/util.go @@ -0,0 +1,311 @@ +package thunder_browser + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + API_URL = "https://x-api-pan.xunlei.com/drive/v1" + FILE_API_URL = API_URL + "/files" + XLUSER_API_URL = "https://xluser-ssl.xunlei.com/v1" +) + +var Algorithms = []string{ + "uWRwO7gPfdPB/0NfPtfQO+71", + "F93x+qPluYy6jdgNpq+lwdH1ap6WOM+nfz8/V", + "0HbpxvpXFsBK5CoTKam", + "dQhzbhzFRcawnsZqRETT9AuPAJ+wTQso82mRv", + "SAH98AmLZLRa6DB2u68sGhyiDh15guJpXhBzI", + "unqfo7Z64Rie9RNHMOB", + "7yxUdFADp3DOBvXdz0DPuKNVT35wqa5z0DEyEvf", + "RBG", + "ThTWPG5eC0UBqlbQ+04nZAptqGCdpv9o55A", +} + +const ( + ClientID = "ZUBzD9J_XPXfn7f7" + ClientSecret = "yESVmHecEe6F0aou69vl-g" + ClientVersion = "1.10.0.2633" + PackageName = "com.xunlei.browser" + DownloadUserAgent = "AndroidDownloadManager/13 (Linux; U; Android 13; M2004J7AC Build/SP1A.210812.016)" + SdkVersion = "233100" +) + +const ( + FOLDER = "drive#folder" + FILE = "drive#file" + RESUMABLE = "drive#resumable" +) + +const ( + UPLOAD_TYPE_UNKNOWN = "UPLOAD_TYPE_UNKNOWN" + //UPLOAD_TYPE_FORM = "UPLOAD_TYPE_FORM" + UPLOAD_TYPE_RESUMABLE = "UPLOAD_TYPE_RESUMABLE" + UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" +) + +const ( + ThunderDriveSpace = "" + ThunderDriveSafeSpace = "SPACE_SAFE" + ThunderBrowserDriveSpace = "SPACE_BROWSER" + ThunderBrowserDriveSafeSpace = "SPACE_BROWSER_SAFE" + ThunderDriveFolderType = "DEFAULT_ROOT" + ThunderBrowserDriveSafeFolderType = "BROWSER_SAFE" +) + +func GetAction(method string, url string) string { + urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] + return method + ":" + urlpath +} + +type Common struct { + client *resty.Client + + captchaToken string + + // 签名相关,二选一 + Algorithms []string + Timestamp, CaptchaSign string + + // 必要值,签名相关 + DeviceID string + ClientID string + ClientSecret string + ClientVersion string + PackageName string + UserAgent string + DownloadUserAgent string + UseVideoUrl bool + RemoveWay string + + // 验证码token刷新成功回调 + refreshCTokenCk func(token string) +} + +func (c *Common) SetDeviceID(deviceID string) { + c.DeviceID = deviceID +} + +func (c *Common) SetCaptchaToken(captchaToken string) { + c.captchaToken = captchaToken +} +func (c *Common) GetCaptchaToken() string { + return c.captchaToken +} + +// RefreshCaptchaTokenAtLogin 刷新验证码token(登录后) +func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { + metas := map[string]string{ + "client_version": c.ClientVersion, + "package_name": c.PackageName, + "user_id": userID, + } + metas["timestamp"], metas["captcha_sign"] = c.GetCaptchaSign() + return c.refreshCaptchaToken(action, metas) +} + +// RefreshCaptchaTokenInLogin 刷新验证码token(登录时) +func (c *Common) RefreshCaptchaTokenInLogin(action, username string) error { + metas := make(map[string]string) + if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { + metas["email"] = username + } else if len(username) >= 11 && len(username) <= 18 { + metas["phone_number"] = username + } else { + metas["username"] = username + } + return c.refreshCaptchaToken(action, metas) +} + +// GetCaptchaSign 获取验证码签名 +func (c *Common) GetCaptchaSign() (timestamp, sign string) { + if len(c.Algorithms) == 0 { + return c.Timestamp, c.CaptchaSign + } + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) + for _, algorithm := range c.Algorithms { + str = utils.GetMD5EncodeStr(str + algorithm) + } + sign = "1." + str + return +} + +// 刷新验证码token +func (c *Common) refreshCaptchaToken(action string, metas map[string]string) error { + param := CaptchaTokenRequest{ + Action: action, + CaptchaToken: c.captchaToken, + ClientID: c.ClientID, + DeviceID: c.DeviceID, + Meta: metas, + RedirectUri: "xlaccsdk01://xunlei.com/callback?state=harbor", + } + var e ErrResp + var resp CaptchaTokenResponse + _, err := c.Request(XLUSER_API_URL+"/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + req.SetError(&e).SetBody(param) + }, &resp) + + if err != nil { + return err + } + + if e.IsError() { + return &e + } + + if resp.Url != "" { + return fmt.Errorf(`need verify: Click Here`, resp.Url) + } + + if resp.CaptchaToken == "" { + return fmt.Errorf("empty captchaToken") + } + + if c.refreshCTokenCk != nil { + c.refreshCTokenCk(resp.CaptchaToken) + } + c.SetCaptchaToken(resp.CaptchaToken) + return nil +} + +// Request 只有基础信息的请求 +func (c *Common) Request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := c.client.R().SetHeaders(map[string]string{ + "user-agent": c.UserAgent, + "accept": "application/json;charset=UTF-8", + "x-device-id": c.DeviceID, + "x-client-id": c.ClientID, + "x-client-version": c.ClientVersion, + }) + + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + + var erron ErrResp + utils.Json.Unmarshal(res.Body(), &erron) + if erron.IsError() { + return nil, &erron + } + + return res.Body(), nil +} + +// 计算文件Gcid +func getGcid(r io.Reader, size int64) (string, error) { + calcBlockSize := func(j int64) int64 { + var psize int64 = 0x40000 + for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { + psize = psize << 1 + } + return psize + } + + hash1 := sha1.New() + hash2 := sha1.New() + readSize := calcBlockSize(size) + for { + hash2.Reset() + if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { + if err != io.EOF { + return "", err + } + break + } + hash1.Write(hash2.Sum(nil)) + } + return hex.EncodeToString(hash1.Sum(nil)), nil +} + +type CustomTime struct { + time.Time +} + +const timeFormat = time.RFC3339 + +func (ct *CustomTime) UnmarshalJSON(b []byte) error { + str := string(b) + if str == `""` { + *ct = CustomTime{Time: time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)} + return nil + } + + t, err := time.Parse(`"`+timeFormat+`"`, str) + if err != nil { + return err + } + *ct = CustomTime{Time: t} + return nil +} + +// EncryptPassword 超级保险箱 加密 +func EncryptPassword(password string) string { + if password == "" { + return "" + } + // 将字符串转换为字节数组 + byteData := []byte(password) + // 计算MD5哈希值 + hash := md5.Sum(byteData) + // 将哈希值转换为十六进制字符串 + return hex.EncodeToString(hash[:]) +} + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "22062", "a5d7416858147a4ab99573872ffccef8") + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} + +func BuildCustomUserAgent(deviceID, appName, sdkVersion, clientVersion, packageName string) string { + //deviceSign := generateDeviceSign(deviceID, packageName) + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) + sb.WriteString("networkType/WIFI ") + sb.WriteString(fmt.Sprintf("appid/%s ", "22062")) + sb.WriteString(fmt.Sprintf("deviceName/Xiaomi_M2004j7ac ")) + sb.WriteString(fmt.Sprintf("deviceModel/M2004J7AC ")) + sb.WriteString(fmt.Sprintf("OSVersion/13 ")) + sb.WriteString(fmt.Sprintf("protocolVersion/301 ")) + sb.WriteString(fmt.Sprintf("platformversion/10 ")) + sb.WriteString(fmt.Sprintf("sdkVersion/%s ", sdkVersion)) + sb.WriteString(fmt.Sprintf("Oauth2Client/0.9 (Linux 4_9_337-perf-sn-uotan-gd9d488809c3d) (JAVA 0) ")) + return sb.String() +} diff --git a/drivers/thunderx/driver.go b/drivers/thunderx/driver.go new file mode 100644 index 00000000000..6ee8901a4fc --- /dev/null +++ b/drivers/thunderx/driver.go @@ -0,0 +1,558 @@ +package thunderx + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/go-resty/resty/v2" +) + +type ThunderX struct { + *XunLeiXCommon + model.Storage + Addition + + identity string +} + +func (x *ThunderX) Config() driver.Config { + return config +} + +func (x *ThunderX) GetAddition() driver.Additional { + return &x.Addition +} + +func (x *ThunderX) Init(ctx context.Context) (err error) { + // 初始化所需参数 + if x.XunLeiXCommon == nil { + x.XunLeiXCommon = &XunLeiXCommon{ + Common: &Common{ + client: base.NewRestyClient(), + Algorithms: Algorithms, + DeviceID: utils.GetMD5EncodeStr(x.Username + x.Password), + ClientID: ClientID, + ClientSecret: ClientSecret, + ClientVersion: ClientVersion, + PackageName: PackageName, + UserAgent: BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, ""), + DownloadUserAgent: DownloadUserAgent, + UseVideoUrl: x.UseVideoUrl, + + refreshCTokenCk: func(token string) { + x.CaptchaToken = token + op.MustSaveDriverStorage(x) + }, + }, + refreshTokenFunc: func() error { + // 通过RefreshToken刷新 + token, err := x.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + // 重新登录 + token, err = x.Login(x.Username, x.Password) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + if token.UserID != "" { + x.SetUserID(token.UserID) + x.UserAgent = BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, token.UserID) + } + op.MustSaveDriverStorage(x) + } + } + x.SetTokenResp(token) + return err + }, + } + } + + // 自定义验证码token + ctoken := strings.TrimSpace(x.CaptchaToken) + if ctoken != "" { + x.SetCaptchaToken(ctoken) + } + if x.DeviceID == "" { + x.SetDeviceID(utils.GetMD5EncodeStr(x.Username + x.Password)) + } + + x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl + x.Addition.RootFolderID = x.RootFolderID + // 防止重复登录 + identity := x.GetIdentity() + if x.identity != identity || !x.IsLogin() { + x.identity = identity + // 登录 + token, err := x.Login(x.Username, x.Password) + if err != nil { + return err + } + x.SetTokenResp(token) + if token.UserID != "" { + x.SetUserID(token.UserID) + x.UserAgent = BuildCustomUserAgent(x.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, token.UserID) + } + } + return nil +} + +func (x *ThunderX) Drop(ctx context.Context) error { + return nil +} + +type ThunderXExpert struct { + *XunLeiXCommon + model.Storage + ExpertAddition + + identity string +} + +func (x *ThunderXExpert) Config() driver.Config { + return configExpert +} + +func (x *ThunderXExpert) GetAddition() driver.Additional { + return &x.ExpertAddition +} + +func (x *ThunderXExpert) Init(ctx context.Context) (err error) { + // 防止重复登录 + identity := x.GetIdentity() + if identity != x.identity || !x.IsLogin() { + x.identity = identity + x.XunLeiXCommon = &XunLeiXCommon{ + Common: &Common{ + client: base.NewRestyClient(), + + DeviceID: func() string { + if len(x.DeviceID) != 32 { + if x.LoginType == "user" { + return utils.GetMD5EncodeStr(x.Username + x.Password) + } + return utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken) + } + return x.DeviceID + }(), + ClientID: x.ClientID, + ClientSecret: x.ClientSecret, + ClientVersion: x.ClientVersion, + PackageName: x.PackageName, + UserAgent: func() string { + if x.ExpertAddition.UserAgent != "" { + return x.ExpertAddition.UserAgent + } + if x.LoginType == "user" { + return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.Username+x.Password), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, "") + } + return BuildCustomUserAgent(utils.GetMD5EncodeStr(x.ExpertAddition.RefreshToken), ClientID, PackageName, SdkVersion, ClientVersion, PackageName, "") + }(), + DownloadUserAgent: func() string { + if x.ExpertAddition.DownloadUserAgent != "" { + return x.ExpertAddition.DownloadUserAgent + } + return DownloadUserAgent + }(), + UseVideoUrl: x.UseVideoUrl, + refreshCTokenCk: func(token string) { + x.CaptchaToken = token + op.MustSaveDriverStorage(x) + }, + }, + } + + if x.ExpertAddition.CaptchaToken != "" { + x.SetCaptchaToken(x.ExpertAddition.CaptchaToken) + op.MustSaveDriverStorage(x) + } + if x.Common.DeviceID != "" { + x.ExpertAddition.DeviceID = x.Common.DeviceID + op.MustSaveDriverStorage(x) + } + if x.Common.DownloadUserAgent != "" { + x.ExpertAddition.DownloadUserAgent = x.Common.DownloadUserAgent + op.MustSaveDriverStorage(x) + } + x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl + x.ExpertAddition.RootFolderID = x.RootFolderID + // 签名方法 + if x.SignType == "captcha_sign" { + x.Common.Timestamp = x.Timestamp + x.Common.CaptchaSign = x.CaptchaSign + } else { + x.Common.Algorithms = strings.Split(x.Algorithms, ",") + } + + // 登录方式 + if x.LoginType == "refresh_token" { + // 通过RefreshToken登录 + token, err := x.XunLeiXCommon.RefreshToken(x.ExpertAddition.RefreshToken) + if err != nil { + return err + } + x.SetTokenResp(token) + // 刷新token方法 + x.SetRefreshTokenFunc(func() error { + token, err := x.XunLeiXCommon.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + } + x.SetTokenResp(token) + op.MustSaveDriverStorage(x) + return err + }) + } else { + // 通过用户密码登录 + token, err := x.Login(x.Username, x.Password) + if err != nil { + return err + } + x.SetTokenResp(token) + x.SetRefreshTokenFunc(func() error { + token, err := x.XunLeiXCommon.RefreshToken(x.TokenResp.RefreshToken) + if err != nil { + token, err = x.Login(x.Username, x.Password) + if err != nil { + x.GetStorage().SetStatus(fmt.Sprintf("%+v", err.Error())) + } + } + x.SetTokenResp(token) + op.MustSaveDriverStorage(x) + return err + }) + } + // 更新 UserAgent + if x.TokenResp.UserID != "" { + x.ExpertAddition.UserAgent = BuildCustomUserAgent(x.ExpertAddition.DeviceID, ClientID, PackageName, SdkVersion, ClientVersion, PackageName, x.TokenResp.UserID) + x.SetUserAgent(x.ExpertAddition.UserAgent) + op.MustSaveDriverStorage(x) + } + } else { + // 仅修改验证码token + if x.CaptchaToken != "" { + x.SetCaptchaToken(x.CaptchaToken) + } + x.XunLeiXCommon.UserAgent = x.ExpertAddition.UserAgent + x.XunLeiXCommon.DownloadUserAgent = x.ExpertAddition.UserAgent + x.XunLeiXCommon.UseVideoUrl = x.UseVideoUrl + x.ExpertAddition.RootFolderID = x.RootFolderID + } + return nil +} + +func (x *ThunderXExpert) Drop(ctx context.Context) error { + return nil +} + +func (x *ThunderXExpert) SetTokenResp(token *TokenResp) { + x.XunLeiXCommon.SetTokenResp(token) + if token != nil { + x.ExpertAddition.RefreshToken = token.RefreshToken + } +} + +type XunLeiXCommon struct { + *Common + *TokenResp // 登录信息 + + refreshTokenFunc func() error +} + +func (xc *XunLeiXCommon) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + return xc.getFiles(ctx, dir.GetID()) +} + +func (xc *XunLeiXCommon) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + var lFile Files + _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", file.GetID()) + //r.SetQueryParam("space", "") + }, &lFile) + if err != nil { + return nil, err + } + link := &model.Link{ + URL: lFile.WebContentLink, + Header: http.Header{ + "User-Agent": {xc.DownloadUserAgent}, + }, + } + + if xc.UseVideoUrl { + for _, media := range lFile.Medias { + if media.Link.URL != "" { + link.URL = media.Link.URL + break + } + } + } + + /* + strs := regexp.MustCompile(`e=([0-9]*)`).FindStringSubmatch(lFile.WebContentLink) + if len(strs) == 2 { + timestamp, err := strconv.ParseInt(strs[1], 10, 64) + if err == nil { + expired := time.Duration(timestamp-time.Now().Unix()) * time.Second + link.Expiration = &expired + } + } + */ + return link, nil +} + +func (xc *XunLeiXCommon) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FOLDER, + "name": dirName, + "parent_id": parentDir.GetID(), + }) + }, nil) + return err +} + +func (xc *XunLeiXCommon) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + _, err := xc.Request(FILE_API_URL+":batchMove", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + }) + }, nil) + return err +} + +func (xc *XunLeiXCommon) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + _, err := xc.Request(FILE_API_URL+"/{fileID}", http.MethodPatch, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", srcObj.GetID()) + r.SetBody(&base.Json{"name": newName}) + }, nil) + return err +} + +func (xc *XunLeiXCommon) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + _, err := xc.Request(FILE_API_URL+":batchCopy", http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "to": base.Json{"parent_id": dstDir.GetID()}, + "ids": []string{srcObj.GetID()}, + }) + }, nil) + return err +} + +func (xc *XunLeiXCommon) Remove(ctx context.Context, obj model.Obj) error { + _, err := xc.Request(FILE_API_URL+"/{fileID}/trash", http.MethodPatch, func(r *resty.Request) { + r.SetContext(ctx) + r.SetPathParam("fileID", obj.GetID()) + r.SetBody("{}") + }, nil) + return err +} + +func (xc *XunLeiXCommon) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + gcid := file.GetHash().GetHash(hash_extend.GCID) + var err error + if len(gcid) < hash_extend.GCID.Width { + _, gcid, err = stream.CacheFullInTempFileAndHash(file, hash_extend.GCID, file.GetSize()) + if err != nil { + return err + } + } + + var resp UploadTaskResponse + _, err = xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FILE, + "parent_id": dstDir.GetID(), + "name": file.GetName(), + "size": file.GetSize(), + "hash": gcid, + "upload_type": UPLOAD_TYPE_RESUMABLE, + }) + }, &resp) + if err != nil { + return err + } + + param := resp.Resumable.Params + if resp.UploadType == UPLOAD_TYPE_RESUMABLE { + param.Endpoint = strings.TrimLeft(param.Endpoint, param.Bucket+".") + s, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(param.AccessKeyID, param.AccessKeySecret, param.SecurityToken), + Region: aws.String("xunlei"), + Endpoint: aws.String(param.Endpoint), + }) + if err != nil { + return err + } + uploader := s3manager.NewUploader(s) + if file.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = file.GetSize() / (s3manager.MaxUploadParts - 1) + } + _, err = uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(param.Bucket), + Key: aws.String(param.Key), + Expires: aws.Time(param.Expiration), + Body: driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: file, + UpdateProgress: up, + }), + }) + return err + } + return nil +} + +func (xc *XunLeiXCommon) getFiles(ctx context.Context, folderId string) ([]model.Obj, error) { + files := make([]model.Obj, 0) + var pageToken string + for { + var fileList FileList + _, err := xc.Request(FILE_API_URL, http.MethodGet, func(r *resty.Request) { + r.SetContext(ctx) + r.SetQueryParams(map[string]string{ + "space": "", + "__type": "drive", + "refresh": "true", + "__sync": "true", + "parent_id": folderId, + "page_token": pageToken, + "with_audit": "true", + "limit": "100", + "filters": `{"phase":{"eq":"PHASE_TYPE_COMPLETE"},"trashed":{"eq":false}}`, + }) + }, &fileList) + if err != nil { + return nil, err + } + + for i := 0; i < len(fileList.Files); i++ { + files = append(files, &fileList.Files[i]) + } + + if fileList.NextPageToken == "" { + break + } + pageToken = fileList.NextPageToken + } + return files, nil +} + +// SetRefreshTokenFunc 设置刷新Token的方法 +func (xc *XunLeiXCommon) SetRefreshTokenFunc(fn func() error) { + xc.refreshTokenFunc = fn +} + +// SetTokenResp 设置Token +func (xc *XunLeiXCommon) SetTokenResp(tr *TokenResp) { + xc.TokenResp = tr +} + +// Request 携带Authorization和CaptchaToken的请求 +func (xc *XunLeiXCommon) Request(url string, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + data, err := xc.Common.Request(url, method, func(req *resty.Request) { + req.SetHeaders(map[string]string{ + "Authorization": xc.Token(), + "X-Captcha-Token": xc.GetCaptchaToken(), + }) + if callback != nil { + callback(req) + } + }, resp) + + errResp, ok := err.(*ErrResp) + if !ok { + return nil, err + } + + switch errResp.ErrorCode { + case 0: + return data, nil + case 4122, 4121, 10, 16: + if xc.refreshTokenFunc != nil { + if err = xc.refreshTokenFunc(); err == nil { + break + } + } + return nil, err + case 9: // 验证码token过期 + if err = xc.RefreshCaptchaTokenAtLogin(GetAction(method, url), xc.UserID); err != nil { + return nil, err + } + default: + return nil, err + } + return xc.Request(url, method, callback, resp) +} + +// RefreshToken 刷新Token +func (xc *XunLeiXCommon) RefreshToken(refreshToken string) (*TokenResp, error) { + var resp TokenResp + _, err := xc.Common.Request(XLUSER_API_URL+"/auth/token", http.MethodPost, func(req *resty.Request) { + req.SetBody(&base.Json{ + "grant_type": "refresh_token", + "refresh_token": refreshToken, + "client_id": xc.ClientID, + "client_secret": xc.ClientSecret, + }) + }, &resp) + if err != nil { + return nil, err + } + + if resp.RefreshToken == "" { + return nil, errs.EmptyToken + } + resp.UserID = resp.Sub + return &resp, nil +} + +// Login 登录 +func (xc *XunLeiXCommon) Login(username, password string) (*TokenResp, error) { + url := XLUSER_API_URL + "/auth/signin" + err := xc.RefreshCaptchaTokenInLogin(GetAction(http.MethodPost, url), username) + if err != nil { + return nil, err + } + + var resp TokenResp + _, err = xc.Common.Request(url, http.MethodPost, func(req *resty.Request) { + req.SetBody(&SignInRequest{ + CaptchaToken: xc.GetCaptchaToken(), + ClientID: xc.ClientID, + ClientSecret: xc.ClientSecret, + Username: username, + Password: password, + }) + }, &resp) + if err != nil { + return nil, err + } + resp.UserID = resp.Sub + return &resp, nil +} + +func (xc *XunLeiXCommon) IsLogin() bool { + if xc.TokenResp == nil { + return false + } + _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) + return err == nil +} diff --git a/drivers/thunderx/meta.go b/drivers/thunderx/meta.go new file mode 100644 index 00000000000..fa60ebbdb0a --- /dev/null +++ b/drivers/thunderx/meta.go @@ -0,0 +1,103 @@ +package thunderx + +import ( + "crypto/md5" + "encoding/hex" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// 高级设置 +type ExpertAddition struct { + driver.RootID + + LoginType string `json:"login_type" type:"select" options:"user,refresh_token" default:"user"` + SignType string `json:"sign_type" type:"select" options:"algorithms,captcha_sign" default:"algorithms"` + + // 登录方式1 + Username string `json:"username" required:"true" help:"login type is user,this is required"` + Password string `json:"password" required:"true" help:"login type is user,this is required"` + // 登录方式2 + RefreshToken string `json:"refresh_token" required:"true" help:"login type is refresh_token,this is required"` + + // 签名方法1 + Algorithms string `json:"algorithms" required:"true" help:"sign type is algorithms,this is required" default:"kVy0WbPhiE4v6oxXZ88DvoA3Q,lON/AUoZKj8/nBtcE85mVbkOaVdVa,rLGffQrfBKH0BgwQ33yZofvO3Or,FO6HWqw,GbgvyA2,L1NU9QvIQIH7DTRt,y7llk4Y8WfYflt6,iuDp1WPbV3HRZudZtoXChxH4HNVBX5ZALe,8C28RTXmVcco0,X5Xh,7xe25YUgfGgD0xW3ezFS,,CKCR,8EmDjBo6h3eLaK7U6vU2Qys0NsMx,t2TeZBXKqbdP09Arh9C3"` + // 签名方法2 + CaptchaSign string `json:"captcha_sign" required:"true" help:"sign type is captcha_sign,this is required"` + Timestamp string `json:"timestamp" required:"true" help:"sign type is captcha_sign,this is required"` + + // 验证码 + CaptchaToken string `json:"captcha_token"` + + // 必要且影响登录,由签名决定 + DeviceID string `json:"device_id" required:"false" default:""` + ClientID string `json:"client_id" required:"true" default:"ZQL_zwA4qhHcoe_2"` + ClientSecret string `json:"client_secret" required:"true" default:"Og9Vr1L8Ee6bh0olFxFDRg"` + ClientVersion string `json:"client_version" required:"true" default:"1.06.0.2132"` + PackageName string `json:"package_name" required:"true" default:"com.thunder.downloader"` + + ////不影响登录,影响下载速度 + UserAgent string `json:"user_agent" required:"false" default:""` + DownloadUserAgent string `json:"download_user_agent" required:"false" default:""` + + //优先使用视频链接代替下载链接 + UseVideoUrl bool `json:"use_video_url"` +} + +// 登录特征,用于判断是否重新登录 +func (i *ExpertAddition) GetIdentity() string { + hash := md5.New() + if i.LoginType == "refresh_token" { + hash.Write([]byte(i.RefreshToken)) + } else { + hash.Write([]byte(i.Username + i.Password)) + } + + if i.SignType == "captcha_sign" { + hash.Write([]byte(i.CaptchaSign + i.Timestamp)) + } else { + hash.Write([]byte(i.Algorithms)) + } + + hash.Write([]byte(i.DeviceID)) + hash.Write([]byte(i.ClientID)) + hash.Write([]byte(i.ClientSecret)) + hash.Write([]byte(i.ClientVersion)) + hash.Write([]byte(i.PackageName)) + return hex.EncodeToString(hash.Sum(nil)) +} + +type Addition struct { + driver.RootID + Username string `json:"username" required:"true"` + Password string `json:"password" required:"true"` + CaptchaToken string `json:"captcha_token"` + UseVideoUrl bool `json:"use_video_url" default:"true"` +} + +// 登录特征,用于判断是否重新登录 +func (i *Addition) GetIdentity() string { + return utils.GetMD5EncodeStr(i.Username + i.Password) +} + +var config = driver.Config{ + Name: "ThunderX", + LocalSort: true, + OnlyProxy: false, +} + +var configExpert = driver.Config{ + Name: "ThunderXExpert", + LocalSort: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &ThunderX{} + }) + op.RegisterDriver(func() driver.Driver { + return &ThunderXExpert{} + }) +} diff --git a/drivers/thunderx/types.go b/drivers/thunderx/types.go new file mode 100644 index 00000000000..77cfa0f2415 --- /dev/null +++ b/drivers/thunderx/types.go @@ -0,0 +1,206 @@ +package thunderx + +import ( + "fmt" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + hash_extend "github.com/alist-org/alist/v3/pkg/utils/hash" +) + +type ErrResp struct { + ErrorCode int64 `json:"error_code"` + ErrorMsg string `json:"error"` + ErrorDescription string `json:"error_description"` + // ErrorDetails interface{} `json:"error_details"` +} + +func (e *ErrResp) IsError() bool { + return e.ErrorCode != 0 || e.ErrorMsg != "" || e.ErrorDescription != "" +} + +func (e *ErrResp) Error() string { + return fmt.Sprintf("ErrorCode: %d ,Error: %s ,ErrorDescription: %s ", e.ErrorCode, e.ErrorMsg, e.ErrorDescription) +} + +/* +* 验证码Token +**/ +type CaptchaTokenRequest struct { + Action string `json:"action"` + CaptchaToken string `json:"captcha_token"` + ClientID string `json:"client_id"` + DeviceID string `json:"device_id"` + Meta map[string]string `json:"meta"` + RedirectUri string `json:"redirect_uri"` +} + +type CaptchaTokenResponse struct { + CaptchaToken string `json:"captcha_token"` + ExpiresIn int64 `json:"expires_in"` + Url string `json:"url"` +} + +/* +* 登录 +**/ +type TokenResp struct { + TokenType string `json:"token_type"` + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + ExpiresIn int64 `json:"expires_in"` + + Sub string `json:"sub"` + UserID string `json:"user_id"` +} + +func (t *TokenResp) Token() string { + return fmt.Sprint(t.TokenType, " ", t.AccessToken) +} + +type SignInRequest struct { + CaptchaToken string `json:"captcha_token"` + + ClientID string `json:"client_id"` + ClientSecret string `json:"client_secret"` + + Username string `json:"username"` + Password string `json:"password"` +} + +/* +* 文件 +**/ +type FileList struct { + Kind string `json:"kind"` + NextPageToken string `json:"next_page_token"` + Files []Files `json:"files"` + Version string `json:"version"` + VersionOutdated bool `json:"version_outdated"` +} + +type Link struct { + URL string `json:"url"` + Token string `json:"token"` + Expire time.Time `json:"expire"` + Type string `json:"type"` +} + +var _ model.Obj = (*Files)(nil) + +type Files struct { + Kind string `json:"kind"` + ID string `json:"id"` + ParentID string `json:"parent_id"` + Name string `json:"name"` + //UserID string `json:"user_id"` + Size string `json:"size"` + //Revision string `json:"revision"` + //FileExtension string `json:"file_extension"` + //MimeType string `json:"mime_type"` + //Starred bool `json:"starred"` + WebContentLink string `json:"web_content_link"` + CreatedTime time.Time `json:"created_time"` + ModifiedTime time.Time `json:"modified_time"` + IconLink string `json:"icon_link"` + ThumbnailLink string `json:"thumbnail_link"` + // Md5Checksum string `json:"md5_checksum"` + Hash string `json:"hash"` + // Links map[string]Link `json:"links"` + // Phase string `json:"phase"` + // Audit struct { + // Status string `json:"status"` + // Message string `json:"message"` + // Title string `json:"title"` + // } `json:"audit"` + Medias []struct { + //Category string `json:"category"` + //IconLink string `json:"icon_link"` + //IsDefault bool `json:"is_default"` + //IsOrigin bool `json:"is_origin"` + //IsVisible bool `json:"is_visible"` + Link Link `json:"link"` + //MediaID string `json:"media_id"` + //MediaName string `json:"media_name"` + //NeedMoreQuota bool `json:"need_more_quota"` + //Priority int `json:"priority"` + //RedirectLink string `json:"redirect_link"` + //ResolutionName string `json:"resolution_name"` + // Video struct { + // AudioCodec string `json:"audio_codec"` + // BitRate int `json:"bit_rate"` + // Duration int `json:"duration"` + // FrameRate int `json:"frame_rate"` + // Height int `json:"height"` + // VideoCodec string `json:"video_codec"` + // VideoType string `json:"video_type"` + // Width int `json:"width"` + // } `json:"video"` + // VipTypes []string `json:"vip_types"` + } `json:"medias"` + Trashed bool `json:"trashed"` + DeleteTime string `json:"delete_time"` + OriginalURL string `json:"original_url"` + //Params struct{} `json:"params"` + //OriginalFileIndex int `json:"original_file_index"` + //Space string `json:"space"` + //Apps []interface{} `json:"apps"` + //Writable bool `json:"writable"` + //FolderType string `json:"folder_type"` + //Collection interface{} `json:"collection"` +} + +func (c *Files) GetHash() utils.HashInfo { + return utils.NewHashInfo(hash_extend.GCID, c.Hash) +} + +func (c *Files) GetSize() int64 { size, _ := strconv.ParseInt(c.Size, 10, 64); return size } +func (c *Files) GetName() string { return c.Name } +func (c *Files) CreateTime() time.Time { return c.CreatedTime } +func (c *Files) ModTime() time.Time { return c.ModifiedTime } +func (c *Files) IsDir() bool { return c.Kind == FOLDER } +func (c *Files) GetID() string { return c.ID } +func (c *Files) GetPath() string { return "" } +func (c *Files) Thumb() string { return c.ThumbnailLink } + +/* +* 上传 +**/ +type UploadTaskResponse struct { + UploadType string `json:"upload_type"` + + /*//UPLOAD_TYPE_FORM + Form struct { + //Headers struct{} `json:"headers"` + Kind string `json:"kind"` + Method string `json:"method"` + MultiParts struct { + OSSAccessKeyID string `json:"OSSAccessKeyId"` + Signature string `json:"Signature"` + Callback string `json:"callback"` + Key string `json:"key"` + Policy string `json:"policy"` + XUserData string `json:"x:user_data"` + } `json:"multi_parts"` + URL string `json:"url"` + } `json:"form"`*/ + + //UPLOAD_TYPE_RESUMABLE + Resumable struct { + Kind string `json:"kind"` + Params struct { + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + Bucket string `json:"bucket"` + Endpoint string `json:"endpoint"` + Expiration time.Time `json:"expiration"` + Key string `json:"key"` + SecurityToken string `json:"security_token"` + } `json:"params"` + Provider string `json:"provider"` + } `json:"resumable"` + + File Files `json:"file"` +} diff --git a/drivers/thunderx/util.go b/drivers/thunderx/util.go new file mode 100644 index 00000000000..661da87e0b0 --- /dev/null +++ b/drivers/thunderx/util.go @@ -0,0 +1,297 @@ +package thunderx + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +const ( + API_URL = "https://api-pan.xunleix.com/drive/v1" + FILE_API_URL = API_URL + "/files" + XLUSER_API_URL = "https://xluser-ssl.xunleix.com/v1" +) + +var Algorithms = []string{ + "kVy0WbPhiE4v6oxXZ88DvoA3Q", + "lON/AUoZKj8/nBtcE85mVbkOaVdVa", + "rLGffQrfBKH0BgwQ33yZofvO3Or", + "FO6HWqw", + "GbgvyA2", + "L1NU9QvIQIH7DTRt", + "y7llk4Y8WfYflt6", + "iuDp1WPbV3HRZudZtoXChxH4HNVBX5ZALe", + "8C28RTXmVcco0", + "X5Xh", + "7xe25YUgfGgD0xW3ezFS", + "", + "CKCR", + "8EmDjBo6h3eLaK7U6vU2Qys0NsMx", + "t2TeZBXKqbdP09Arh9C3", +} + +const ( + ClientID = "ZQL_zwA4qhHcoe_2" + ClientSecret = "Og9Vr1L8Ee6bh0olFxFDRg" + ClientVersion = "1.06.0.2132" + PackageName = "com.thunder.downloader" + DownloadUserAgent = "Dalvik/2.1.0 (Linux; U; Android 13; M2004J7AC Build/SP1A.210812.016)" + SdkVersion = "2.0.3.203100 " +) + +const ( + FOLDER = "drive#folder" + FILE = "drive#file" + RESUMABLE = "drive#resumable" +) + +const ( + UPLOAD_TYPE_UNKNOWN = "UPLOAD_TYPE_UNKNOWN" + //UPLOAD_TYPE_FORM = "UPLOAD_TYPE_FORM" + UPLOAD_TYPE_RESUMABLE = "UPLOAD_TYPE_RESUMABLE" + UPLOAD_TYPE_URL = "UPLOAD_TYPE_URL" +) + +func GetAction(method string, url string) string { + urlpath := regexp.MustCompile(`://[^/]+((/[^/\s?#]+)*)`).FindStringSubmatch(url)[1] + return method + ":" + urlpath +} + +type Common struct { + client *resty.Client + + captchaToken string + userID string + // 签名相关,二选一 + Algorithms []string + Timestamp, CaptchaSign string + + // 必要值,签名相关 + DeviceID string + ClientID string + ClientSecret string + ClientVersion string + PackageName string + UserAgent string + DownloadUserAgent string + UseVideoUrl bool + + // 验证码token刷新成功回调 + refreshCTokenCk func(token string) +} + +func (c *Common) SetDeviceID(deviceID string) { + c.DeviceID = deviceID +} + +func (c *Common) SetUserID(userID string) { + c.userID = userID +} + +func (c *Common) SetUserAgent(userAgent string) { + c.UserAgent = userAgent +} + +func (c *Common) SetCaptchaToken(captchaToken string) { + c.captchaToken = captchaToken +} +func (c *Common) GetCaptchaToken() string { + return c.captchaToken +} + +// 刷新验证码token(登录后) +func (c *Common) RefreshCaptchaTokenAtLogin(action, userID string) error { + metas := map[string]string{ + "client_version": c.ClientVersion, + "package_name": c.PackageName, + "user_id": userID, + } + metas["timestamp"], metas["captcha_sign"] = c.GetCaptchaSign() + return c.refreshCaptchaToken(action, metas) +} + +// 刷新验证码token(登录时) +func (c *Common) RefreshCaptchaTokenInLogin(action, username string) error { + metas := make(map[string]string) + if ok, _ := regexp.MatchString(`\w+([-+.]\w+)*@\w+([-.]\w+)*\.\w+([-.]\w+)*`, username); ok { + metas["email"] = username + } else if len(username) >= 11 && len(username) <= 18 { + metas["phone_number"] = username + } else { + metas["username"] = username + } + return c.refreshCaptchaToken(action, metas) +} + +// 获取验证码签名 +func (c *Common) GetCaptchaSign() (timestamp, sign string) { + if len(c.Algorithms) == 0 { + return c.Timestamp, c.CaptchaSign + } + timestamp = fmt.Sprint(time.Now().UnixMilli()) + str := fmt.Sprint(c.ClientID, c.ClientVersion, c.PackageName, c.DeviceID, timestamp) + for _, algorithm := range c.Algorithms { + str = utils.GetMD5EncodeStr(str + algorithm) + } + sign = "1." + str + return +} + +// 刷新验证码token +func (c *Common) refreshCaptchaToken(action string, metas map[string]string) error { + param := CaptchaTokenRequest{ + Action: action, + CaptchaToken: c.captchaToken, + ClientID: c.ClientID, + DeviceID: c.DeviceID, + Meta: metas, + RedirectUri: "xlaccsdk01://xbase.cloud/callback?state=harbor", + } + var e ErrResp + var resp CaptchaTokenResponse + _, err := c.Request(XLUSER_API_URL+"/shield/captcha/init", http.MethodPost, func(req *resty.Request) { + req.SetError(&e).SetBody(param) + }, &resp) + + if err != nil { + return err + } + + if e.IsError() { + return &e + } + + if resp.Url != "" { + return fmt.Errorf(`need verify: Click Here`, resp.Url) + } + + if resp.CaptchaToken == "" { + return fmt.Errorf("empty captchaToken") + } + + if c.refreshCTokenCk != nil { + c.refreshCTokenCk(resp.CaptchaToken) + } + c.SetCaptchaToken(resp.CaptchaToken) + return nil +} + +// Request 只有基础信息的请求 +func (c *Common) Request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := c.client.R().SetHeaders(map[string]string{ + "user-agent": c.UserAgent, + "accept": "application/json;charset=UTF-8", + "x-device-id": c.DeviceID, + "x-client-id": c.ClientID, + "x-client-version": c.ClientVersion, + }) + + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + + var erron ErrResp + utils.Json.Unmarshal(res.Body(), &erron) + if erron.IsError() { + return nil, &erron + } + + return res.Body(), nil +} + +// 计算文件Gcid +func getGcid(r io.Reader, size int64) (string, error) { + calcBlockSize := func(j int64) int64 { + var psize int64 = 0x40000 + for float64(j)/float64(psize) > 0x200 && psize < 0x200000 { + psize = psize << 1 + } + return psize + } + + hash1 := sha1.New() + hash2 := sha1.New() + readSize := calcBlockSize(size) + for { + hash2.Reset() + if n, err := utils.CopyWithBufferN(hash2, r, readSize); err != nil && n == 0 { + if err != io.EOF { + return "", err + } + break + } + hash1.Write(hash2.Sum(nil)) + } + return hex.EncodeToString(hash1.Sum(nil)), nil +} + +func generateDeviceSign(deviceID, packageName string) string { + + signatureBase := fmt.Sprintf("%s%s%s%s", deviceID, packageName, "1", "appkey") + + sha1Hash := sha1.New() + sha1Hash.Write([]byte(signatureBase)) + sha1Result := sha1Hash.Sum(nil) + + sha1String := hex.EncodeToString(sha1Result) + + md5Hash := md5.New() + md5Hash.Write([]byte(sha1String)) + md5Result := md5Hash.Sum(nil) + + md5String := hex.EncodeToString(md5Result) + + deviceSign := fmt.Sprintf("div101.%s%s", deviceID, md5String) + + return deviceSign +} + +func BuildCustomUserAgent(deviceID, clientID, appName, sdkVersion, clientVersion, packageName, userID string) string { + deviceSign := generateDeviceSign(deviceID, packageName) + var sb strings.Builder + + sb.WriteString(fmt.Sprintf("ANDROID-%s/%s ", appName, clientVersion)) + sb.WriteString("protocolVersion/200 ") + sb.WriteString("accesstype/ ") + sb.WriteString(fmt.Sprintf("clientid/%s ", clientID)) + sb.WriteString(fmt.Sprintf("clientversion/%s ", clientVersion)) + sb.WriteString("action_type/ ") + sb.WriteString("networktype/WIFI ") + sb.WriteString("sessionid/ ") + sb.WriteString(fmt.Sprintf("deviceid/%s ", deviceID)) + sb.WriteString("providername/NONE ") + sb.WriteString(fmt.Sprintf("devicesign/%s ", deviceSign)) + sb.WriteString("refresh_token/ ") + sb.WriteString(fmt.Sprintf("sdkversion/%s ", sdkVersion)) + sb.WriteString(fmt.Sprintf("datetime/%d ", time.Now().UnixMilli())) + sb.WriteString(fmt.Sprintf("usrno/%s ", userID)) + sb.WriteString(fmt.Sprintf("appname/%s ", appName)) + sb.WriteString(fmt.Sprintf("session_origin/ ")) + sb.WriteString(fmt.Sprintf("grant_type/ ")) + sb.WriteString(fmt.Sprintf("appid/ ")) + sb.WriteString(fmt.Sprintf("clientip/ ")) + sb.WriteString(fmt.Sprintf("devicename/Xiaomi_M2004j7ac ")) + sb.WriteString(fmt.Sprintf("osversion/13 ")) + sb.WriteString(fmt.Sprintf("platformversion/10 ")) + sb.WriteString(fmt.Sprintf("accessmode/ ")) + sb.WriteString(fmt.Sprintf("devicemodel/M2004J7AC ")) + + return sb.String() +} diff --git a/drivers/trainbit/driver.go b/drivers/trainbit/driver.go index 63bd0627f63..f4f4bf3fa90 100644 --- a/drivers/trainbit/driver.go +++ b/drivers/trainbit/driver.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "math" "net/http" "net/url" "strings" @@ -59,7 +58,7 @@ func (d *Trainbit) List(ctx context.Context, dir model.Obj, args model.ListArgs) return nil, err } var jsonData any - json.Unmarshal(data, &jsonData) + err = json.Unmarshal(data, &jsonData) if err != nil { return nil, err } @@ -115,23 +114,18 @@ func (d *Trainbit) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *Trainbit) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { endpoint, _ := url.Parse("https://tb28.trainbit.com/api/upload/send_raw/") query := &url.Values{} query.Add("q", strings.Split(dstDir.GetID(), "_")[1]) query.Add("guid", guid) - query.Add("name", url.QueryEscape(local2provider(stream.GetName(), false)+".")) + query.Add("name", url.QueryEscape(local2provider(s.GetName(), false)+".")) endpoint.RawQuery = query.Encode() - var total int64 - total = 0 - progressReader := &ProgressReader{ - stream, - func(byteNum int) { - total += int64(byteNum) - up(int(math.Round(float64(total) / float64(stream.GetSize()) * 100))) - }, - } - req, err := http.NewRequest(http.MethodPost, endpoint.String(), progressReader) + progressReader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), progressReader) if err != nil { return err } diff --git a/drivers/trainbit/util.go b/drivers/trainbit/util.go index afc111a8290..486e88516ec 100644 --- a/drivers/trainbit/util.go +++ b/drivers/trainbit/util.go @@ -13,17 +13,6 @@ import ( "github.com/alist-org/alist/v3/internal/model" ) -type ProgressReader struct { - io.Reader - reporter func(byteNum int) -} - -func (progressReader *ProgressReader) Read(data []byte) (int, error) { - byteNum, err := progressReader.Reader.Read(data) - progressReader.reporter(byteNum) - return byteNum, err -} - func get(url string, apiKey string, AUSHELLPORTAL string) (*http.Response, error) { req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { diff --git a/drivers/url_tree/driver.go b/drivers/url_tree/driver.go index 6a45bb7d4e1..049bd2db63f 100644 --- a/drivers/url_tree/driver.go +++ b/drivers/url_tree/driver.go @@ -2,11 +2,15 @@ package url_tree import ( "context" + "errors" stdpath "path" + "strings" + "sync" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" log "github.com/sirupsen/logrus" ) @@ -14,7 +18,8 @@ import ( type Urls struct { model.Storage Addition - root *Node + root *Node + mutex sync.RWMutex } func (d *Urls) Config() driver.Config { @@ -40,11 +45,15 @@ func (d *Urls) Drop(ctx context.Context) error { } func (d *Urls) Get(ctx context.Context, path string) (model.Obj, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, path) return nodeToObj(node, path) } func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, dir.GetPath()) log.Debugf("path: %s, node: %+v", dir.GetPath(), node) if node == nil { @@ -59,6 +68,8 @@ func (d *Urls) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([] } func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + d.mutex.RLock() + defer d.mutex.RUnlock() node := GetNodeFromRootByPath(d.root, file.GetPath()) log.Debugf("path: %s, node: %+v", file.GetPath(), node) if node == nil { @@ -72,6 +83,192 @@ func (d *Urls) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (* return nil, errs.NotFile } +func (d *Urls) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + node := GetNodeFromRootByPath(d.root, parentDir.GetPath()) + if node == nil { + return nil, errs.ObjectNotFound + } + if node.isFile() { + return nil, errs.NotFolder + } + dir := &Node{ + Name: dirName, + Level: node.Level + 1, + } + node.Children = append(node.Children, dir) + d.updateStorage() + return nodeToObj(dir, stdpath.Join(parentDir.GetPath(), dirName)) +} + +func (d *Urls) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return nil, errors.New("cannot move parent dir to child") + } + d.mutex.Lock() + defer d.mutex.Unlock() + dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dstNode == nil || dstNode.isFile() { + return nil, errs.NotFolder + } + srcDir, srcName := stdpath.Split(srcObj.GetPath()) + srcParentNode := GetNodeFromRootByPath(d.root, srcDir) + if srcParentNode == nil { + return nil, errs.ObjectNotFound + } + newChildren := make([]*Node, 0, len(srcParentNode.Children)) + var srcNode *Node + for _, child := range srcParentNode.Children { + if child.Name == srcName { + srcNode = child + } else { + newChildren = append(newChildren, child) + } + } + if srcNode == nil { + return nil, errs.ObjectNotFound + } + srcParentNode.Children = newChildren + srcNode.setLevel(dstNode.Level + 1) + dstNode.Children = append(dstNode.Children, srcNode) + d.root.calSize() + d.updateStorage() + return nodeToObj(srcNode, stdpath.Join(dstDir.GetPath(), srcName)) +} + +func (d *Urls) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath()) + if srcNode == nil { + return nil, errs.ObjectNotFound + } + srcNode.Name = newName + d.updateStorage() + return nodeToObj(srcNode, stdpath.Join(stdpath.Dir(srcObj.GetPath()), newName)) +} + +func (d *Urls) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return nil, errors.New("cannot copy parent dir to child") + } + d.mutex.Lock() + defer d.mutex.Unlock() + dstNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dstNode == nil || dstNode.isFile() { + return nil, errs.NotFolder + } + srcNode := GetNodeFromRootByPath(d.root, srcObj.GetPath()) + if srcNode == nil { + return nil, errs.ObjectNotFound + } + newNode := srcNode.deepCopy(dstNode.Level + 1) + dstNode.Children = append(dstNode.Children, newNode) + d.root.calSize() + d.updateStorage() + return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), stdpath.Base(srcObj.GetPath()))) +} + +func (d *Urls) Remove(ctx context.Context, obj model.Obj) error { + if !d.Writable { + return errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + objDir, objName := stdpath.Split(obj.GetPath()) + nodeParent := GetNodeFromRootByPath(d.root, objDir) + if nodeParent == nil { + return errs.ObjectNotFound + } + newChildren := make([]*Node, 0, len(nodeParent.Children)) + var deletedObj *Node + for _, child := range nodeParent.Children { + if child.Name != objName { + newChildren = append(newChildren, child) + } else { + deletedObj = child + } + } + if deletedObj == nil { + return errs.ObjectNotFound + } + nodeParent.Children = newChildren + if deletedObj.Size > 0 { + d.root.calSize() + } + d.updateStorage() + return nil +} + +func (d *Urls) PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) { + if !d.Writable { + return nil, errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + dirNode := GetNodeFromRootByPath(d.root, dstDir.GetPath()) + if dirNode == nil || dirNode.isFile() { + return nil, errs.NotFolder + } + newNode := &Node{ + Name: name, + Level: dirNode.Level + 1, + Url: url, + } + dirNode.Children = append(dirNode.Children, newNode) + if d.HeadSize { + size, err := getSizeFromUrl(url) + if err != nil { + log.Errorf("get size from url error: %s", err) + } else { + newNode.Size = size + d.root.calSize() + } + } + d.updateStorage() + return nodeToObj(newNode, stdpath.Join(dstDir.GetPath(), name)) +} + +func (d *Urls) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + if !d.Writable { + return errs.PermissionDenied + } + d.mutex.Lock() + defer d.mutex.Unlock() + node := GetNodeFromRootByPath(d.root, dstDir.GetPath()) // parent + if node == nil { + return errs.ObjectNotFound + } + if node.isFile() { + return errs.NotFolder + } + file, err := parseFileLine(stream.GetName(), d.HeadSize) + if err != nil { + return err + } + node.Children = append(node.Children, file) + d.updateStorage() + return nil +} + +func (d *Urls) updateStorage() { + d.UrlStructure = StringifyTree(d.root) + op.MustSaveDriverStorage(d) +} + //func (d *Template) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { // return nil, errs.NotSupport //} diff --git a/drivers/url_tree/meta.go b/drivers/url_tree/meta.go index b3ae33dc059..c40414f58c3 100644 --- a/drivers/url_tree/meta.go +++ b/drivers/url_tree/meta.go @@ -12,6 +12,7 @@ type Addition struct { // define other UrlStructure string `json:"url_structure" type:"text" required:"true" default:"https://jsd.nn.ci/gh/alist-org/alist/README.md\nhttps://jsd.nn.ci/gh/alist-org/alist/README_cn.md\nfolder:\n CONTRIBUTING.md:1635:https://jsd.nn.ci/gh/alist-org/alist/CONTRIBUTING.md\n CODE_OF_CONDUCT.md:2093:https://jsd.nn.ci/gh/alist-org/alist/CODE_OF_CONDUCT.md" help:"structure:FolderName:\n [FileName:][FileSize:][Modified:]Url"` HeadSize bool `json:"head_size" type:"bool" default:"false" help:"Use head method to get file size, but it may be failed."` + Writable bool `json:"writable" type:"bool" default:"false"` } var config = driver.Config{ @@ -20,7 +21,7 @@ var config = driver.Config{ OnlyLocal: false, OnlyProxy: false, NoCache: true, - NoUpload: true, + NoUpload: false, NeedMs: false, DefaultRoot: "", CheckStatus: true, diff --git a/drivers/url_tree/types.go b/drivers/url_tree/types.go index 7e8ca3d93ae..cf62d29d65a 100644 --- a/drivers/url_tree/types.go +++ b/drivers/url_tree/types.go @@ -1,5 +1,7 @@ package url_tree +import "github.com/alist-org/alist/v3/pkg/utils" + // Node is a node in the folder tree type Node struct { Url string @@ -44,3 +46,19 @@ func (node *Node) calSize() int64 { node.Size = size return size } + +func (node *Node) setLevel(level int) { + node.Level = level + for _, child := range node.Children { + child.setLevel(level + 1) + } +} + +func (node *Node) deepCopy(level int) *Node { + ret := *node + ret.Level = level + ret.Children, _ = utils.SliceConvert(ret.Children, func(child *Node) (*Node, error) { + return child.deepCopy(level + 1), nil + }) + return &ret +} diff --git a/drivers/url_tree/util.go b/drivers/url_tree/util.go index 4065218fcc1..61a3fde2c1f 100644 --- a/drivers/url_tree/util.go +++ b/drivers/url_tree/util.go @@ -153,6 +153,9 @@ func splitPath(path string) []string { if path == "/" { return []string{"root"} } + if strings.HasSuffix(path, "/") { + path = path[:len(path)-1] + } parts := strings.Split(path, "/") parts[0] = "root" return parts @@ -190,3 +193,46 @@ func getSizeFromUrl(url string) (int64, error) { } return size, nil } + +func StringifyTree(node *Node) string { + sb := strings.Builder{} + if node.Level == -1 { + for i, child := range node.Children { + sb.WriteString(StringifyTree(child)) + if i < len(node.Children)-1 { + sb.WriteString("\n") + } + } + return sb.String() + } + for i := 0; i < node.Level; i++ { + sb.WriteString(" ") + } + if node.Url == "" { + sb.WriteString(node.Name) + sb.WriteString(":") + for _, child := range node.Children { + sb.WriteString("\n") + sb.WriteString(StringifyTree(child)) + } + } else if node.Size == 0 && node.Modified == 0 { + if stdpath.Base(node.Url) == node.Name { + sb.WriteString(node.Url) + } else { + sb.WriteString(fmt.Sprintf("%s:%s", node.Name, node.Url)) + } + } else { + sb.WriteString(node.Name) + sb.WriteString(":") + if node.Size != 0 || node.Modified != 0 { + sb.WriteString(strconv.FormatInt(node.Size, 10)) + sb.WriteString(":") + } + if node.Modified != 0 { + sb.WriteString(strconv.FormatInt(node.Modified, 10)) + sb.WriteString(":") + } + sb.WriteString(node.Url) + } + return sb.String() +} diff --git a/drivers/uss/driver.go b/drivers/uss/driver.go index 447515d8d36..2e219050649 100644 --- a/drivers/uss/driver.go +++ b/drivers/uss/driver.go @@ -3,6 +3,7 @@ package uss import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/stream" "net/url" "path" "strings" @@ -122,11 +123,13 @@ func (d *USS) Remove(ctx context.Context, obj model.Obj) error { }) } -func (d *USS) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { - // TODO not support cancel?? +func (d *USS) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { return d.client.Put(&upyun.PutObjectConfig{ - Path: getKey(path.Join(dstDir.GetPath(), stream.GetName()), false), - Reader: stream, + Path: getKey(path.Join(dstDir.GetPath(), s.GetName()), false), + Reader: driver.NewLimitedUploadStream(ctx, &stream.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }), }) } diff --git a/drivers/vtencent/drive.go b/drivers/vtencent/drive.go new file mode 100644 index 00000000000..36a9167234e --- /dev/null +++ b/drivers/vtencent/drive.go @@ -0,0 +1,210 @@ +package vtencent + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" +) + +type Vtencent struct { + model.Storage + Addition + cron *cron.Cron + config driver.Config + conf Conf +} + +func (d *Vtencent) Config() driver.Config { + return d.config +} + +func (d *Vtencent) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Vtencent) Init(ctx context.Context) error { + tfUid, err := d.LoadUser() + if err != nil { + d.Status = err.Error() + op.MustSaveDriverStorage(d) + return nil + } + d.Addition.TfUid = tfUid + op.MustSaveDriverStorage(d) + d.cron = cron.NewCron(time.Hour * 12) + d.cron.Do(func() { + _, err := d.LoadUser() + if err != nil { + d.Status = err.Error() + op.MustSaveDriverStorage(d) + } + }) + return nil +} + +func (d *Vtencent) Drop(ctx context.Context) error { + if d.cron != nil { + d.cron.Stop() + } + return nil +} + +func (d *Vtencent) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.GetFiles(dir.GetID()) + if err != nil { + return nil, err + } + return utils.SliceConvert(files, func(src File) (model.Obj, error) { + return fileToObj(src), nil + }) +} + +func (d *Vtencent) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + form := fmt.Sprintf(`{"MaterialIds":["%s"]}`, file.GetID()) + var dat map[string]interface{} + if err := json.Unmarshal([]byte(form), &dat); err != nil { + return nil, err + } + var resps RspDown + api := "https://api.vs.tencent.com/SaaS/Material/DescribeMaterialDownloadUrl" + rsp, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(dat) + }, &resps) + if err != nil { + return nil, err + } + if err := json.Unmarshal(rsp, &resps); err != nil { + return nil, err + } + if len(resps.Data.DownloadURLInfoSet) == 0 { + return nil, err + } + u := resps.Data.DownloadURLInfoSet[0].DownloadURL + link := &model.Link{ + URL: u, + Header: http.Header{ + "Referer": []string{d.conf.referer}, + "User-Agent": []string{d.conf.ua}, + }, + Concurrency: 2, + PartSize: 10 * utils.MB, + } + if file.GetSize() == 0 { + link.Concurrency = 0 + link.PartSize = 0 + } + return link, nil +} + +func (d *Vtencent) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + classId, err := strconv.Atoi(parentDir.GetID()) + if err != nil { + return err + } + _, err = d.request("https://api.vs.tencent.com/PaaS/Material/CreateClass", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "Owner": base.Json{ + "Type": "PERSON", + "Id": d.TfUid, + }, + "ParentClassId": classId, + "Name": dirName, + "VerifySign": ""}) + }, nil) + return err +} + +func (d *Vtencent) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + srcType := "MATERIAL" + if srcObj.IsDir() { + srcType = "CLASS" + } + form := fmt.Sprintf(`{"SourceInfos":[ + {"Owner":{"Id":"%s","Type":"PERSON"}, + "Resource":{"Type":"%s","Id":"%s"}} + ], + "Destination":{"Owner":{"Id":"%s","Type":"PERSON"}, + "Resource":{"Type":"CLASS","Id":"%s"}} + }`, d.TfUid, srcType, srcObj.GetID(), d.TfUid, dstDir.GetID()) + var dat map[string]interface{} + if err := json.Unmarshal([]byte(form), &dat); err != nil { + return err + } + _, err := d.request("https://api.vs.tencent.com/PaaS/Material/MoveResource", http.MethodPost, func(req *resty.Request) { + req.SetBody(dat) + }, nil) + return err +} + +func (d *Vtencent) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + api := "https://api.vs.tencent.com/PaaS/Material/ModifyMaterial" + form := fmt.Sprintf(`{ + "Owner":{"Type":"PERSON","Id":"%s"}, + "MaterialId":"%s","Name":"%s"}`, d.TfUid, srcObj.GetID(), newName) + if srcObj.IsDir() { + classId, err := strconv.Atoi(srcObj.GetID()) + if err != nil { + return err + } + api = "https://api.vs.tencent.com/PaaS/Material/ModifyClass" + form = fmt.Sprintf(`{"Owner":{"Type":"PERSON","Id":"%s"}, + "ClassId":%d,"Name":"%s"}`, d.TfUid, classId, newName) + } + var dat map[string]interface{} + if err := json.Unmarshal([]byte(form), &dat); err != nil { + return err + } + _, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(dat) + }, nil) + return err +} + +func (d *Vtencent) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + // TODO copy obj, optional + return errs.NotImplement +} + +func (d *Vtencent) Remove(ctx context.Context, obj model.Obj) error { + srcType := "MATERIAL" + if obj.IsDir() { + srcType = "CLASS" + } + form := fmt.Sprintf(`{ + "SourceInfos":[ + {"Owner":{"Type":"PERSON","Id":"%s"}, + "Resource":{"Type":"%s","Id":"%s"}} + ] + }`, d.TfUid, srcType, obj.GetID()) + var dat map[string]interface{} + if err := json.Unmarshal([]byte(form), &dat); err != nil { + return err + } + _, err := d.request("https://api.vs.tencent.com/PaaS/Material/DeleteResource", http.MethodPost, func(req *resty.Request) { + req.SetBody(dat) + }, nil) + return err +} + +func (d *Vtencent) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + err := d.FileUpload(ctx, dstDir, stream, up) + return err +} + +//func (d *Vtencent) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { +// return nil, errs.NotSupport +//} + +var _ driver.Driver = (*Vtencent)(nil) diff --git a/drivers/vtencent/meta.go b/drivers/vtencent/meta.go new file mode 100644 index 00000000000..3bb6cf74639 --- /dev/null +++ b/drivers/vtencent/meta.go @@ -0,0 +1,39 @@ +package vtencent + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Cookie string `json:"cookie" required:"true"` + TfUid string `json:"tf_uid"` + OrderBy string `json:"order_by" type:"select" options:"Name,Size,UpdateTime,CreatTime"` + OrderDirection string `json:"order_direction" type:"select" options:"Asc,Desc"` +} + +type Conf struct { + ua string + referer string + origin string +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Vtencent{ + config: driver.Config{ + Name: "VTencent", + OnlyProxy: true, + OnlyLocal: false, + DefaultRoot: "9", + NoOverwriteUpload: true, + }, + conf: Conf{ + ua: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) quark-cloud-drive/2.5.20 Chrome/100.0.4896.160 Electron/18.3.5.4-b478491100 Safari/537.36 Channel/pckk_other_ch", + referer: "https://app.v.tencent.com/", + origin: "https://app.v.tencent.com", + }, + } + }) +} diff --git a/drivers/vtencent/signature.go b/drivers/vtencent/signature.go new file mode 100644 index 00000000000..14fda9bdc21 --- /dev/null +++ b/drivers/vtencent/signature.go @@ -0,0 +1,33 @@ +package vtencent + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/hex" +) + +func QSignatureKey(timeKey string, signPath string, key string) string { + signKey := hmac.New(sha1.New, []byte(key)) + signKey.Write([]byte(timeKey)) + signKeyBytes := signKey.Sum(nil) + signKeyHex := hex.EncodeToString(signKeyBytes) + sha := sha1.New() + sha.Write([]byte(signPath)) + shaBytes := sha.Sum(nil) + shaHex := hex.EncodeToString(shaBytes) + + O := "sha1\n" + timeKey + "\n" + shaHex + "\n" + dataSignKey := hmac.New(sha1.New, []byte(signKeyHex)) + dataSignKey.Write([]byte(O)) + dataSignKeyBytes := dataSignKey.Sum(nil) + dataSignKeyHex := hex.EncodeToString(dataSignKeyBytes) + return dataSignKeyHex +} + +func QTwoSignatureKey(timeKey string, key string) string { + signKey := hmac.New(sha1.New, []byte(key)) + signKey.Write([]byte(timeKey)) + signKeyBytes := signKey.Sum(nil) + signKeyHex := hex.EncodeToString(signKeyBytes) + return signKeyHex +} diff --git a/drivers/vtencent/types.go b/drivers/vtencent/types.go new file mode 100644 index 00000000000..b967481e253 --- /dev/null +++ b/drivers/vtencent/types.go @@ -0,0 +1,252 @@ +package vtencent + +import ( + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type RespErr struct { + Code string `json:"Code"` + Message string `json:"Message"` +} + +type Reqfiles struct { + ScrollToken string `json:"ScrollToken"` + Text string `json:"Text"` + Offset int `json:"Offset"` + Limit int `json:"Limit"` + Sort struct { + Field string `json:"Field"` + Order string `json:"Order"` + } `json:"Sort"` + CreateTimeRanges []any `json:"CreateTimeRanges"` + MaterialTypes []any `json:"MaterialTypes"` + ReviewStatuses []any `json:"ReviewStatuses"` + Tags []any `json:"Tags"` + SearchScopes []struct { + Owner struct { + Type string `json:"Type"` + ID string `json:"Id"` + } `json:"Owner"` + ClassID int `json:"ClassId"` + SearchOneDepth bool `json:"SearchOneDepth"` + } `json:"SearchScopes"` +} + +type File struct { + Type string `json:"Type"` + ClassInfo struct { + ClassID int `json:"ClassId"` + Name string `json:"Name"` + UpdateTime time.Time `json:"UpdateTime"` + CreateTime time.Time `json:"CreateTime"` + FileInboxID string `json:"FileInboxId"` + Owner struct { + Type string `json:"Type"` + ID string `json:"Id"` + } `json:"Owner"` + ClassPath string `json:"ClassPath"` + ParentClassID int `json:"ParentClassId"` + AttachmentInfo struct { + SubClassCount int `json:"SubClassCount"` + MaterialCount int `json:"MaterialCount"` + Size int64 `json:"Size"` + } `json:"AttachmentInfo"` + ClassPreviewURLSet []string `json:"ClassPreviewUrlSet"` + } `json:"ClassInfo"` + MaterialInfo struct { + BasicInfo struct { + MaterialID string `json:"MaterialId"` + MaterialType string `json:"MaterialType"` + Name string `json:"Name"` + CreateTime time.Time `json:"CreateTime"` + UpdateTime time.Time `json:"UpdateTime"` + ClassPath string `json:"ClassPath"` + ClassID int `json:"ClassId"` + TagInfoSet []any `json:"TagInfoSet"` + TagSet []any `json:"TagSet"` + PreviewURL string `json:"PreviewUrl"` + MediaURL string `json:"MediaUrl"` + UnifiedMediaPreviewURL string `json:"UnifiedMediaPreviewUrl"` + Owner struct { + Type string `json:"Type"` + ID string `json:"Id"` + } `json:"Owner"` + PermissionSet any `json:"PermissionSet"` + PermissionInfoSet []any `json:"PermissionInfoSet"` + TfUID string `json:"TfUid"` + GroupID string `json:"GroupId"` + VersionMaterialIDSet []any `json:"VersionMaterialIdSet"` + FileType string `json:"FileType"` + CmeMaterialPlayList []any `json:"CmeMaterialPlayList"` + Status string `json:"Status"` + DownloadSwitch string `json:"DownloadSwitch"` + } `json:"BasicInfo"` + MediaInfo struct { + Width int `json:"Width"` + Height int `json:"Height"` + Size int `json:"Size"` + Duration float64 `json:"Duration"` + Fps int `json:"Fps"` + BitRate int `json:"BitRate"` + Codec string `json:"Codec"` + MediaType string `json:"MediaType"` + FavoriteStatus string `json:"FavoriteStatus"` + } `json:"MediaInfo"` + MaterialStatus struct { + ContentReviewStatus string `json:"ContentReviewStatus"` + EditorUsableStatus string `json:"EditorUsableStatus"` + UnifiedPreviewStatus string `json:"UnifiedPreviewStatus"` + EditPreviewImageSpiritStatus string `json:"EditPreviewImageSpiritStatus"` + TranscodeStatus string `json:"TranscodeStatus"` + AdaptiveStreamingStatus string `json:"AdaptiveStreamingStatus"` + StreamConnectable string `json:"StreamConnectable"` + AiAnalysisStatus string `json:"AiAnalysisStatus"` + AiRecognitionStatus string `json:"AiRecognitionStatus"` + } `json:"MaterialStatus"` + ImageMaterial struct { + Height int `json:"Height"` + Width int `json:"Width"` + Size int `json:"Size"` + MaterialURL string `json:"MaterialUrl"` + Resolution string `json:"Resolution"` + VodFileID string `json:"VodFileId"` + OriginalURL string `json:"OriginalUrl"` + } `json:"ImageMaterial"` + VideoMaterial struct { + MetaData struct { + Size int `json:"Size"` + Container string `json:"Container"` + Bitrate int `json:"Bitrate"` + Height int `json:"Height"` + Width int `json:"Width"` + Duration float64 `json:"Duration"` + Rotate int `json:"Rotate"` + VideoStreamInfoSet []struct { + Bitrate int `json:"Bitrate"` + Height int `json:"Height"` + Width int `json:"Width"` + Codec string `json:"Codec"` + Fps int `json:"Fps"` + } `json:"VideoStreamInfoSet"` + AudioStreamInfoSet []struct { + Bitrate int `json:"Bitrate"` + SamplingRate int `json:"SamplingRate"` + Codec string `json:"Codec"` + } `json:"AudioStreamInfoSet"` + } `json:"MetaData"` + ImageSpriteInfo any `json:"ImageSpriteInfo"` + MaterialURL string `json:"MaterialUrl"` + CoverURL string `json:"CoverUrl"` + Resolution string `json:"Resolution"` + VodFileID string `json:"VodFileId"` + OriginalURL string `json:"OriginalUrl"` + AudioWaveformURL string `json:"AudioWaveformUrl"` + SubtitleURL string `json:"SubtitleUrl"` + TranscodeInfoSet []any `json:"TranscodeInfoSet"` + ImageSpriteInfoSet []any `json:"ImageSpriteInfoSet"` + } `json:"VideoMaterial"` + } `json:"MaterialInfo"` +} + +type RspFiles struct { + Code string `json:"Code"` + Message string `json:"Message"` + EnglishMessage string `json:"EnglishMessage"` + Data struct { + TotalCount int `json:"TotalCount"` + ResourceInfoSet []File `json:"ResourceInfoSet"` + ScrollToken string `json:"ScrollToken"` + } `json:"Data"` +} + +type RspDown struct { + Code string `json:"Code"` + Message string `json:"Message"` + EnglishMessage string `json:"EnglishMessage"` + Data struct { + DownloadURLInfoSet []struct { + MaterialID string `json:"MaterialId"` + DownloadURL string `json:"DownloadUrl"` + } `json:"DownloadUrlInfoSet"` + } `json:"Data"` +} + +type RspCreatrMaterial struct { + Code string `json:"Code"` + Message string `json:"Message"` + EnglishMessage string `json:"EnglishMessage"` + Data struct { + UploadContext string `json:"UploadContext"` + VodUploadSign string `json:"VodUploadSign"` + QuickUpload bool `json:"QuickUpload"` + } `json:"Data"` +} + +type RspApplyUploadUGC struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Video struct { + StorageSignature string `json:"storageSignature"` + StoragePath string `json:"storagePath"` + } `json:"video"` + StorageAppID int `json:"storageAppId"` + StorageBucket string `json:"storageBucket"` + StorageRegion string `json:"storageRegion"` + StorageRegionV5 string `json:"storageRegionV5"` + Domain string `json:"domain"` + VodSessionKey string `json:"vodSessionKey"` + TempCertificate struct { + SecretID string `json:"secretId"` + SecretKey string `json:"secretKey"` + Token string `json:"token"` + ExpiredTime int `json:"expiredTime"` + } `json:"tempCertificate"` + AppID int `json:"appId"` + Timestamp int `json:"timestamp"` + StorageRegionV50 string `json:"StorageRegionV5"` + MiniProgramAccelerateHost string `json:"MiniProgramAccelerateHost"` + } `json:"data"` +} + +type RspCommitUploadUGC struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Video struct { + URL string `json:"url"` + VerifyContent string `json:"verify_content"` + } `json:"video"` + FileID string `json:"fileId"` + } `json:"data"` +} + +type RspFinishUpload struct { + Code string `json:"Code"` + Message string `json:"Message"` + EnglishMessage string `json:"EnglishMessage"` + Data struct { + MaterialID string `json:"MaterialId"` + } `json:"Data"` +} + +func fileToObj(f File) *model.Object { + obj := &model.Object{} + if f.Type == "CLASS" { + obj.Name = f.ClassInfo.Name + obj.ID = strconv.Itoa(f.ClassInfo.ClassID) + obj.IsFolder = true + obj.Modified = f.ClassInfo.CreateTime + obj.Size = 0 + } else if f.Type == "MATERIAL" { + obj.Name = f.MaterialInfo.BasicInfo.Name + obj.ID = f.MaterialInfo.BasicInfo.MaterialID + obj.IsFolder = false + obj.Modified = f.MaterialInfo.BasicInfo.CreateTime + obj.Size = int64(f.MaterialInfo.MediaInfo.Size) + } + return obj +} diff --git a/drivers/vtencent/util.go b/drivers/vtencent/util.go new file mode 100644 index 00000000000..4ba72d1b2e3 --- /dev/null +++ b/drivers/vtencent/util.go @@ -0,0 +1,299 @@ +package vtencent + +import ( + "context" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "strconv" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/go-resty/resty/v2" +) + +func (d *Vtencent) request(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "cookie": d.Cookie, + "content-type": "application/json", + "origin": d.conf.origin, + "referer": d.conf.referer, + }) + if callback != nil { + callback(req) + } else { + req.SetBody("{}") + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + code := utils.Json.Get(res.Body(), "Code").ToString() + if code != "Success" { + switch code { + case "AuthFailure.SessionInvalid": + if err != nil { + return nil, errors.New(code) + } + default: + return nil, errors.New(code) + } + return d.request(url, method, callback, resp) + } + return res.Body(), nil +} + +func (d *Vtencent) ugcRequest(url, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "cookie": d.Cookie, + "content-type": "application/json", + "origin": d.conf.origin, + "referer": d.conf.referer, + }) + if callback != nil { + callback(req) + } else { + req.SetBody("{}") + } + if resp != nil { + req.SetResult(resp) + } + res, err := req.Execute(method, url) + if err != nil { + return nil, err + } + code := utils.Json.Get(res.Body(), "Code").ToInt() + if code != 0 { + message := utils.Json.Get(res.Body(), "message").ToString() + if len(message) == 0 { + message = utils.Json.Get(res.Body(), "msg").ToString() + } + return nil, errors.New(message) + } + return res.Body(), nil +} + +func (d *Vtencent) LoadUser() (string, error) { + api := "https://api.vs.tencent.com/SaaS/Account/DescribeAccount" + res, err := d.request(api, http.MethodPost, func(req *resty.Request) {}, nil) + if err != nil { + return "", err + } + return utils.Json.Get(res, "Data", "TfUid").ToString(), nil +} + +func (d *Vtencent) GetFiles(dirId string) ([]File, error) { + var res []File + //offset := 0 + for { + api := "https://api.vs.tencent.com/PaaS/Material/SearchResource" + form := fmt.Sprintf(`{ + "Text":"", + "Text":"", + "Offset":%d, + "Limit":50, + "Sort":{"Field":"%s","Order":"%s"}, + "CreateTimeRanges":[], + "MaterialTypes":[], + "ReviewStatuses":[], + "Tags":[], + "SearchScopes":[{"Owner":{"Type":"PERSON","Id":"%s"},"ClassId":%s,"SearchOneDepth":true}] + }`, len(res), d.Addition.OrderBy, d.Addition.OrderDirection, d.TfUid, dirId) + var resp RspFiles + _, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resp) + if err != nil { + return nil, err + } + res = append(res, resp.Data.ResourceInfoSet...) + if len(resp.Data.ResourceInfoSet) <= 0 || len(res) >= resp.Data.TotalCount { + break + } + } + return res, nil +} + +func (d *Vtencent) CreateUploadMaterial(classId int, fileName string, UploadSummaryKey string) (RspCreatrMaterial, error) { + api := "https://api.vs.tencent.com/PaaS/Material/CreateUploadMaterial" + form := base.Json{"Owner": base.Json{"Type": "PERSON", "Id": d.TfUid}, + "MaterialType": "VIDEO", "Name": fileName, "ClassId": classId, + "UploadSummaryKey": UploadSummaryKey} + var resps RspCreatrMaterial + _, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspCreatrMaterial{}, err + } + return resps, nil +} + +func (d *Vtencent) ApplyUploadUGC(signature string, stream model.FileStreamer) (RspApplyUploadUGC, error) { + api := "https://vod2.qcloud.com/v3/index.php?Action=ApplyUploadUGC" + form := base.Json{ + "signature": signature, + "videoName": stream.GetName(), + "videoType": utils.Ext(stream.GetName()), + "videoSize": stream.GetSize(), + } + var resps RspApplyUploadUGC + _, err := d.ugcRequest(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspApplyUploadUGC{}, err + } + return resps, nil +} + +func (d *Vtencent) CommitUploadUGC(signature string, vodSessionKey string) (RspCommitUploadUGC, error) { + api := "https://vod2.qcloud.com/v3/index.php?Action=CommitUploadUGC" + form := base.Json{ + "signature": signature, + "vodSessionKey": vodSessionKey, + } + var resps RspCommitUploadUGC + rsp, err := d.ugcRequest(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspCommitUploadUGC{}, err + } + if len(resps.Data.Video.URL) == 0 { + return RspCommitUploadUGC{}, errors.New(string(rsp)) + } + return resps, nil +} + +func (d *Vtencent) FinishUploadMaterial(SummaryKey string, VodVerifyKey string, UploadContext, VodFileId string) (RspFinishUpload, error) { + api := "https://api.vs.tencent.com/PaaS/Material/FinishUploadMaterial" + form := base.Json{ + "UploadContext": UploadContext, + "VodVerifyKey": VodVerifyKey, + "VodFileId": VodFileId, + "UploadFullKey": SummaryKey} + var resps RspFinishUpload + rsp, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspFinishUpload{}, err + } + if len(resps.Data.MaterialID) == 0 { + return RspFinishUpload{}, errors.New(string(rsp)) + } + return resps, nil +} + +func (d *Vtencent) FinishHashUploadMaterial(SummaryKey string, UploadContext string) (RspFinishUpload, error) { + api := "https://api.vs.tencent.com/PaaS/Material/FinishUploadMaterial" + var resps RspFinishUpload + form := base.Json{ + "UploadContext": UploadContext, + "UploadFullKey": SummaryKey} + rsp, err := d.request(api, http.MethodPost, func(req *resty.Request) { + req.SetBody(form).ForceContentType("application/json") + }, &resps) + if err != nil { + return RspFinishUpload{}, err + } + if len(resps.Data.MaterialID) == 0 { + return RspFinishUpload{}, errors.New(string(rsp)) + } + return resps, nil +} + +func (d *Vtencent) FileUpload(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + classId, err := strconv.Atoi(dstDir.GetID()) + if err != nil { + return err + } + const chunkLength int64 = 1024 * 1024 * 10 + reader, err := stream.RangeRead(http_range.Range{Start: 0, Length: chunkLength}) + if err != nil { + return err + } + chunkHash, err := utils.HashReader(utils.SHA1, reader) + if err != nil { + return err + } + rspCreatrMaterial, err := d.CreateUploadMaterial(classId, stream.GetName(), chunkHash) + if err != nil { + return err + } + if rspCreatrMaterial.Data.QuickUpload { + SummaryKey := stream.GetHash().GetHash(utils.SHA1) + if len(SummaryKey) < utils.SHA1.Width { + if SummaryKey, err = utils.HashReader(utils.SHA1, stream); err != nil { + return err + } + } + UploadContext := rspCreatrMaterial.Data.UploadContext + _, err = d.FinishHashUploadMaterial(SummaryKey, UploadContext) + if err != nil { + return err + } + return nil + } + hash := sha1.New() + rspUGC, err := d.ApplyUploadUGC(rspCreatrMaterial.Data.VodUploadSign, stream) + if err != nil { + return err + } + params := rspUGC.Data + certificate := params.TempCertificate + cfg := &aws.Config{ + HTTPClient: base.HttpClient, + // S3ForcePathStyle: aws.Bool(true), + Credentials: credentials.NewStaticCredentials(certificate.SecretID, certificate.SecretKey, certificate.Token), + Region: aws.String(params.StorageRegionV5), + Endpoint: aws.String(fmt.Sprintf("cos.%s.myqcloud.com", params.StorageRegionV5)), + } + ss, err := session.NewSession(cfg) + if err != nil { + return err + } + uploader := s3manager.NewUploader(ss) + if stream.GetSize() > s3manager.MaxUploadParts*s3manager.DefaultUploadPartSize { + uploader.PartSize = stream.GetSize() / (s3manager.MaxUploadParts - 1) + } + input := &s3manager.UploadInput{ + Bucket: aws.String(fmt.Sprintf("%s-%d", params.StorageBucket, params.StorageAppID)), + Key: ¶ms.Video.StoragePath, + Body: driver.NewLimitedUploadStream(ctx, + io.TeeReader(stream, io.MultiWriter(hash, driver.NewProgress(stream.GetSize(), up)))), + } + _, err = uploader.UploadWithContext(ctx, input) + if err != nil { + return err + } + rspCommitUGC, err := d.CommitUploadUGC(rspCreatrMaterial.Data.VodUploadSign, rspUGC.Data.VodSessionKey) + if err != nil { + return err + } + VodVerifyKey := rspCommitUGC.Data.Video.VerifyContent + VodFileId := rspCommitUGC.Data.FileID + UploadContext := rspCreatrMaterial.Data.UploadContext + SummaryKey := hex.EncodeToString(hash.Sum(nil)) + _, err = d.FinishUploadMaterial(SummaryKey, VodVerifyKey, UploadContext, VodFileId) + if err != nil { + return err + } + return nil +} diff --git a/drivers/webdav/driver.go b/drivers/webdav/driver.go index b402b1db0fa..45150fca57d 100644 --- a/drivers/webdav/driver.go +++ b/drivers/webdav/driver.go @@ -93,13 +93,16 @@ func (d *WebDav) Remove(ctx context.Context, obj model.Obj) error { return d.client.RemoveAll(getPath(obj)) } -func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *WebDav) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { callback := func(r *http.Request) { - r.Header.Set("Content-Type", stream.GetMimetype()) - r.ContentLength = stream.GetSize() + r.Header.Set("Content-Type", s.GetMimetype()) + r.ContentLength = s.GetSize() } - // TODO: support cancel - err := d.client.WriteStream(path.Join(dstDir.GetPath(), stream.GetName()), stream, 0644, callback) + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + err := d.client.WriteStream(path.Join(dstDir.GetPath(), s.GetName()), reader, 0644, callback) return err } diff --git a/drivers/webdav/util.go b/drivers/webdav/util.go index 92557c4f553..dfd6e5b2457 100644 --- a/drivers/webdav/util.go +++ b/drivers/webdav/util.go @@ -1,9 +1,12 @@ package webdav import ( + "crypto/tls" "net/http" + "net/http/cookiejar" "github.com/alist-org/alist/v3/drivers/webdav/odrvcookie" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/gowebdav" ) @@ -16,6 +19,10 @@ func (d *WebDav) isSharepoint() bool { func (d *WebDav) setClient() error { c := gowebdav.NewClient(d.Address, d.Username, d.Password) + c.SetTransport(&http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + }) if d.isSharepoint() { cookie, err := odrvcookie.GetCookie(d.Username, d.Password, d.Address) if err == nil { @@ -26,6 +33,13 @@ func (d *WebDav) setClient() error { } else { return err } + } else { + cookieJar, err := cookiejar.New(nil) + if err == nil { + c.SetJar(cookieJar) + } else { + return err + } } d.client = c return nil diff --git a/drivers/weiyun/driver.go b/drivers/weiyun/driver.go index e6d5897c313..90793d333f8 100644 --- a/drivers/weiyun/driver.go +++ b/drivers/weiyun/driver.go @@ -7,6 +7,7 @@ import ( "math" "net/http" "strconv" + "sync/atomic" "time" "github.com/alist-org/alist/v3/drivers/base" @@ -69,7 +70,7 @@ func (d *WeiYun) Init(ctx context.Context) error { if d.client.LoginType() == 1 { d.cron = cron.NewCron(time.Minute * 5) d.cron.Do(func() { - d.client.KeepAlive() + _ = d.client.KeepAlive() }) } @@ -311,77 +312,83 @@ func (d *WeiYun) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr // NOTE: // 秒传需要sha1最后一个状态,但sha1无法逆运算需要读完整个文件(或许可以??) // 服务器支持上传进度恢复,不需要额外实现 - if folder, ok := dstDir.(*Folder); ok { - file, err := stream.CacheFullInTempFile() - if err != nil { - return nil, err - } + var folder *Folder + var ok bool + if folder, ok = dstDir.(*Folder); !ok { + return nil, errs.NotSupport + } + file, err := stream.CacheFullInTempFile() + if err != nil { + return nil, err + } - // step 1. - preData, err := d.client.PreUpload(ctx, weiyunsdkgo.UpdloadFileParam{ - PdirKey: folder.GetPKey(), - DirKey: folder.DirKey, + // step 1. + preData, err := d.client.PreUpload(ctx, weiyunsdkgo.UpdloadFileParam{ + PdirKey: folder.GetPKey(), + DirKey: folder.DirKey, - FileName: stream.GetName(), - FileSize: stream.GetSize(), - File: file, + FileName: stream.GetName(), + FileSize: stream.GetSize(), + File: file, - ChannelCount: 4, - FileExistOption: 1, - }) - if err != nil { - return nil, err - } - - // not fast upload - if !preData.FileExist { - // step.2 增加上传通道 - if len(preData.ChannelList) < d.uploadThread { - newCh, err := d.client.AddUploadChannel(len(preData.ChannelList), d.uploadThread, preData.UploadAuthData) - if err != nil { - return nil, err - } - preData.ChannelList = append(preData.ChannelList, newCh.AddChannels...) - } - // step.3 上传 - threadG, upCtx := errgroup.NewGroupWithContext(ctx, len(preData.ChannelList), - retry.Attempts(3), - retry.Delay(time.Second), - retry.DelayType(retry.BackOffDelay)) - - for _, channel := range preData.ChannelList { - if utils.IsCanceled(upCtx) { - break - } + ChannelCount: 4, + FileExistOption: 1, + }) + if err != nil { + return nil, err + } - var channel = channel - threadG.Go(func(ctx context.Context) error { - for { - channel.Len = int(math.Min(float64(stream.GetSize()-channel.Offset), float64(channel.Len))) - upData, err := d.client.UploadFile(upCtx, channel, preData.UploadAuthData, - io.NewSectionReader(file, channel.Offset, int64(channel.Len))) - if err != nil { - return err - } - // 上传完成 - if upData.UploadState != 1 { - return nil - } - channel = upData.Channel - } - }) - } - if err = threadG.Wait(); err != nil { + // not fast upload + if !preData.FileExist { + // step.2 增加上传通道 + if len(preData.ChannelList) < d.uploadThread { + newCh, err := d.client.AddUploadChannel(len(preData.ChannelList), d.uploadThread, preData.UploadAuthData) + if err != nil { return nil, err } + preData.ChannelList = append(preData.ChannelList, newCh.AddChannels...) } + // step.3 上传 + threadG, upCtx := errgroup.NewGroupWithContext(ctx, len(preData.ChannelList), + retry.Attempts(3), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + + total := atomic.Int64{} + for _, channel := range preData.ChannelList { + if utils.IsCanceled(upCtx) { + break + } - return &File{ - PFolder: folder, - File: preData.File, - }, nil + var channel = channel + threadG.Go(func(ctx context.Context) error { + for { + channel.Len = int(math.Min(float64(stream.GetSize()-channel.Offset), float64(channel.Len))) + len64 := int64(channel.Len) + upData, err := d.client.UploadFile(upCtx, channel, preData.UploadAuthData, + driver.NewLimitedUploadStream(ctx, io.NewSectionReader(file, channel.Offset, len64))) + if err != nil { + return err + } + cur := total.Add(len64) + up(float64(cur) * 100.0 / float64(stream.GetSize())) + // 上传完成 + if upData.UploadState != 1 { + return nil + } + channel = upData.Channel + } + }) + } + if err = threadG.Wait(); err != nil { + return nil, err + } } - return nil, errs.NotSupport + + return &File{ + PFolder: folder, + File: preData.File, + }, nil } // func (d *WeiYun) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { diff --git a/drivers/wopan/driver.go b/drivers/wopan/driver.go index a3f222e8ef3..82ec05a919e 100644 --- a/drivers/wopan/driver.go +++ b/drivers/wopan/driver.go @@ -5,12 +5,12 @@ import ( "fmt" "strconv" - "github.com/Xhofe/wopan-sdk-go" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + "github.com/xhofe/wopan-sdk-go" ) type Wopan struct { @@ -155,12 +155,13 @@ func (d *Wopan) Put(ctx context.Context, dstDir model.Obj, stream model.FileStre _, err := d.client.Upload2C(d.getSpaceType(), wopan.Upload2CFile{ Name: stream.GetName(), Size: stream.GetSize(), - Content: stream, + Content: driver.NewLimitedUploadStream(ctx, stream), ContentType: stream.GetMimetype(), }, dstDir.GetID(), d.FamilyID, wopan.Upload2COption{ OnProgress: func(current, total int64) { - up(int(100 * current / total)) + up(100 * float64(current) / float64(total)) }, + Ctx: ctx, }) return err } diff --git a/drivers/wopan/types.go b/drivers/wopan/types.go index 8a39c18a807..4025dbab237 100644 --- a/drivers/wopan/types.go +++ b/drivers/wopan/types.go @@ -1,8 +1,8 @@ package template import ( - "github.com/Xhofe/wopan-sdk-go" "github.com/alist-org/alist/v3/internal/model" + "github.com/xhofe/wopan-sdk-go" ) type Object struct { diff --git a/drivers/wopan/util.go b/drivers/wopan/util.go index 2ac4e61baff..b825d6ea9ac 100644 --- a/drivers/wopan/util.go +++ b/drivers/wopan/util.go @@ -3,7 +3,7 @@ package template import ( "time" - "github.com/Xhofe/wopan-sdk-go" + "github.com/xhofe/wopan-sdk-go" ) // do others that not defined in Driver interface diff --git a/drivers/wukong/driver.go b/drivers/wukong/driver.go new file mode 100644 index 00000000000..cb5dee1ae40 --- /dev/null +++ b/drivers/wukong/driver.go @@ -0,0 +1,1116 @@ +package wukong + +import ( + "context" + "crypto/hmac" + "crypto/md5" + crand "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "io" + "net/http" + "net/url" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/go-resty/resty/v2" +) + +const ( + wukongBaseURL = "https://api.wkbrowser.com" + webReferer = "https://pan.wkbrowser.com/" + vodBaseURL = "https://vod.bytedanceapi.com" + vodRegion = "cn-north-1" + vodService = "vod" + videoSpaceName = "wukong_netdisk_ugc" + minUploadSubmitSuccess = 2000 + multipartChunkSize = int64(5 * 1024 * 1024) +) + +type Wukong struct { + model.Storage + Addition + client *resty.Client +} + +func (d *Wukong) Config() driver.Config { + return config +} + +func (d *Wukong) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Wukong) Init(ctx context.Context) error { + d.client = base.NewRestyClient(). + SetBaseURL(wukongBaseURL). + SetHeader("accept", "application/json, text/plain, */*"). + SetHeader("content-type", "application/json"). + SetHeader("referer", webReferer). + SetHeader("origin", "https://pan.wkbrowser.com") + if d.Cookie != "" { + d.client.SetHeader("cookie", d.Cookie) + } + if d.RootFolderID == "" { + d.RootFolderID = "0" + } + if strings.TrimSpace(d.Aid) == "" { + d.Aid = "590353" + } + if strings.TrimSpace(d.Language) == "" { + d.Language = "zh" + } + if d.PageSize <= 0 { + d.PageSize = 100 + } + return nil +} + +func (d *Wukong) Drop(ctx context.Context) error { + return nil +} + +func (d *Wukong) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + fatherID := dir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + offset := 0 + limit := d.PageSize + objs := make([]model.Obj, 0) + for { + var resp filterFileResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "offset": strconv.Itoa(offset), + "limit": strconv.Itoa(limit), + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "father_id": asIDValue(fatherID), + "filter_type": 2, + "is_desc": 1, + "file_type": 0, + }). + SetResult(&resp). + Post("/netdisk/user_file/filter_file") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong list failed: code=%d message=%s", resp.Code, resp.Message) + } + + for _, item := range resp.Data.FileList { + objs = append(objs, &model.Object{ + ID: strconv.FormatInt(item.FileID, 10), + Path: strconv.FormatInt(item.FatherID, 10), + Name: item.FileName, + Size: item.Size, + Modified: parseUnix(item.UpdatedAt), + Ctime: parseUnix(item.CreatedAt), + IsFolder: item.IsDirectory == 1, + }) + } + + if !hasMore(resp.Data.HasMore) || len(resp.Data.FileList) == 0 { + break + } + offset += len(resp.Data.FileList) + } + return objs, nil +} + +func (d *Wukong) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if file.IsDir() { + return nil, errs.NotFile + } + fileID := file.GetID() + if fileID == "" { + return nil, errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(fileID)}, + }). + SetResult(&resp). + Post("/netdisk/user_file/detail") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong detail failed: code=%d message=%s", resp.Code, resp.Message) + } + + url := extractDetailMainURL(resp.Data) + if url == "" { + url = extractURL(resp.Data) + } + if url == "" { + return nil, errs.NotImplement + } + + return &model.Link{ + URL: url, + Header: http.Header{ + "Referer": []string{webReferer}, + "Cookie": []string{d.Cookie}, + }, + }, nil +} + +func (d *Wukong) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + fatherID := parentDir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "father_id": asIDValue(fatherID), + "file_name": dirName, + }). + SetResult(&resp). + Post("/netdisk/user_file/create_directory") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong create directory failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + srcID := srcObj.GetID() + if srcID == "" { + return errors.New("missing source file id") + } + + dstID := dstDir.GetID() + if dstID == "" { + dstID = d.RootFolderID + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(srcID)}, + "new_father_id": asIDValue(dstID), + }). + SetResult(&resp). + Post("/netdisk/user_file/move_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong move failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + srcID := srcObj.GetID() + if srcID == "" { + return errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id": asIDValue(srcID), + "new_name": newName, + }). + SetResult(&resp). + Post("/netdisk/user_file/rename_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong rename failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Remove(ctx context.Context, obj model.Obj) error { + fileID := obj.GetID() + if fileID == "" { + return errors.New("missing file id") + } + + var resp rawResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(map[string]any{ + "file_id_list": []any{asIDValue(fileID)}, + }). + SetResult(&resp). + Post("/netdisk/user_file/delete_file") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong delete failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + fatherID := dstDir.GetID() + if fatherID == "" { + fatherID = d.RootFolderID + } + + tempFile, err := file.CacheFullInTempFile() + if err != nil { + return err + } + defer tempFile.Close() + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + + md5Hex, crc32Hex, err := calcFileMD5AndCRC32(tempFile) + if err != nil { + return err + } + ext := strings.TrimPrefix(strings.ToLower(filepath.Ext(file.GetName())), ".") + fileType := detectWukongFileType(file.GetMimetype(), file.GetName()) + size := file.GetSize() + up(5) + + uploadType := detectUploadType(file.GetMimetype(), file.GetName()) + authToken, err := d.getUploadAuthToken(ctx, uploadType) + if err != nil { + return err + } + up(10) + + candidates, err := d.getUploadCandidates(ctx, authToken) + if err != nil { + return err + } + bestHosts := collectCandidateHosts(candidates) + if len(bestHosts) == 0 { + return errors.New("wukong upload candidates is empty") + } + up(20) + + applyResp, err := d.applyUploadInner(ctx, authToken, uploadType, size, strings.Join(bestHosts, ",")) + if err != nil { + return err + } + if len(applyResp.Result.InnerUploadAddress.UploadNodes) == 0 || + len(applyResp.Result.InnerUploadAddress.UploadNodes[0].StoreInfos) == 0 { + return errors.New("wukong apply upload inner returns empty upload node") + } + node := applyResp.Result.InnerUploadAddress.UploadNodes[0] + store := node.StoreInfos[0] + up(30) + + if size > multipartChunkSize { + if err = d.uploadToTOSMultipart(ctx, node.UploadHost, store.StoreURI, store.Auth, getStorageUserID(store.StorageHeader), tempFile, size, up); err != nil { + return err + } + } else { + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + reader := &driver.ReaderUpdatingProgress{ + Reader: &driver.SimpleReaderWithSize{Reader: tempFile, Size: size}, + UpdateProgress: func(percent float64) { + up(30 + percent*0.5) + }, + } + if err = d.uploadToTOS(ctx, node.UploadHost, store.StoreURI, store.Auth, getStorageUserID(store.StorageHeader), crc32Hex, file.GetName(), reader, size); err != nil { + return err + } + } + up(85) + + videoVid, err := d.commitUploadInner(ctx, authToken, chooseCommitSpace(uploadType, authToken.SpaceName), node.SessionKey) + if err != nil { + return err + } + up(92) + + if fileType == 3000 && videoVid == "" { + return errors.New("wukong video upload missing vid in commit response") + } + if err = d.uploadSubmit(ctx, fatherID, file.GetName(), ext, fileType, size, md5Hex, store.StoreURI, videoVid); err != nil { + return err + } + up(100) + return nil +} + +func (d *Wukong) getUploadAuthToken(ctx context.Context, uploadType string) (*uploadAuthTokenResp, error) { + var resp uploadAuthTokenResp + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "upload_source": uploadSourceByType(uploadType), + "type": uploadType, + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetResult(&resp). + Get("/toutiao/upload/auth_token/v1/") + if err != nil { + return nil, err + } + if resp.Code != 0 { + return nil, fmt.Errorf("wukong get upload auth token failed: code=%d message=%s", resp.Code, resp.Message) + } + return &resp, nil +} + +func (d *Wukong) getUploadCandidates(ctx context.Context, auth *uploadAuthTokenResp) (*getUploadCandidatesResp, error) { + q := map[string]string{ + "Action": "GetUploadCandidates", + "Version": "2020-11-19", + "SpaceName": videoSpaceName, + } + var resp getUploadCandidatesResp + if err := d.vodRequest(ctx, http.MethodGet, q, nil, auth, &resp); err != nil { + return nil, err + } + if resp.ResponseMetadata.Error.Code != "" { + return nil, fmt.Errorf("wukong get upload candidates failed: %s", resp.ResponseMetadata.Error.Message) + } + return &resp, nil +} + +func (d *Wukong) applyUploadInner(ctx context.Context, auth *uploadAuthTokenResp, uploadType string, fileSize int64, bestHosts string) (*applyUploadInnerResp, error) { + spaceName := auth.SpaceName + if uploadType == "video" { + spaceName = videoSpaceName + } + q := map[string]string{ + "Action": "ApplyUploadInner", + "Version": "2020-11-19", + "SpaceName": spaceName, + "FileType": uploadType, + "IsInner": "1", + "ClientBestHosts": bestHosts, + "NeedFallback": "true", + "FileSize": strconv.FormatInt(fileSize, 10), + "s": randomString(8), + } + var resp applyUploadInnerResp + if err := d.vodRequest(ctx, http.MethodGet, q, nil, auth, &resp); err != nil { + return nil, err + } + if resp.ResponseMetadata.Error.Code != "" { + return nil, fmt.Errorf("wukong apply upload inner failed: %s", resp.ResponseMetadata.Error.Message) + } + return &resp, nil +} + +func (d *Wukong) commitUploadInner(ctx context.Context, auth *uploadAuthTokenResp, spaceName, sessionKey string) (string, error) { + q := map[string]string{ + "Action": "CommitUploadInner", + "Version": "2020-11-19", + "SpaceName": spaceName, + } + body, _ := json.Marshal(map[string]any{ + "SessionKey": sessionKey, + "Functions": []any{}, + }) + var resp commitUploadInnerResp + if err := d.vodRequest(ctx, http.MethodPost, q, body, auth, &resp); err != nil { + return "", err + } + if resp.ResponseMetadata.Error.Code != "" { + return "", fmt.Errorf("wukong commit upload inner failed: %s", resp.ResponseMetadata.Error.Message) + } + if len(resp.Result.Results) > 0 { + status := resp.Result.Results[0].URIStatus + if status != 0 && status != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong commit upload inner failed: uri_status=%d", status) + } + } + return extractVideoVid(&resp), nil +} + +func (d *Wukong) uploadSubmit(ctx context.Context, fatherID, fileName, ext string, fileType int, size int64, md5Hex, storeURI, videoVid string) error { + var resp uploadSubmitResp + body := map[string]any{ + "base_info": map[string]any{ + "father_id": asIDValue(fatherID), + "file_type": fileType, + "size": size, + "extension": ext, + "file_name": fileName, + "is_directory": 0, + "md5": md5Hex, + "slice_md5": md5Hex, + }, + } + switch fileType { + case 3000: + if videoVid != "" { + body["video_info"] = map[string]any{"vid": videoVid} + } + case 2000: + body["image_info"] = map[string]any{"uri": storeURI} + default: + body["general_info"] = map[string]any{"key": storeURI} + } + _, err := d.client.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "aid": d.Aid, + "device_platform": "web", + "language": d.Language, + }). + SetBody(body). + SetResult(&resp). + Post("/netdisk/upload_submit/") + if err != nil { + return err + } + if resp.Code != 0 { + return fmt.Errorf("wukong upload submit failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func extractVideoVid(resp *commitUploadInnerResp) string { + for _, item := range resp.Result.Results { + if item.Vid != "" { + return item.Vid + } + } + if resp.Result.PluginResult != nil { + if vid := findStringByKey(resp.Result.PluginResult, "Vid"); vid != "" { + return vid + } + if vid := findStringByKey(resp.Result.PluginResult, "vid"); vid != "" { + return vid + } + } + return "" +} + +func findStringByKey(v any, key string) string { + switch cur := v.(type) { + case map[string]any: + if val, ok := cur[key]; ok { + if s, ok := val.(string); ok && s != "" { + return s + } + } + for _, child := range cur { + if s := findStringByKey(child, key); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findStringByKey(child, key); s != "" { + return s + } + } + } + return "" +} + +func (d *Wukong) uploadToTOS(ctx context.Context, host, storeURI, auth, storageUser, crc32Hex, fileName string, body io.Reader, size int64) error { + var resp tosUploadResp + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Crc32", crc32Hex). + SetHeader("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, url.QueryEscape(fileName))). + SetHeader("Content-Length", strconv.FormatInt(size, 10)). + SetBody(body). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + _, err := req.Post(uploadURL) + if err != nil { + return err + } + if resp.Code != minUploadSubmitSuccess { + return fmt.Errorf("wukong upload to tos failed: code=%d message=%s", resp.Code, resp.Message) + } + if resp.Data.Crc32 != "" && !strings.EqualFold(resp.Data.Crc32, crc32Hex) { + return fmt.Errorf("wukong upload to tos crc32 mismatch: local=%s remote=%s", crc32Hex, resp.Data.Crc32) + } + return nil +} + +func (d *Wukong) uploadToTOSMultipart(ctx context.Context, host, storeURI, auth, storageUser string, tempFile model.File, size int64, up driver.UpdateProgress) error { + uploadID, err := d.initMultipartUpload(ctx, host, storeURI, auth, storageUser) + if err != nil { + return err + } + + totalParts := int((size + multipartChunkSize - 1) / multipartChunkSize) + if totalParts <= 0 { + return errors.New("invalid multipart parts") + } + parts := make([]string, 0, totalParts) + for i := 0; i < totalParts; i++ { + partNumber := i + 1 + offset := int64(i) * multipartChunkSize + partSize := multipartChunkSize + if remain := size - offset; remain < partSize { + partSize = remain + } + buf := make([]byte, partSize) + n, readErr := tempFile.ReadAt(buf, offset) + if readErr != nil && readErr != io.EOF { + return readErr + } + buf = buf[:n] + crc32Hex := fmt.Sprintf("%08x", crc32.ChecksumIEEE(buf)) + remoteCRC32, err := d.uploadMultipartPart(ctx, host, storeURI, auth, storageUser, uploadID, partNumber, buf, crc32Hex) + if err != nil { + return err + } + if remoteCRC32 != "" && !strings.EqualFold(remoteCRC32, crc32Hex) { + return fmt.Errorf("multipart part crc32 mismatch: part=%d local=%s remote=%s", partNumber, crc32Hex, remoteCRC32) + } + parts = append(parts, fmt.Sprintf("%d:%s", partNumber, crc32Hex)) + up(30 + float64(partNumber)/float64(totalParts)*50) + } + + return d.finishMultipartUpload(ctx, host, storeURI, auth, storageUser, uploadID, strings.Join(parts, ",")) +} + +func (d *Wukong) initMultipartUpload(ctx context.Context, host, storeURI, auth, storageUser string) (string, error) { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetQueryParams(map[string]string{ + "uploadmode": "part", + "phase": "init", + }). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return "", err + } + if resp.Code != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong init multipart upload failed: code=%d message=%s", resp.Code, resp.Message) + } + if resp.Data.UploadID == "" { + return "", errors.New("wukong init multipart upload returns empty uploadid") + } + return resp.Data.UploadID, nil +} + +func (d *Wukong) uploadMultipartPart(ctx context.Context, host, storeURI, auth, storageUser, uploadID string, partNumber int, data []byte, crc32Hex string) (string, error) { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Crc32", crc32Hex). + SetHeader("Content-Length", strconv.Itoa(len(data))). + SetQueryParams(map[string]string{ + "uploadid": uploadID, + "part_number": strconv.Itoa(partNumber), + "phase": "transfer", + }). + SetBody(data). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return "", err + } + if resp.Code != minUploadSubmitSuccess { + return "", fmt.Errorf("wukong multipart transfer failed: code=%d message=%s part=%d", resp.Code, resp.Message, partNumber) + } + return resp.Data.Crc32, nil +} + +func (d *Wukong) finishMultipartUpload(ctx context.Context, host, storeURI, auth, storageUser, uploadID, body string) error { + var resp tosUploadResp + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Host", host). + SetHeader("Referer", webReferer). + SetHeader("Origin", "https://pan.wkbrowser.com"). + SetHeader("Authorization", auth). + SetQueryParams(map[string]string{ + "uploadid": uploadID, + "phase": "finish", + "uploadmode": "part", + }). + SetBody(body). + SetResult(&resp) + if storageUser != "" { + req.SetHeader("X-Storage-U", storageUser) + } + uploadURL := fmt.Sprintf("https://%s/upload/v1/%s", host, storeURI) + _, err := req.Post(uploadURL) + if err != nil { + return err + } + if resp.Code != minUploadSubmitSuccess && resp.Code != 4024 { + return fmt.Errorf("wukong multipart finish failed: code=%d message=%s", resp.Code, resp.Message) + } + return nil +} + +func (d *Wukong) vodRequest(ctx context.Context, method string, query map[string]string, body []byte, auth *uploadAuthTokenResp, resp any) error { + reqURL := vodBaseURL + "/" + amzDate := time.Now().UTC().Format("20060102T150405Z") + dateStamp := amzDate[:8] + headers := map[string]string{ + "x-amz-date": amzDate, + "x-amz-security-token": auth.SessionToken, + } + if method == http.MethodPost { + headers["x-amz-content-sha256"] = hashSHA256Bytes(body) + } + authorization := buildVodAuthorization(method, "/", query, headers, body, auth, dateStamp) + + req := base.NewRestyClient().R(). + SetContext(ctx). + SetHeader("Authorization", authorization). + SetHeader("x-amz-date", amzDate). + SetHeader("x-amz-security-token", auth.SessionToken). + SetQueryParams(query). + SetResult(resp) + if method == http.MethodPost { + req.SetHeader("x-amz-content-sha256", headers["x-amz-content-sha256"]) + req.SetHeader("Content-Type", "text/plain;charset=UTF-8") + req.SetBody(body) + } + _, err := req.Execute(method, reqURL) + return err +} + +func buildVodAuthorization(method, canonicalURI string, query map[string]string, headers map[string]string, body []byte, auth *uploadAuthTokenResp, dateStamp string) string { + canonicalQueryString := getCanonicalQueryStringFromMap(query) + canonicalHeaders, signedHeaders := getCanonicalHeaders(headers) + payloadHash := hashSHA256Bytes(body) + canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQueryString + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash + credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", dateStamp, vodRegion, vodService) + stringToSign := "AWS4-HMAC-SHA256\n" + headers["x-amz-date"] + "\n" + credentialScope + "\n" + hashSHA256String(canonicalRequest) + signingKey := getSigningKey(auth.SecretAccessKey, dateStamp, vodRegion, vodService) + signature := hmacSHA256Hex(signingKey, stringToSign) + return fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", auth.AccessKeyID, credentialScope, signedHeaders, signature) +} + +func getCanonicalQueryStringFromMap(query map[string]string) string { + if len(query) == 0 { + return "" + } + keys := make([]string, 0, len(query)) + for k := range query { + keys = append(keys, k) + } + sort.Strings(keys) + parts := make([]string, 0, len(keys)) + for _, k := range keys { + parts = append(parts, awsURLEncode(k)+"="+awsURLEncode(query[k])) + } + return strings.Join(parts, "&") +} + +func getCanonicalHeaders(headers map[string]string) (string, string) { + keys := make([]string, 0, len(headers)) + for k := range headers { + keys = append(keys, strings.ToLower(k)) + } + sort.Strings(keys) + var h strings.Builder + for _, k := range keys { + h.WriteString(k) + h.WriteString(":") + h.WriteString(strings.TrimSpace(headers[k])) + h.WriteString("\n") + } + return h.String(), strings.Join(keys, ";") +} + +func awsURLEncode(s string) string { + s = url.QueryEscape(s) + return strings.ReplaceAll(s, "+", "%20") +} + +func hashSHA256Bytes(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + +func hashSHA256String(s string) string { + return hashSHA256Bytes([]byte(s)) +} + +func hmacSHA256(key []byte, data string) []byte { + h := hmac.New(sha256.New, key) + _, _ = h.Write([]byte(data)) + return h.Sum(nil) +} + +func hmacSHA256Hex(key []byte, data string) string { + return hex.EncodeToString(hmacSHA256(key, data)) +} + +func getSigningKey(secret, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secret), dateStamp) + kRegion := hmacSHA256(kDate, region) + kService := hmacSHA256(kRegion, service) + return hmacSHA256(kService, "aws4_request") +} + +func collectCandidateHosts(resp *getUploadCandidatesResp) []string { + seen := map[string]struct{}{} + hosts := make([]string, 0, len(resp.Result.Domains)) + add := func(domain vodDomain) { + if domain.Name == "" { + return + } + if _, ok := seen[domain.Name]; ok { + return + } + seen[domain.Name] = struct{}{} + hosts = append(hosts, domain.Name) + } + for _, candidate := range resp.Result.Candidates { + for _, domain := range candidate.Domains { + add(domain) + } + } + for _, domain := range resp.Result.Domains { + add(domain) + } + return hosts +} + +func getStorageUserID(header map[string]any) string { + if header == nil { + return "" + } + if s, ok := header["USER_ID"].(string); ok { + return s + } + if f, ok := header["USER_ID"].(float64); ok { + return strconv.FormatInt(int64(f), 10) + } + return "" +} + +func calcFileMD5AndCRC32(f model.File) (string, string, error) { + if _, err := f.Seek(0, io.SeekStart); err != nil { + return "", "", err + } + md5Hasher := md5.New() + crc := crc32.NewIEEE() + _, err := io.Copy(io.MultiWriter(md5Hasher, crc), f) + if err != nil { + return "", "", err + } + if _, err = f.Seek(0, io.SeekStart); err != nil { + return "", "", err + } + return hex.EncodeToString(md5Hasher.Sum(nil)), fmt.Sprintf("%08x", crc.Sum32()), nil +} + +func detectWukongFileType(mimetype, fileName string) int { + lowerName := strings.ToLower(fileName) + switch { + case strings.HasPrefix(mimetype, "image/"): + return 2000 + case strings.HasPrefix(mimetype, "video/"), strings.HasSuffix(lowerName, ".flv"), strings.HasSuffix(lowerName, ".mkv"): + return 3000 + case strings.HasPrefix(mimetype, "audio/"), strings.HasSuffix(lowerName, ".mp3"), strings.HasSuffix(lowerName, ".m4a"), strings.HasSuffix(lowerName, ".wav"): + return 4000 + case strings.HasSuffix(lowerName, ".zip"), strings.HasSuffix(lowerName, ".rar"), strings.HasSuffix(lowerName, ".7z"), strings.HasSuffix(lowerName, ".tar"), strings.HasSuffix(lowerName, ".gz"), strings.HasSuffix(lowerName, ".tgz"): + return 6000 + default: + return 5000 + } +} + +func randomString(n int) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + if n <= 0 { + return "" + } + buf := make([]byte, n) + if _, err := crand.Read(buf); err == nil { + for i := range buf { + buf[i] = letters[int(buf[i])%len(letters)] + } + return string(buf) + } + + now := uint64(time.Now().UnixNano()) + b := make([]byte, n) + for i := range b { + now = now*6364136223846793005 + 1 + b[i] = letters[int(now%uint64(len(letters)))] + } + return string(b) +} + +func uploadSourceByType(uploadType string) string { + switch uploadType { + case "video": + return "10150001" + case "image": + return "20150001" + default: + return "50150001" + } +} + +func detectUploadType(mimetype, fileName string) string { + lowerName := strings.ToLower(fileName) + if strings.HasPrefix(mimetype, "video/") || strings.HasPrefix(mimetype, "audio/") || + strings.HasSuffix(lowerName, ".flv") || strings.HasSuffix(lowerName, ".mkv") || + strings.HasSuffix(lowerName, ".mp3") || strings.HasSuffix(lowerName, ".m4a") || strings.HasSuffix(lowerName, ".wav") { + return "video" + } + if strings.HasPrefix(mimetype, "image/") { + return "image" + } + return "object" +} + +func chooseCommitSpace(uploadType, authSpace string) string { + if uploadType == "video" { + return videoSpaceName + } + return authSpace +} + +func asIDValue(id string) any { + if n, err := strconv.ParseInt(id, 10, 64); err == nil { + return n + } + return id +} + +func parseUnix(ts int64) time.Time { + if ts <= 0 { + return time.Time{} + } + if ts > 1e12 { + return time.UnixMilli(ts) + } + return time.Unix(ts, 0) +} + +func hasMore(v any) bool { + switch val := v.(type) { + case bool: + return val + case float64: + return val != 0 + case int: + return val != 0 + case int64: + return val != 0 + case string: + return val == "1" || strings.EqualFold(val, "true") + default: + return false + } +} + +func extractURL(data map[string]any) string { + priority := []string{ + "download_url", + "main_url", + "MainUrl", + "MainHTTPUrl", + "url", + "source_url", + "play_url", + "backup_url", + "BackupUrl", + "BackupHTTPUrl", + } + for _, key := range priority { + if url := findURLByKey(data, key); url != "" { + return url + } + } + return findAnyHTTPURL(data) +} + +func extractDetailMainURL(data map[string]any) string { + rawList, ok := data["list"] + if !ok { + return "" + } + list, ok := rawList.([]any) + if !ok || len(list) == 0 { + return "" + } + first, ok := list[0].(map[string]any) + if !ok { + return "" + } + generalInfo, ok := first["general_info"].(map[string]any) + if !ok { + return "" + } + mainURL, ok := generalInfo["main_url"].(string) + if !ok || !isHTTPURL(mainURL) { + return "" + } + return mainURL +} + +func findURLByKey(v any, key string) string { + switch cur := v.(type) { + case map[string]any: + if val, ok := cur[key]; ok { + if s, ok := val.(string); ok && isHTTPURL(s) { + return s + } + if decoded := tryDecodeJSONAny(val); decoded != nil { + if s := findURLByKey(decoded, key); s != "" { + return s + } + } + } + for _, child := range cur { + if s := findURLByKey(child, key); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findURLByKey(child, key); s != "" { + return s + } + } + case string: + if decoded := tryDecodeJSONString(cur); decoded != nil { + return findURLByKey(decoded, key) + } + } + return "" +} + +func findAnyHTTPURL(v any) string { + switch cur := v.(type) { + case string: + if isHTTPURL(cur) { + return cur + } + if decoded := tryDecodeJSONString(cur); decoded != nil { + return findAnyHTTPURL(decoded) + } + case map[string]any: + for _, child := range cur { + if s := findAnyHTTPURL(child); s != "" { + return s + } + } + case []any: + for _, child := range cur { + if s := findAnyHTTPURL(child); s != "" { + return s + } + } + } + return "" +} + +func isHTTPURL(v string) bool { + return strings.HasPrefix(v, "http://") || strings.HasPrefix(v, "https://") +} + +func tryDecodeJSONAny(v any) any { + s, ok := v.(string) + if !ok { + return nil + } + return tryDecodeJSONString(s) +} + +func tryDecodeJSONString(s string) any { + s = strings.TrimSpace(s) + if s == "" { + return nil + } + if !(strings.HasPrefix(s, "{") || strings.HasPrefix(s, "[")) { + return nil + } + var out any + if err := json.Unmarshal([]byte(s), &out); err != nil { + return nil + } + return out +} + +var _ driver.Driver = (*Wukong)(nil) diff --git a/drivers/wukong/meta.go b/drivers/wukong/meta.go new file mode 100644 index 00000000000..451fd7942b4 --- /dev/null +++ b/drivers/wukong/meta.go @@ -0,0 +1,34 @@ +package wukong + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootID + Cookie string `json:"cookie" type:"text" required:"true" help:"Cookie from https://pan.wkbrowser.com/"` + Aid string `json:"aid" default:"590353" help:"aid query param used by web requests"` + Language string `json:"language" default:"zh"` + PageSize int `json:"page_size" type:"number" default:"100"` +} + +var config = driver.Config{ + Name: "WuKongNetdisk", + LocalSort: false, + OnlyLocal: false, + OnlyProxy: false, + NoCache: false, + NoUpload: false, + NeedMs: false, + DefaultRoot: "0", + CheckStatus: false, + Alert: "", + NoOverwriteUpload: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Wukong{} + }) +} diff --git a/drivers/wukong/types.go b/drivers/wukong/types.go new file mode 100644 index 00000000000..86ada25e06d --- /dev/null +++ b/drivers/wukong/types.go @@ -0,0 +1,113 @@ +package wukong + +type filterFileResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + FileList []wukongFile `json:"file_list"` + HasMore any `json:"has_more"` + } `json:"data"` +} + +type wukongFile struct { + FileID int64 `json:"file_id"` + FatherID int64 `json:"father_id"` + IsDirectory int `json:"is_directory"` + FileType int `json:"file_type"` + Size int64 `json:"size"` + FileName string `json:"file_name"` + CreatedAt int64 `json:"created_at"` + UpdatedAt int64 `json:"updated_at"` +} + +type rawResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data map[string]any `json:"data"` +} + +type uploadAuthTokenResp struct { + Code int `json:"code"` + Message string `json:"message"` + CurrentTime int64 `json:"current_time"` + ExpireTime int64 `json:"expire_time"` + SpaceName string `json:"space_name"` + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + SessionToken string `json:"session_token"` +} + +type vodResponseMetadata struct { + RequestID string `json:"RequestId"` + Action string `json:"Action"` + Version string `json:"Version"` + Service string `json:"Service"` + Region string `json:"Region"` + Error struct { + CodeN int `json:"CodeN,omitempty"` + Code string `json:"Code,omitempty"` + Message string `json:"Message,omitempty"` + } `json:"Error,omitempty"` +} + +type vodDomain struct { + Name string `json:"Name"` + Sign string `json:"Sign"` + StoreID string `json:"StoreID"` +} + +type getUploadCandidatesResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + Candidates []struct { + Domains []vodDomain `json:"Domains"` + } `json:"Candidates"` + Domains []vodDomain `json:"Domains"` + } `json:"Result"` +} + +type applyUploadInnerResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + InnerUploadAddress struct { + UploadNodes []struct { + StoreInfos []struct { + StoreURI string `json:"StoreUri"` + Auth string `json:"Auth"` + UploadID string `json:"UploadID"` + StorageHeader map[string]any `json:"StorageHeader"` + } `json:"StoreInfos"` + UploadHost string `json:"UploadHost"` + SessionKey string `json:"SessionKey"` + } `json:"UploadNodes"` + } `json:"InnerUploadAddress"` + } `json:"Result"` +} + +type tosUploadResp struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Crc32 string `json:"crc32"` + UploadID string `json:"uploadid"` + PartNumber string `json:"part_number"` + Etag string `json:"etag"` + } `json:"data"` +} + +type commitUploadInnerResp struct { + ResponseMetadata vodResponseMetadata `json:"ResponseMetadata"` + Result struct { + Results []struct { + URI string `json:"Uri"` + URIStatus int `json:"UriStatus"` + Vid string `json:"Vid"` + } `json:"Results"` + PluginResult any `json:"PluginResult"` + } `json:"Result"` +} + +type uploadSubmitResp struct { + Code int `json:"code"` + Message string `json:"message"` +} diff --git a/drivers/yandex_disk/driver.go b/drivers/yandex_disk/driver.go index 5af9f2e4fb0..6e5ca05c7d0 100644 --- a/drivers/yandex_disk/driver.go +++ b/drivers/yandex_disk/driver.go @@ -106,25 +106,31 @@ func (d *YandexDisk) Remove(ctx context.Context, obj model.Obj) error { return err } -func (d *YandexDisk) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { +func (d *YandexDisk) Put(ctx context.Context, dstDir model.Obj, s model.FileStreamer, up driver.UpdateProgress) error { var resp UploadResp _, err := d.request("/upload", http.MethodGet, func(req *resty.Request) { req.SetQueryParams(map[string]string{ - "path": path.Join(dstDir.GetPath(), stream.GetName()), + "path": path.Join(dstDir.GetPath(), s.GetName()), "overwrite": "true", }) }, &resp) if err != nil { return err } - req, err := http.NewRequest(resp.Method, resp.Href, stream) + reader := driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: s, + UpdateProgress: up, + }) + req, err := http.NewRequestWithContext(ctx, resp.Method, resp.Href, reader) if err != nil { return err } - req = req.WithContext(ctx) - req.Header.Set("Content-Length", strconv.FormatInt(stream.GetSize(), 10)) + req.Header.Set("Content-Length", strconv.FormatInt(s.GetSize(), 10)) req.Header.Set("Content-Type", "application/octet-stream") res, err := base.HttpClient.Do(req) + if err != nil { + return err + } _ = res.Body.Close() return err } diff --git a/drivers/yunpan360/driver.go b/drivers/yunpan360/driver.go new file mode 100644 index 00000000000..c92e918a0ad --- /dev/null +++ b/drivers/yunpan360/driver.go @@ -0,0 +1,286 @@ +package yunpan360 + +import ( + "context" + "errors" + stdpath "path" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Yunpan360 struct { + model.Storage + Addition + + authMu sync.Mutex + cachedOpenAuth *OpenAuthInfo + openAuthExpire time.Time + + cachedCookieSession *CookieDownloadSession + cookieSessionExpire time.Time +} + +func (d *Yunpan360) Config() driver.Config { + return config +} + +func (d *Yunpan360) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Yunpan360) Init(ctx context.Context) error { + if d.PageSize <= 0 { + d.PageSize = 100 + } + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + if d.RootFolderPath == "" { + d.RootFolderPath = "/" + } + d.OrderDirection = strings.ToLower(strings.TrimSpace(d.OrderDirection)) + if d.OrderDirection != "desc" { + d.OrderDirection = "asc" + } + d.AuthType = strings.ToLower(strings.TrimSpace(d.AuthType)) + if d.AuthType == "" { + d.AuthType = authTypeCookie + } + d.SubChannel = strings.TrimSpace(d.SubChannel) + if d.SubChannel == "" { + d.SubChannel = defaultSubChannel + } + d.EcsEnv = strings.ToLower(strings.TrimSpace(d.EcsEnv)) + if d.EcsEnv == "" { + d.EcsEnv = openEnvProd + } + d.Cookie = strings.TrimSpace(d.Cookie) + d.APIKey = strings.TrimSpace(d.APIKey) + d.OwnerQID = strings.TrimSpace(d.OwnerQID) + d.DownloadToken = strings.TrimSpace(d.DownloadToken) + d.cachedOpenAuth = nil + d.openAuthExpire = time.Time{} + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} + + switch d.authMode() { + case authTypeAPIKey: + if d.APIKey == "" { + return errors.New("api_key is empty") + } + _, err := d.openUserInfo(ctx) + return err + case authTypeCookie: + if d.Cookie == "" { + return errors.New("cookie is empty") + } + // Web download URLs require browser-session headers; force local proxying + // so AList can forward Referer/Origin instead of exposing a bare 302 URL. + d.WebProxy = true + _, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + return err + default: + return errors.New("invalid auth_type") + } +} + +func (d *Yunpan360) Drop(ctx context.Context) error { + d.cachedOpenAuth = nil + d.openAuthExpire = time.Time{} + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} + return nil +} + +func (d *Yunpan360) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + dirPath := dir.GetPath() + if dirPath == "" { + dirPath = d.RootFolderPath + } + + objs := make([]model.Obj, 0, d.PageSize) + for page := 0; ; page++ { + resp, err := d.listPage(ctx, dirPath, page, d.PageSize) + if err != nil { + return nil, err + } + pageObjs := resp.Objects(dirPath) + for _, item := range pageObjs { + objs = append(objs, item) + } + if len(pageObjs) == 0 { + break + } + if d.authMode() == authTypeAPIKey { + if len(pageObjs) < d.PageSize { + break + } + continue + } + if !resp.GetHasNextPage() { + break + } + } + return objs, nil +} + +func (d *Yunpan360) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + if d.authMode() == authTypeCookie { + resp, err := d.cookieDownloadURL(ctx, file) + if err != nil { + return nil, err + } + downloadURL := strings.TrimSpace(resp.GetURL()) + if downloadURL == "" { + return nil, errors.New("download url is empty") + } + return &model.Link{ + URL: downloadURL, + Header: map[string][]string{ + "Accept": {"text/javascript, text/html, application/xml, text/xml, */*"}, + "Origin": {baseURL}, + "Referer": {baseURL + indexPath}, + }, + }, nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + resp, err := d.openDownloadURL(ctx, file) + if err != nil { + return nil, err + } + downloadURL := strings.TrimSpace(resp.GetURL()) + if downloadURL == "" { + return nil, errors.New("download url is empty") + } + return &model.Link{URL: downloadURL}, nil +} + +func (d *Yunpan360) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) (model.Obj, error) { + if d.authMode() == authTypeCookie { + fullPath := ensureDirAPIPath(stdpath.Join(parentDir.GetPath(), dirName)) + resp, err := d.cookieMakeDir(ctx, fullPath) + if err != nil { + return nil, err + } + return &YunpanObject{ + Object: model.Object{ + ID: resp.Data.NID, + Path: normalizeRemotePath(fullPath), + Name: dirName, + Size: 0, + Modified: time.Now(), + IsFolder: true, + }, + }, nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + fullPath := ensureDirAPIPath(stdpath.Join(parentDir.GetPath(), dirName)) + resp, err := d.openMakeDir(ctx, fullPath) + if err != nil { + return nil, err + } + obj := &model.Object{ + ID: resp.Data.NID, + Path: normalizeRemotePath(fullPath), + Name: dirName, + Size: 0, + Modified: time.Now(), + IsFolder: true, + } + return obj, nil +} + +func (d *Yunpan360) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + if d.authMode() == authTypeCookie { + srcPath := apiPathForObj(srcObj) + dstPath := ensureDirAPIPath(dstDir.GetPath()) + if err := d.cookieMove(ctx, srcPath, dstPath); err != nil { + return nil, err + } + return cloneObj(srcObj, stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetName()), nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + srcPath := apiPathForObj(srcObj) + dstPath := ensureDirAPIPath(dstDir.GetPath()) + if err := d.openMove(ctx, srcPath, dstPath); err != nil { + return nil, err + } + return cloneObj(srcObj, stdpath.Join(dstDir.GetPath(), srcObj.GetName()), srcObj.GetName()), nil +} + +func (d *Yunpan360) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + if d.authMode() == authTypeCookie { + targetName := strings.TrimSuffix(strings.TrimSpace(newName), "/") + if targetName == "" { + return nil, errors.New("new name is empty") + } + if err := d.cookieRename(ctx, srcObj, targetName); err != nil { + return nil, err + } + parentPath := stdpath.Dir(srcObj.GetPath()) + if parentPath == "." { + parentPath = "/" + } + return cloneObj(srcObj, stdpath.Join(parentPath, targetName), targetName), nil + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + + srcPath := apiPathForObj(srcObj) + targetName := newName + if srcObj.IsDir() { + targetName = ensureDirSuffix(newName) + } + if err := d.openRename(ctx, srcPath, targetName); err != nil { + return nil, err + } + + parentPath := stdpath.Dir(srcObj.GetPath()) + if parentPath == "." { + parentPath = "/" + } + return cloneObj(srcObj, stdpath.Join(parentPath, strings.TrimSuffix(newName, "/")), strings.TrimSuffix(newName, "/")), nil +} + +func (d *Yunpan360) Remove(ctx context.Context, obj model.Obj) error { + if d.authMode() == authTypeCookie { + return d.cookieRecycle(ctx, obj) + } + if d.authMode() != authTypeAPIKey { + return errs.NotImplement + } + return d.openDelete(ctx, apiPathForObj(obj)) +} + +func (d *Yunpan360) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if d.authMode() == authTypeCookie { + return nil, errs.NotImplement + } + if d.authMode() != authTypeAPIKey { + return nil, errs.NotImplement + } + return d.putOpenFile(ctx, dstDir, file, up) +} + +func (d *Yunpan360) authMode() string { + if d.AuthType == authTypeAPIKey { + return authTypeAPIKey + } + return authTypeCookie +} + +var _ driver.Driver = (*Yunpan360)(nil) diff --git a/drivers/yunpan360/meta.go b/drivers/yunpan360/meta.go new file mode 100644 index 00000000000..5c4e1c8d245 --- /dev/null +++ b/drivers/yunpan360/meta.go @@ -0,0 +1,34 @@ +package yunpan360 + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + AuthType string `json:"auth_type" type:"select" options:"cookie,api_key" default:"cookie"` + Cookie string `json:"cookie" type:"text" help:"Cookie copied from a logged-in yunpan.com session; used when auth_type=cookie"` + OwnerQID string `json:"owner_qid" type:"text" help:"Optional owner_qid for cookie-mode download; leave empty to auto-detect"` + DownloadToken string `json:"download_token" type:"text" help:"Optional web token for cookie-mode download; leave empty to auto-detect"` + APIKey string `json:"api_key" type:"text" help:"360 AI YunPan API key; used when auth_type=api_key"` + EcsEnv string `json:"ecs_env" type:"select" options:"prod,test,hgtest" default:"prod"` + SubChannel string `json:"sub_channel" default:"open"` + OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"` + PageSize int `json:"page_size" type:"number" default:"100" help:"List page size"` +} + +var config = driver.Config{ + Name: "360AIYunPan", + LocalSort: false, + CheckStatus: true, + NoUpload: false, + DefaultRoot: "/", + Alert: "info|api_key mode supports list/link/upload/mkdir/rename/move/delete; cookie mode supports list/link/mkdir/rename/move/delete only, and forces web proxy because direct download URLs require web headers.", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Yunpan360{} + }) +} diff --git a/drivers/yunpan360/types.go b/drivers/yunpan360/types.go new file mode 100644 index 00000000000..f5c1b7a2135 --- /dev/null +++ b/drivers/yunpan360/types.go @@ -0,0 +1,462 @@ +package yunpan360 + +import ( + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + authTypeCookie = "cookie" + authTypeAPIKey = "api_key" + openEnvProd = "prod" + defaultSubChannel = "open" + openSignSecret = "e7b24b112a44fdd9ee93bdf998c6ca0e" + openClientID = "e4757e933b6486c08ed206ecb6d5d9e684fcb4e2" + openClientSecret = "885fd3231f1c1e37c9f462261a09b8c38cde0c2b" + openClientSecretQA = "b11b8fff1c75a5d227c8cc93aaeb0bb70c8eee47" +) + +type BaseResp struct { + Errno int `json:"errno"` + Errmsg string `json:"errmsg"` +} + +type CookieDownloadSession struct { + OwnerQID string + Token string +} + +type ListResp interface { + Objects(parentPath string) []model.Obj + GetHasNextPage() bool +} + +type CookieListResp struct { + BaseResp + Token string `json:"token"` + OwnerQid string `json:"owner_qid"` + Qid string `json:"qid"` + Data []ListItem `json:"data"` + HasNextPage bool `json:"has_next_page"` +} + +func (r *CookieListResp) Objects(parentPath string) []model.Obj { + ownerQID := r.GetOwnerQID() + return utils.MustSliceConvert(r.Data, func(src ListItem) model.Obj { + return src.toObj(parentPath, ownerQID, r.Token) + }) +} + +func (r *CookieListResp) GetHasNextPage() bool { + return r.HasNextPage +} + +func (r *CookieListResp) GetOwnerQID() string { + return firstNonEmpty(r.OwnerQid, r.Qid) +} + +type ListItem struct { + NID string `json:"nid"` + FileName string `json:"file_name"` + FilePath string `json:"file_path"` + FileSize string `json:"file_size"` + IsDir bool `json:"is_dir"` + Fhash string `json:"fhash"` + CreateTime string `json:"create_time"` + ModifyTime string `json:"modify_time"` + Mtime string `json:"mtime"` + ServerTime string `json:"server_time"` + Preview string `json:"preview"` + Thumb string `json:"thumb"` + SrcPic string `json:"srcpic"` + OwnerQid string `json:"owner_qid"` + Qid string `json:"qid"` + Token string `json:"token"` +} + +func (i ListItem) toObj(parentPath, ownerQID, token string) model.Obj { + objPath := normalizeRemotePath(i.FilePath) + if objPath == "" || !pathLooksLikeObject(objPath, i.FileName) { + objPath = joinRemotePath(parentPath, i.FileName) + } + thumb := "" + if !i.IsDir { + thumb = absoluteURL(firstNonEmpty(i.Thumb, i.SrcPic, i.Preview)) + } + + return &YunpanObject{ + Object: model.Object{ + ID: i.NID, + Path: objPath, + Name: i.FileName, + Size: parseSize(i.FileSize), + Modified: parseYunpanTime(i.ModifyTime, i.Mtime), + Ctime: parseYunpanTime(i.CreateTime, i.ServerTime), + IsFolder: i.IsDir, + HashInfo: parseHash(i.Fhash), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + OwnerQID: firstNonEmpty(i.OwnerQid, i.Qid, ownerQID), + DownloadToken: firstNonEmpty(i.Token, token), + } +} + +func parseSize(raw string) int64 { + size, _ := strconv.ParseInt(raw, 10, 64) + return size +} + +func parseHash(raw string) utils.HashInfo { + if len(raw) == 40 { + return utils.NewHashInfo(utils.SHA1, raw) + } + return utils.HashInfo{} +} + +func parseYunpanTime(unixStr, text string) time.Time { + if t := parseUnixTime(unixStr); !t.IsZero() { + return t + } + return parseTextTime(text) +} + +func parseUnixTime(raw string) time.Time { + if raw != "" { + sec, err := strconv.ParseInt(raw, 10, 64) + if err == nil && sec > 0 { + return time.Unix(sec, 0) + } + } + return time.Time{} +} + +func parseTextTime(text string) time.Time { + if text == "" { + return time.Time{} + } + t, err := time.ParseInLocation("2006-01-02 15:04:05", text, utils.CNLoc) + if err == nil { + return t + } + return time.Time{} +} + +func normalizeRemotePath(p string) string { + if p == "" { + return "" + } + if p != "/" { + p = strings.TrimSuffix(p, "/") + } + return utils.FixAndCleanPath(p) +} + +type OpenAuthResp struct { + BaseResp + Data struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + AccessTokenExpire int64 `json:"access_token_expire"` + Qid string `json:"qid"` + } `json:"data"` +} + +type OpenAuthInfo struct { + AccessToken string + Qid string + Token string + SubChannel string +} + +type OpenListResp struct { + BaseResp + Data struct { + NodeList []OpenNode `json:"node_list"` + List []OpenNode `json:"list"` + Data []OpenNode `json:"data"` + TotalCount int `json:"total_count"` + Total int `json:"total"` + PageNum int `json:"page_num"` + } `json:"data"` +} + +func (r *OpenListResp) Objects(parentPath string) []model.Obj { + nodes := r.Data.NodeList + if len(nodes) == 0 { + nodes = r.Data.List + } + if len(nodes) == 0 { + nodes = r.Data.Data + } + return utils.MustSliceConvert(nodes, func(src OpenNode) model.Obj { + return src.toObj(parentPath) + }) +} + +func (r *OpenListResp) GetHasNextPage() bool { + total := r.Data.TotalCount + if total <= 0 { + total = r.Data.Total + } + if total <= 0 { + return false + } + loaded := len(r.Data.NodeList) + if loaded == 0 { + loaded = len(r.Data.List) + } + if loaded == 0 { + loaded = len(r.Data.Data) + } + return loaded > 0 && loaded < total +} + +type OpenNode struct { + NID string `json:"nid"` + Name string `json:"name"` + FName string `json:"fname"` + Path string `json:"path"` + FPath string `json:"fpath"` + Type interface{} `json:"type"` + IsDir interface{} `json:"is_dir"` + CountSize interface{} `json:"count_size"` + Size interface{} `json:"size"` + CreateTime interface{} `json:"create_time"` + ModifyTime interface{} `json:"modify_time"` + MTime interface{} `json:"mtime"` + FileHash string `json:"file_hash"` + Fhash string `json:"fhash"` + Thumb string `json:"thumb"` + Preview string `json:"preview"` + SrcPic string `json:"srcpic"` +} + +func (n OpenNode) toObj(parentPath string) model.Obj { + name := firstNonEmpty(strings.TrimSpace(n.Name), strings.TrimSpace(n.FName)) + objPath := normalizeRemotePath(firstNonEmpty(n.FPath, n.Path)) + if objPath == "" || !pathLooksLikeObject(objPath, name) { + objPath = joinRemotePath(parentPath, name) + } + isDir := parseOpenDir(n.IsDir, n.Type) + thumb := "" + if !isDir { + thumb = absoluteURL(firstNonEmpty(n.Thumb, n.SrcPic, n.Preview)) + } + + return &YunpanObject{ + Object: model.Object{ + ID: n.NID, + Path: objPath, + Name: name, + Size: parseAnySize(n.CountSize, n.Size), + Modified: parseAnyTime(n.ModifyTime, n.MTime), + Ctime: parseAnyTime(n.CreateTime), + IsFolder: isDir, + HashInfo: parseHash(firstNonEmpty(n.FileHash, n.Fhash)), + }, + Thumbnail: model.Thumbnail{ + Thumbnail: thumb, + }, + } +} + +type OpenUserInfoResp struct { + BaseResp + Data map[string]interface{} `json:"data"` +} + +type OpenMkdirResp struct { + BaseResp + Data struct { + NID string `json:"nid"` + } `json:"data"` +} + +type CookieMkdirResp struct { + BaseResp + Data struct { + NID string `json:"nid"` + } `json:"data"` +} + +type CookieMoveResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + IsAsync bool `json:"is_async"` + } `json:"data"` +} + +type CookieRecycleResp struct { + BaseResp + Data struct { + TaskID string `json:"task_id"` + IsAsync bool `json:"is_async"` + } `json:"data"` +} + +type CookieAsyncQueryResp struct { + BaseResp + Data map[string]CookieAsyncTask `json:"data"` +} + +type CookieAsyncTask struct { + MessageID string `json:"message_id"` + SendTime string `json:"send_time"` + Status int `json:"status"` + Action string `json:"action"` + Errno int `json:"errno"` + Errstr string `json:"errstr"` + Result string `json:"result"` +} + +type CookieDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"download_url"` + Store string `json:"store"` + Host string `json:"host"` + } `json:"data"` +} + +func (r *CookieDownloadResp) GetURL() string { + return r.Data.DownloadURL +} + +type OpenDownloadResp struct { + BaseResp + Data struct { + DownloadURL string `json:"downloadUrl"` + } `json:"data"` + DownloadURL string `json:"downloadUrl"` +} + +func (r *OpenDownloadResp) GetURL() string { + return firstNonEmpty(r.Data.DownloadURL, r.DownloadURL) +} + +type YunpanObject struct { + model.Object + model.Thumbnail + OwnerQID string + DownloadToken string +} + +func parseAnySize(values ...interface{}) int64 { + for _, value := range values { + switch v := value.(type) { + case string: + if v == "" { + continue + } + size, err := strconv.ParseInt(v, 10, 64) + if err == nil { + return size + } + case float64: + return int64(v) + case int64: + return v + case int: + return int64(v) + } + } + return 0 +} + +func parseAnyTime(values ...interface{}) time.Time { + for _, value := range values { + switch v := value.(type) { + case string: + if t := parseUnixTime(v); !t.IsZero() { + return t + } + if t := parseTextTime(v); !t.IsZero() { + return t + } + case float64: + if v > 0 { + return time.Unix(int64(v), 0) + } + case int64: + if v > 0 { + return time.Unix(v, 0) + } + case int: + if v > 0 { + return time.Unix(int64(v), 0) + } + } + } + return time.Time{} +} + +func parseOpenDir(values ...interface{}) bool { + for _, value := range values { + switch v := value.(type) { + case bool: + if v { + return true + } + case string: + switch strings.ToLower(strings.TrimSpace(v)) { + case "1", "true", "dir", "folder": + return true + } + case float64: + if int64(v) == 1 { + return true + } + case int64: + if v == 1 { + return true + } + case int: + if v == 1 { + return true + } + } + } + return false +} + +func pathLooksLikeObject(objPath, name string) bool { + if objPath == "" || name == "" { + return false + } + return strings.TrimSuffix(stdPathBase(objPath), "/") == strings.TrimSuffix(name, "/") +} + +func stdPathBase(p string) string { + if p == "/" { + return "/" + } + idx := strings.LastIndex(strings.TrimSuffix(p, "/"), "/") + if idx < 0 { + return p + } + return p[idx+1:] +} + +func joinRemotePath(parentPath, name string) string { + parentPath = normalizeRemotePath(parentPath) + if parentPath == "" { + parentPath = "/" + } + return normalizeRemotePath(strings.TrimSuffix(parentPath, "/") + "/" + strings.TrimPrefix(name, "/")) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} diff --git a/drivers/yunpan360/upload.go b/drivers/yunpan360/upload.go new file mode 100644 index 00000000000..38c3fded57b --- /dev/null +++ b/drivers/yunpan360/upload.go @@ -0,0 +1,926 @@ +package yunpan360 + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + stdpath "path" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + aliststream "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + yunpanUploadChunkSize = int64(512 * 1024) + yunpanUploadBoundary = "WebKitFormBoundaryQ5OJVvzZwEkg4ttY" + yunpanUploadVersion = "1.0.1" + yunpanUploadDevType = "ecs_openapi" + yunpanUploadDevName = "EYUN_WEB_UPLOAD" +) + +type openUploadPlan struct { + DirPath string + TargetPath string + FileName string + Size int64 + FileHash string + FileSHA1 string + FileSum string + CreatedAt int64 + DeviceID string + Chunks []openUploadChunk +} + +type openUploadChunk struct { + Index int + Offset int64 + Size int64 + Hash string +} + +type openUploadDetectResp struct { + BaseResp + Data struct { + Exists []openUploadDuplicate `json:"exists"` + IsSlice int `json:"is_slice"` + } `json:"data"` +} + +type openUploadDuplicate struct { + FullName string `json:"fullName"` +} + +type openUploadAddressResp struct { + BaseResp + Data struct { + HTTP string `json:"http"` + Addr1 string `json:"addr_1"` + Addr2 string `json:"addr_2"` + Backup string `json:"backup"` + TK string `json:"tk"` + GroupSize string `json:"group_size"` + AutoCommit interface{} `json:"autoCommit"` + IsHTTPS interface{} `json:"is_https"` + } `json:"data"` +} + +type openUploadRequestResp struct { + BaseResp + Data struct { + Tid string `json:"tid"` + BlockInfo []map[string]interface{} `json:"block_info"` + } `json:"data"` +} + +type openUploadFinalizeResp struct { + BaseResp + Data map[string]interface{} `json:"data"` +} + +type uploadEnvelope struct { + Errno *int `json:"errno"` + Errmsg string `json:"errmsg"` + Data json.RawMessage `json:"data"` +} + +func (d *Yunpan360) putOpenFile(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return nil, err + } + + dirPath := dstDir.GetPath() + if dirPath == "" { + dirPath = d.RootFolderPath + } + dirPath = ensureDirAPIPath(dirPath) + targetPath := joinRemotePath(dirPath, file.GetName()) + + cached, err := d.cacheUploadSource(ctx, file, progressRange(up, 0, 5)) + if err != nil { + return nil, err + } + defer func() { + _, _ = cached.Seek(0, io.SeekStart) + }() + + plan, err := d.buildUploadPlan(ctx, cached, targetPath, file.GetSize(), file.ModTime(), progressRange(up, 5, 10)) + if err != nil { + return nil, err + } + + detectResp, err := d.openDetectUpload(ctx, auth, plan) + if err != nil { + return nil, err + } + if detectResp.Data.IsSlice == 0 { + detectResp.Data.IsSlice = 1 + } + + addrResp, err := d.openGetUploadAddress(ctx, auth, plan) + if err != nil { + return nil, err + } + + finalResp := &openUploadFinalizeResp{BaseResp: BaseResp{Errno: 0}, Data: map[string]interface{}{}} + if strings.TrimSpace(addrResp.Data.HTTP) == "" { + finalResp.Data = map[string]interface{}{"autoCommit": true} + if tk := strings.TrimSpace(addrResp.Data.TK); tk != "" { + finalResp.Data["tk"] = tk + finalResp.Data["autoCommit"] = false + } + } else { + reqResp, err := d.openRequestUpload(ctx, auth, plan, addrResp) + if err != nil { + return nil, err + } + if err := d.openUploadBlocks(ctx, auth, cached, plan, addrResp, reqResp, up); err != nil { + return nil, err + } + finalResp, err = d.openCommitUpload(ctx, auth, plan, addrResp, reqResp) + if err != nil { + return nil, err + } + } + + if err := d.openFinalizeUpload(ctx, auth, finalResp); err != nil { + return nil, err + } + if up != nil { + up(100) + } + + obj, err := d.findUploadedObject(ctx, targetPath) + if err == nil { + return obj, nil + } + if !errors.Is(err, errs.ObjectNotFound) { + return nil, err + } + + return &model.Object{ + Path: normalizeRemotePath(targetPath), + Name: file.GetName(), + Size: file.GetSize(), + Modified: time.Now(), + Ctime: time.Now(), + HashInfo: utils.NewHashInfo(utils.SHA1, firstNonEmpty(plan.FileSHA1, plan.FileHash)), + }, nil +} + +func (d *Yunpan360) cacheUploadSource(ctx context.Context, file model.FileStreamer, up driver.UpdateProgress) (model.File, error) { + if cached := file.GetFile(); cached != nil { + _, _ = cached.Seek(0, io.SeekStart) + if up != nil { + up(100) + } + return cached, nil + } + if up == nil { + return file.CacheFullInTempFile() + } + return aliststream.CacheFullInTempFileAndUpdateProgress(file, up) +} + +func (d *Yunpan360) buildUploadPlan(ctx context.Context, cached model.File, targetPath string, size int64, modTime time.Time, up driver.UpdateProgress) (*openUploadPlan, error) { + if _, err := cached.Seek(0, io.SeekStart); err != nil { + return nil, err + } + + createdAt := time.Now().Unix() + if !modTime.IsZero() { + createdAt = modTime.Unix() + } + plan := &openUploadPlan{ + DirPath: ensureDirAPIPath(stdpath.Dir(targetPath)), + TargetPath: normalizeRemotePath(targetPath), + FileName: stdpath.Base(targetPath), + Size: size, + CreatedAt: createdAt, + DeviceID: sha1HexString("node-sdk-" + runtime.Version()), + } + + if plan.DirPath == "./" || plan.DirPath == "." { + plan.DirPath = "/" + } + + totalChunks := 0 + if size > 0 { + totalChunks = int((size + yunpanUploadChunkSize - 1) / yunpanUploadChunkSize) + } + chunks := make([]openUploadChunk, 0, totalChunks) + var hashConcat strings.Builder + var hashed int64 + + for idx := 0; idx < totalChunks; idx++ { + if err := ctx.Err(); err != nil { + return nil, err + } + offset := int64(idx) * yunpanUploadChunkSize + chunkSize := yunpanUploadChunkSize + if remain := size - offset; remain < chunkSize { + chunkSize = remain + } + + chunkHash, err := sha1HexReader(io.NewSectionReader(cached, offset, chunkSize)) + if err != nil { + return nil, err + } + + chunks = append(chunks, openUploadChunk{ + Index: idx + 1, + Offset: offset, + Size: chunkSize, + Hash: chunkHash, + }) + hashConcat.WriteString(chunkHash) + hashed += chunkSize + reportByteProgress(up, hashed, size) + } + + plan.Chunks = chunks + plan.FileHash = sha1HexString(hashConcat.String()) + if _, err := cached.Seek(0, io.SeekStart); err != nil { + return nil, err + } + sha1Hasher := sha1.New() + md5Hasher := md5.New() + if _, err := io.Copy(io.MultiWriter(sha1Hasher, md5Hasher), cached); err != nil { + return nil, err + } + plan.FileSHA1 = hex.EncodeToString(sha1Hasher.Sum(nil)) + plan.FileSum = hex.EncodeToString(md5Hasher.Sum(nil)) + if size == 0 && up != nil { + up(100) + } + return plan, nil +} + +func (d *Yunpan360) openDetectUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan) (*openUploadDetectResp, error) { + payload, err := json.Marshal([]map[string]interface{}{ + {"fname": plan.FileName, "fsize": plan.Size}, + }) + if err != nil { + return nil, err + } + + signParams := map[string]string{ + "data": string(payload), + "path": plan.DirPath, + } + body, contentType, err := createMultipartForm("", map[string]string{ + "qid": auth.Qid, + "data": string(payload), + "path": plan.DirPath, + "sign": openSign(auth.AccessToken, auth.Qid, "Sync.detectFileExists", signParams), + }, nil) + if err != nil { + return nil, err + } + + var resp openUploadDetectResp + err = d.uploadRequest(ctx, http.MethodPost, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.detectFileExists", nil), auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openGetUploadAddress(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan) (*openUploadAddressResp, error) { + query := d.uploadCookieParams(auth, plan, "") + signParams := map[string]string{ + "access_token": auth.AccessToken, + "fhash": plan.FileHash, + "fname": plan.TargetPath, + "fsize": strconv.FormatInt(plan.Size, 10), + } + query["sign"] = openSign(auth.AccessToken, auth.Qid, "Sync.getUploadFileAddr", signParams) + + var resp openUploadAddressResp + err := d.uploadRequest(ctx, http.MethodGet, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.getUploadFileAddr", query), auth.AccessToken, "", nil, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openRequestUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan, addrResp *openUploadAddressResp) (*openUploadRequestResp, error) { + chunkInfos := make([]map[string]interface{}, 0, len(plan.Chunks)) + for _, chunk := range plan.Chunks { + chunkInfos = append(chunkInfos, map[string]interface{}{ + "bhash": chunk.Hash, + "bidx": chunk.Index, + "boffset": chunk.Offset, + "bsize": chunk.Size, + }) + } + payload, err := json.Marshal(map[string]interface{}{ + "request": map[string]interface{}{ + "block_info": chunkInfos, + }, + }) + if err != nil { + return nil, err + } + + body, contentType, err := createMultipartForm( + yunpanUploadBoundary, + d.uploadCookieParams(auth, plan, strings.TrimSpace(addrResp.Data.TK)), + &multipartFile{ + FieldName: "file", + FileName: "file.dat", + ContentType: "application/octet-stream", + Content: payload, + }, + ) + if err != nil { + return nil, err + } + + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.request4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var resp openUploadRequestResp + err = d.uploadRequest(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openUploadBlocks(ctx context.Context, auth *OpenAuthInfo, cached model.File, plan *openUploadPlan, addrResp *openUploadAddressResp, reqResp *openUploadRequestResp, up driver.UpdateProgress) error { + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.block4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var uploaded int64 + for _, chunk := range plan.Chunks { + info := reqResp.blockInfoForChunk(chunk.Index) + if info.found() > 0 { + uploaded += chunk.Size + reportUploadProgress(up, uploaded, plan.Size) + continue + } + + chunkBytes := make([]byte, chunk.Size) + if _, err := cached.ReadAt(chunkBytes, chunk.Offset); err != nil && !errors.Is(err, io.EOF) { + return err + } + + fields := map[string]string{ + "bhash": chunk.Hash, + "bidx": strconv.Itoa(chunk.Index), + "boffset": strconv.FormatInt(chunk.Offset, 10), + "bsize": strconv.FormatInt(chunk.Size, 10), + "filename": plan.TargetPath, + "filesize": strconv.FormatInt(plan.Size, 10), + "q": info.stringValue("q"), + "t": info.stringValue("t"), + "token": auth.Token, + "tid": info.stringValue("tid"), + } + for key, value := range info.extraFields() { + fields[key] = value + } + + body, contentType, err := createMultipartForm( + yunpanUploadBoundary, + fields, + &multipartFile{ + FieldName: "file", + FileName: "file.dat", + ContentType: "application/octet-stream", + Content: chunkBytes, + }, + ) + if err != nil { + return err + } + + chunkStart := uploaded + chunkSize := chunk.Size + err = d.uploadRequestWithProgress(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, func(p float64) { + done := chunkStart + int64(float64(chunkSize)*(p/100.0)) + reportUploadProgress(up, done, plan.Size) + }, nil) + if err != nil { + return err + } + + uploaded += chunk.Size + reportUploadProgress(up, uploaded, plan.Size) + } + return nil +} + +func (d *Yunpan360) openCommitUpload(ctx context.Context, auth *OpenAuthInfo, plan *openUploadPlan, addrResp *openUploadAddressResp, reqResp *openUploadRequestResp) (*openUploadFinalizeResp, error) { + body, contentType, err := createMultipartForm("", map[string]string{ + "q": "", + "t": "", + "token": auth.Token, + "tid": strings.TrimSpace(reqResp.Data.Tid), + }, nil) + if err != nil { + return nil, err + } + + url := buildJSQueryURL(d.uploadBaseURL(addrResp), "Upload.commit4Web", d.uploadDataParams(auth, plan)) + url = appendHostQuery(url, addrResp.Data.HTTP) + + var resp openUploadFinalizeResp + err = d.uploadRequest(ctx, http.MethodPost, url, auth.AccessToken, contentType, body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openFinalizeUpload(ctx context.Context, auth *OpenAuthInfo, resp *openUploadFinalizeResp) error { + if resp == nil || resp.autoCommit() { + return nil + } + tk := strings.TrimSpace(resp.stringValue("tk")) + if tk == "" { + return errors.New("upload tk is empty") + } + + signParams := map[string]string{ + "tk": tk, + } + body, contentType, err := createMultipartForm("", map[string]string{ + "qid": auth.Qid, + "tk": tk, + "sign": openSign(auth.AccessToken, auth.Qid, "Sync.addFileToApi", signParams), + }, nil) + if err != nil { + return err + } + + return d.uploadRequest(ctx, http.MethodPost, buildJSQueryURL(openAPIURL(d.EcsEnv), "Sync.addFileToApi", nil), auth.AccessToken, contentType, body, nil) +} + +func (d *Yunpan360) findUploadedObject(ctx context.Context, targetPath string) (model.Obj, error) { + targetPath = normalizeRemotePath(targetPath) + parentPath := normalizeRemotePath(stdpath.Dir(targetPath)) + if parentPath == "." || parentPath == "" { + parentPath = "/" + } + targetName := stdpath.Base(targetPath) + + for page := 0; ; page++ { + resp, err := d.listPage(ctx, parentPath, page, d.PageSize) + if err != nil { + return nil, err + } + pageObjs := resp.Objects(parentPath) + for _, obj := range pageObjs { + if normalizeRemotePath(obj.GetPath()) == targetPath || obj.GetName() == targetName { + return obj, nil + } + } + if len(pageObjs) == 0 || len(pageObjs) < d.PageSize { + break + } + } + return nil, errs.ObjectNotFound +} + +func (d *Yunpan360) uploadCookieParams(auth *OpenAuthInfo, plan *openUploadPlan, uploadTK string) map[string]string { + params := map[string]string{ + "owner_qid": auth.Qid, + "fname": plan.TargetPath, + "fsize": strconv.FormatInt(plan.Size, 10), + "fctime": strconv.FormatInt(plan.CreatedAt, 10), + "fmtime": strconv.FormatInt(plan.CreatedAt, 10), + "fhash": plan.FileHash, + "qid": auth.Qid, + "fattr": "0", + "token": auth.Token, + "devtype": yunpanUploadDevType, + } + if uploadTK != "" { + params["tk"] = uploadTK + } + return params +} + +func (d *Yunpan360) uploadDataParams(auth *OpenAuthInfo, plan *openUploadPlan) map[string]string { + return map[string]string{ + "owner_qid": auth.Qid, + "qid": auth.Qid, + "devtype": yunpanUploadDevType, + "devid": plan.DeviceID, + "v": yunpanUploadVersion, + "ofmt": "json", + "devname": yunpanUploadDevName, + "rtick": strconv.FormatInt(time.Now().UnixMilli(), 10), + } +} + +func (d *Yunpan360) uploadBaseURL(addrResp *openUploadAddressResp) string { + host := "" + isHTTPS := false + if addrResp != nil { + host = strings.TrimSpace(addrResp.Data.HTTP) + isHTTPS = parseOpenDir(addrResp.Data.IsHTTPS) + } + scheme := "http" + if isHTTPS { + scheme = "https" + } + if host == "" { + return openAPIURL(d.EcsEnv) + } + return fmt.Sprintf("%s://%s/intf.php", scheme, host) +} + +func (d *Yunpan360) uploadRequest(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, out interface{}) error { + return d.uploadRequestWithProgress(ctx, method, reqURL, accessToken, contentType, body, nil, out) +} + +func (d *Yunpan360) uploadRequestWithProgress(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, progress driver.UpdateProgress, out interface{}) error { + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + if err := sleepWithContext(ctx, time.Duration(attempt)*500*time.Millisecond); err != nil { + return err + } + } + err := d.doUploadRequest(ctx, method, reqURL, accessToken, contentType, body, progress, out) + if err == nil { + return nil + } + lastErr = err + if ctx.Err() != nil { + return ctx.Err() + } + } + return lastErr +} + +func (d *Yunpan360) doUploadRequest(ctx context.Context, method, reqURL, accessToken, contentType string, body []byte, progress driver.UpdateProgress, out interface{}) error { + var bodyReader io.ReadCloser + if body != nil { + reader := &driver.SimpleReaderWithSize{ + Reader: bytes.NewReader(body), + Size: int64(len(body)), + } + if progress != nil { + bodyReader = driver.NewLimitedUploadStream(ctx, &driver.ReaderUpdatingProgress{ + Reader: reader, + UpdateProgress: progress, + }) + } else { + bodyReader = driver.NewLimitedUploadStream(ctx, reader) + } + } + + req, err := http.NewRequestWithContext(ctx, method, reqURL, bodyReader) + if err != nil { + if bodyReader != nil { + _ = bodyReader.Close() + } + return err + } + if body != nil { + req.ContentLength = int64(len(body)) + } + req.Header.Set("Accept", "application/json") + if accessToken != "" { + req.Header.Set("Access-Token", accessToken) + } + if contentType != "" { + req.Header.Set("Content-Type", contentType) + } + + resp, err := base.HttpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + if resp.StatusCode >= http.StatusBadRequest { + return fmt.Errorf("yunpan upload request failed: status=%d body=%s", resp.StatusCode, strings.TrimSpace(string(respBody))) + } + return decodeUploadResp(respBody, out) +} + +func decodeUploadResp(body []byte, out interface{}) error { + var env uploadEnvelope + if err := utils.Json.Unmarshal(body, &env); err != nil { + return err + } + if env.Errno != nil && *env.Errno != 0 { + if env.Errmsg == "" { + return fmt.Errorf("yunpan upload request failed: errno=%d", *env.Errno) + } + return errors.New(env.Errmsg) + } + if env.Errno == nil && strings.TrimSpace(env.Errmsg) != "" && len(env.Data) > 0 && string(env.Data) == "[]" { + return errors.New(env.Errmsg) + } + if out == nil { + return nil + } + if err := utils.Json.Unmarshal(body, out); err != nil { + if strings.TrimSpace(env.Errmsg) != "" { + return errors.New(env.Errmsg) + } + return err + } + return nil +} + +type multipartFile struct { + FieldName string + FileName string + ContentType string + Content []byte +} + +func createMultipartForm(boundary string, fields map[string]string, file *multipartFile) ([]byte, string, error) { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + if boundary != "" { + if err := writer.SetBoundary(boundary); err != nil { + return nil, "", err + } + } + + for key, value := range fields { + if err := writer.WriteField(key, value); err != nil { + return nil, "", err + } + } + + if file != nil { + partHeader := make(textproto.MIMEHeader) + partHeader.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`, file.FieldName, file.FileName)) + partHeader.Set("Content-Type", firstNonEmpty(file.ContentType, "application/octet-stream")) + part, err := writer.CreatePart(partHeader) + if err != nil { + return nil, "", err + } + if _, err := part.Write(file.Content); err != nil { + return nil, "", err + } + } + + if err := writer.Close(); err != nil { + return nil, "", err + } + return body.Bytes(), writer.FormDataContentType(), nil +} + +func buildJSQueryURL(baseURL, method string, params map[string]string) string { + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + + var builder strings.Builder + builder.WriteString(baseURL) + if strings.Contains(baseURL, "?") { + builder.WriteByte('&') + } else { + builder.WriteByte('?') + } + builder.WriteString("method=") + builder.WriteString(jsQueryEscape(method)) + for _, key := range keys { + value := params[key] + if value == "" { + continue + } + builder.WriteByte('&') + builder.WriteString(key) + builder.WriteByte('=') + builder.WriteString(jsQueryEscape(value)) + } + return builder.String() +} + +func buildQueryURL(baseURL string, params map[string]string) string { + u, err := url.Parse(baseURL) + if err != nil { + return baseURL + } + u.RawQuery = encodeSortedQuery(params) + return u.String() +} + +func encodeSortedQuery(params map[string]string) string { + if len(params) == 0 { + return "" + } + q := make(url.Values, len(params)) + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + q.Set(key, params[key]) + } + return q.Encode() +} + +func appendHostQuery(rawURL, host string) string { + host = strings.TrimSpace(host) + if host == "" { + return rawURL + } + return rawURL + "&host=" + jsQueryEscape(host) +} + +func jsQueryEscape(raw string) string { + replacer := strings.NewReplacer( + "+", "%20", + "%21", "!", + "%27", "'", + "%28", "(", + "%29", ")", + "%2A", "*", + "%7E", "~", + ) + return replacer.Replace(url.QueryEscape(raw)) +} + +func sha1HexReader(r io.Reader) (string, error) { + h := sha1.New() + if _, err := io.Copy(h, r); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func sha1HexString(raw string) string { + sum := sha1.Sum([]byte(raw)) + return hex.EncodeToString(sum[:]) +} + +func progressRange(up driver.UpdateProgress, start, end float64) driver.UpdateProgress { + if up == nil { + return nil + } + return model.UpdateProgressWithRange(up, start, end) +} + +func reportByteProgress(up driver.UpdateProgress, done, total int64) { + if up == nil { + return + } + if total <= 0 { + up(100) + return + } + up(float64(done) / float64(total) * 100) +} + +func reportUploadProgress(up driver.UpdateProgress, done, total int64) { + if up == nil { + return + } + if total <= 0 { + up(100) + return + } + if done > total { + done = total + } + up(10 + float64(done)/float64(total)*90) +} + +func sleepWithContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +type blockInfoMap map[string]interface{} + +func (r *openUploadRequestResp) blockInfoForChunk(index int) blockInfoMap { + if index <= 0 || index > len(r.Data.BlockInfo) { + return blockInfoMap{} + } + return blockInfoMap(r.Data.BlockInfo[index-1]) +} + +func (m blockInfoMap) stringValue(key string) string { + value, ok := m[key] + if !ok || value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case float64: + return strconv.FormatInt(int64(v), 10) + case int: + return strconv.Itoa(v) + case int64: + return strconv.FormatInt(v, 10) + case bool: + if v { + return "1" + } + return "0" + default: + return fmt.Sprint(v) + } +} + +func (m blockInfoMap) found() int64 { + raw := strings.TrimSpace(m.stringValue("found")) + if raw == "" { + return 0 + } + value, _ := strconv.ParseInt(raw, 10, 64) + return value +} + +func (m blockInfoMap) extraFields() map[string]string { + extras := make(map[string]string) + for key := range m { + switch key { + case "bhash", "bidx", "boffset", "bsize", "filename", "filesize", "q", "t", "token", "tid", "found", "url": + continue + } + value := strings.TrimSpace(m.stringValue(key)) + if value != "" { + extras[key] = value + } + } + return extras +} + +func (r *openUploadFinalizeResp) autoCommit() bool { + raw, ok := r.Data["autoCommit"] + if !ok { + return false + } + switch v := raw.(type) { + case bool: + return v + case string: + return strings.EqualFold(v, "true") || v == "1" + case float64: + return int64(v) == 1 + case int: + return v == 1 + case int64: + return v == 1 + default: + return false + } +} + +func (r *openUploadFinalizeResp) stringValue(key string) string { + if r == nil || r.Data == nil { + return "" + } + value, ok := r.Data[key] + if !ok || value == nil { + return "" + } + switch v := value.(type) { + case string: + return v + case float64: + return strconv.FormatInt(int64(v), 10) + case int64: + return strconv.FormatInt(v, 10) + case int: + return strconv.Itoa(v) + default: + return fmt.Sprint(v) + } +} diff --git a/drivers/yunpan360/util.go b/drivers/yunpan360/util.go new file mode 100644 index 00000000000..2718f9c99cc --- /dev/null +++ b/drivers/yunpan360/util.go @@ -0,0 +1,909 @@ +package yunpan360 + +import ( + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "html" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + baseURL = "https://www.yunpan.com" + indexPath = "/file/index" + listPath = "/file/list" + downloadPath = "/file/download" + openAPIProdURL = "https://openapi.eyun.360.cn/intf.php" + openAPITestURL = "https://qaopen.eyun.360.cn/intf.php" + openAPIHGTestURL = "https://hg-openapi.eyun.360.cn/intf.php" +) + +func (d *Yunpan360) listPage(ctx context.Context, dirPath string, page, pageSize int) (ListResp, error) { + if d.authMode() == authTypeAPIKey { + return d.listOpenPage(ctx, dirPath, page, pageSize) + } + return d.listCookiePage(ctx, dirPath, page, pageSize) +} + +func (d *Yunpan360) listCookiePage(ctx context.Context, dirPath string, page, pageSize int) (*CookieListResp, error) { + var resp CookieListResp + err := d.cookieRequestForm(ctx, listPath, map[string]string{ + "path": requestPath(dirPath), + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(pageSize), + "order": requestOrder(d.OrderDirection), + "field": "file_name", + "focus_nid": "0", + }, &resp) + if err != nil { + return nil, err + } + d.cacheCookieDownloadSession(resp.GetOwnerQID(), resp.Token) + return &resp, nil +} + +func (d *Yunpan360) cookieRequestForm(ctx context.Context, apiPath string, form map[string]string, out interface{}) error { + req := base.RestyClient.R(). + SetContext(ctx). + SetHeaders(map[string]string{ + "Accept": "text/javascript, text/html, application/xml, text/xml, */*", + "Content-Type": "application/x-www-form-urlencoded", + "Cookie": d.Cookie, + "Origin": baseURL, + "Referer": baseURL + "/file/index", + "X-Requested-With": "XMLHttpRequest", + }). + SetFormData(form) + + res, err := req.Execute(http.MethodPost, baseURL+apiPath) + if err != nil { + return err + } + + var baseResp BaseResp + if err := utils.Json.Unmarshal(res.Body(), &baseResp); err != nil { + return err + } + if baseResp.Errno != 0 { + if baseResp.Errmsg == "" { + return fmt.Errorf("yunpan request failed: errno=%d", baseResp.Errno) + } + return errors.New(baseResp.Errmsg) + } + if out == nil { + return nil + } + return utils.Json.Unmarshal(res.Body(), out) +} + +func requestPath(dirPath string) string { + path := normalizeRemotePath(dirPath) + if path == "" { + return "/" + } + return path +} + +func requestOrder(order string) string { + if strings.EqualFold(order, "desc") { + return "desc" + } + return "asc" +} + +func (d *Yunpan360) cookiePage(ctx context.Context, pagePath string) ([]byte, error) { + req := base.RestyClient.R(). + SetContext(ctx). + SetHeaders(map[string]string{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Cookie": d.Cookie, + "Referer": baseURL + indexPath, + }) + res, err := req.Get(baseURL + pagePath) + if err != nil { + return nil, err + } + return res.Body(), nil +} + +func openAPIURL(env string) string { + switch env { + case "test": + return openAPITestURL + case "hgtest": + return openAPIHGTestURL + default: + return openAPIProdURL + } +} + +func openClientSecretForEnv(env string) string { + if env == "test" { + return openClientSecretQA + } + return openClientSecret +} + +func phpQueryEscape(raw string) string { + escaped := url.QueryEscape(raw) + return strings.ReplaceAll(escaped, "~", "%7E") +} + +func openSign(accessToken, qid, method string, extra map[string]string) string { + params := map[string]string{ + "access_token": accessToken, + "method": method, + "qid": qid, + } + for key, value := range extra { + if value != "" { + params[key] = value + } + } + keys := make([]string, 0, len(params)) + for key := range params { + keys = append(keys, key) + } + sortStrings(keys) + pairs := make([]string, 0, len(keys)) + for _, key := range keys { + pairs = append(pairs, key+"="+phpQueryEscape(params[key])) + } + sum := md5.Sum([]byte(strings.Join(pairs, "&") + openSignSecret)) + return hex.EncodeToString(sum[:]) +} + +func sortStrings(values []string) { + for i := 0; i < len(values); i++ { + for j := i + 1; j < len(values); j++ { + if values[j] < values[i] { + values[i], values[j] = values[j], values[i] + } + } + } +} + +func (d *Yunpan360) getOpenAuth(ctx context.Context) (*OpenAuthInfo, error) { + d.authMu.Lock() + defer d.authMu.Unlock() + + if d.cachedOpenAuth != nil && time.Now().Before(d.openAuthExpire) { + auth := *d.cachedOpenAuth + return &auth, nil + } + + reqURL := openAPIURL(d.EcsEnv) + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Accept", "application/json"). + SetHeader("api_key", d.APIKey). + SetQueryParams(map[string]string{ + "method": "Oauth.getAccessTokenByApiKey", + "client_id": openClientID, + "client_secret": openClientSecretForEnv(d.EcsEnv), + "grant_type": "authorization_code", + "sub_channel": d.SubChannel, + "api_key": d.APIKey, + }) + + res, err := req.Get(reqURL) + if err != nil { + return nil, err + } + + var resp OpenAuthResp + if err := utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + if resp.Errno != 0 { + if resp.Errmsg == "" { + return nil, fmt.Errorf("yunpan auth failed: errno=%d", resp.Errno) + } + return nil, errors.New(resp.Errmsg) + } + + auth := &OpenAuthInfo{ + AccessToken: resp.Data.AccessToken, + Qid: resp.Data.Qid, + Token: resp.Data.Token, + SubChannel: d.SubChannel, + } + d.cachedOpenAuth = auth + d.openAuthExpire = time.Now().Add(50 * time.Minute) + + copied := *auth + return &copied, nil +} + +func (d *Yunpan360) openBaseParams(auth *OpenAuthInfo, method string, signParams map[string]string, withSign bool) map[string]string { + params := map[string]string{ + "method": method, + "access_token": auth.AccessToken, + "qid": auth.Qid, + "sub_channel": auth.SubChannel, + } + if withSign { + params["sign"] = openSign(auth.AccessToken, auth.Qid, method, signParams) + } else { + params["sign"] = "" + } + return params +} + +func (d *Yunpan360) openGET(ctx context.Context, method string, signParams map[string]string, query map[string]string, out interface{}, withSign bool) error { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return err + } + params := d.openBaseParams(auth, method, signParams, withSign) + for key, value := range query { + params[key] = value + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Access-Token", auth.AccessToken). + SetQueryParams(params) + res, err := req.Get(openAPIURL(d.EcsEnv)) + if err != nil { + return err + } + return decodeBaseResp(res.Body(), out) +} + +func (d *Yunpan360) openPOST(ctx context.Context, method string, signParams map[string]string, query, body map[string]string, out interface{}, withSign bool) error { + auth, err := d.getOpenAuth(ctx) + if err != nil { + return err + } + queryParams := map[string]string{} + for key, value := range query { + queryParams[key] = value + } + bodyParams := map[string]string{} + for key, value := range body { + bodyParams[key] = value + } + + baseParams := d.openBaseParams(auth, method, signParams, withSign) + if len(queryParams) == 0 { + bodyParams = mergeStringMaps(baseParams, bodyParams) + } else { + queryParams = mergeStringMaps(baseParams, queryParams) + } + + req := base.RestyClient.R(). + SetContext(ctx). + SetHeader("Access-Token", auth.AccessToken). + SetHeader("Content-Type", "application/x-www-form-urlencoded") + if len(queryParams) > 0 { + req.SetQueryParams(queryParams) + } + if len(bodyParams) > 0 { + req.SetFormData(bodyParams) + } + res, err := req.Post(openAPIURL(d.EcsEnv)) + if err != nil { + return err + } + return decodeBaseResp(res.Body(), out) +} + +func decodeBaseResp(body []byte, out interface{}) error { + var baseResp BaseResp + if err := utils.Json.Unmarshal(body, &baseResp); err != nil { + return err + } + if baseResp.Errno != 0 { + if baseResp.Errmsg == "" { + return fmt.Errorf("yunpan request failed: errno=%d", baseResp.Errno) + } + return errors.New(baseResp.Errmsg) + } + if out == nil { + return nil + } + return utils.Json.Unmarshal(body, out) +} + +func mergeStringMaps(baseMap, extra map[string]string) map[string]string { + merged := map[string]string{} + for key, value := range baseMap { + merged[key] = value + } + for key, value := range extra { + merged[key] = value + } + return merged +} + +func (d *Yunpan360) cookieDownloadURL(ctx context.Context, file model.Obj) (*CookieDownloadResp, error) { + resp, err := d.cookieDownloadURLOnce(ctx, file, false) + if err == nil { + return resp, nil + } + + d.invalidateCookieDownloadSession() + return d.cookieDownloadURLOnce(ctx, file, true) +} + +func (d *Yunpan360) cookieDownloadURLOnce(ctx context.Context, file model.Obj, refresh bool) (*CookieDownloadResp, error) { + nid := strings.TrimSpace(file.GetID()) + if nid == "" { + return nil, errors.New("missing file id") + } + + fname := normalizeRemotePath(file.GetPath()) + if fname == "" { + return nil, errors.New("missing file path") + } + + ownerQID, token, err := d.resolveCookieDownloadParams(ctx, file, refresh) + if err != nil { + return nil, err + } + + var resp CookieDownloadResp + err = d.cookieRequestForm(ctx, downloadPath, map[string]string{ + "nid": nid, + "fname": fname, + "owner_qid": ownerQID, + "token": token, + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) cookieRename(ctx context.Context, srcObj model.Obj, newName string) error { + path := normalizeRemotePath(srcObj.GetPath()) + if path == "" { + return errors.New("missing object path") + } + nid := strings.TrimSpace(srcObj.GetID()) + if nid == "" { + return errors.New("missing object id") + } + + ownerQID, err := d.resolveCookieOwnerQID(ctx, srcObj, false) + if err != nil { + return err + } + + return d.cookieRequestForm(ctx, "/file/rename", map[string]string{ + "path": path, + "nid": nid, + "newpath": strings.TrimSuffix(strings.TrimSpace(newName), "/"), + "owner_qid": ownerQID, + }, nil) +} + +func (d *Yunpan360) resolveCookieDownloadParams(ctx context.Context, file model.Obj, refresh bool) (string, string, error) { + ownerQID := sanitizeOwnerQID(d.OwnerQID) + token := strings.TrimSpace(d.DownloadToken) + + if obj, ok := file.(*YunpanObject); ok { + ownerQID = firstNonEmpty(sanitizeOwnerQID(obj.OwnerQID), ownerQID) + token = firstNonEmpty(strings.TrimSpace(obj.DownloadToken), token) + } + + ownerQID = firstNonEmpty(ownerQID, + sanitizeOwnerQID(getCookieValue(d.Cookie, "owner_qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "ownerQid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "QID")), + ) + token = firstNonEmpty(token, + getCookieValue(d.Cookie, "download_token"), + getCookieValue(d.Cookie, "token"), + getCookieValue(d.Cookie, "Token"), + ) + + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + return ownerQID, token, nil + } + + if !refresh { + if cached := d.getCachedCookieDownloadSession(); cached != nil { + ownerQID = firstNonEmpty(ownerQID, cached.OwnerQID) + token = firstNonEmpty(token, cached.Token) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + return ownerQID, token, nil + } + } + + resp, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + if err == nil && resp != nil { + ownerQID = firstNonEmpty(ownerQID, resp.GetOwnerQID()) + token = firstNonEmpty(token, strings.TrimSpace(resp.Token)) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID != "" && token != "" { + d.cacheCookieDownloadSession(ownerQID, token) + return ownerQID, token, nil + } + + pageSession, err := d.getCookieDownloadSessionFromPage(ctx) + if err == nil && pageSession != nil { + ownerQID = firstNonEmpty(ownerQID, pageSession.OwnerQID) + token = firstNonEmpty(token, pageSession.Token) + } + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID == "" || token == "" { + return "", "", errors.New("missing owner_qid or download_token for cookie mode") + } + + d.cacheCookieDownloadSession(ownerQID, token) + return ownerQID, token, nil +} + +func (d *Yunpan360) resolveCookieOwnerQID(ctx context.Context, file model.Obj, refresh bool) (string, error) { + ownerQID := sanitizeOwnerQID(d.OwnerQID) + + if obj, ok := file.(*YunpanObject); ok { + ownerQID = firstNonEmpty(sanitizeOwnerQID(obj.OwnerQID), ownerQID) + } + + ownerQID = firstNonEmpty(ownerQID, + sanitizeOwnerQID(getCookieValue(d.Cookie, "owner_qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "ownerQid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "qid")), + sanitizeOwnerQID(getCookieValue(d.Cookie, "QID")), + ) + if ownerQID != "" { + return ownerQID, nil + } + + if !refresh { + if cached := d.getCachedCookieDownloadSession(); cached != nil { + ownerQID = firstNonEmpty(ownerQID, cached.OwnerQID) + } + if ownerQID != "" { + return ownerQID, nil + } + } + + resp, err := d.listCookiePage(ctx, d.RootFolderPath, 0, 1) + if err == nil && resp != nil { + ownerQID = firstNonEmpty(ownerQID, resp.GetOwnerQID()) + } + if ownerQID != "" { + return ownerQID, nil + } + + pageSession, err := d.getCookieDownloadSessionFromPage(ctx) + if err == nil && pageSession != nil { + ownerQID = firstNonEmpty(ownerQID, pageSession.OwnerQID) + } + if ownerQID == "" { + return "", errors.New("missing owner_qid for cookie mode") + } + return ownerQID, nil +} + +func (d *Yunpan360) getCookieDownloadSessionFromPage(ctx context.Context) (*CookieDownloadSession, error) { + body, err := d.cookiePage(ctx, indexPath) + if err != nil { + return nil, err + } + session := parseCookieDownloadSessionFromText(string(body)) + if session == nil { + return nil, errors.New("failed to parse cookie download session from page") + } + d.cacheCookieSession(session) + return session, nil +} + +func (d *Yunpan360) getCachedCookieDownloadSession() *CookieDownloadSession { + d.authMu.Lock() + defer d.authMu.Unlock() + + if d.cachedCookieSession == nil || time.Now().After(d.cookieSessionExpire) { + return nil + } + session := *d.cachedCookieSession + return &session +} + +func (d *Yunpan360) cacheCookieDownloadSession(ownerQID, token string) { + d.cacheCookieSession(&CookieDownloadSession{ + OwnerQID: ownerQID, + Token: token, + }) +} + +func (d *Yunpan360) cacheCookieSession(session *CookieDownloadSession) { + if session == nil { + return + } + + cached := &CookieDownloadSession{ + OwnerQID: sanitizeOwnerQID(session.OwnerQID), + Token: strings.TrimSpace(session.Token), + } + if cached.OwnerQID == "" && cached.Token != "" { + cached.OwnerQID = ownerQIDFromToken(cached.Token) + } + if cached.OwnerQID == "" || cached.Token == "" { + return + } + + d.authMu.Lock() + defer d.authMu.Unlock() + + d.cachedCookieSession = cached + d.cookieSessionExpire = time.Now().Add(10 * time.Minute) +} + +func (d *Yunpan360) invalidateCookieDownloadSession() { + d.authMu.Lock() + defer d.authMu.Unlock() + + d.cachedCookieSession = nil + d.cookieSessionExpire = time.Time{} +} + +func getCookieValue(rawCookie, name string) string { + for _, item := range strings.Split(rawCookie, ";") { + part := strings.TrimSpace(item) + if part == "" { + continue + } + key, value, ok := strings.Cut(part, "=") + if !ok || key != name { + continue + } + value = strings.TrimSpace(value) + value = strings.Trim(value, "\"") + unescaped, err := url.QueryUnescape(value) + if err == nil { + return strings.TrimSpace(unescaped) + } + return value + } + return "" +} + +func ownerQIDFromToken(token string) string { + parts := strings.Split(strings.TrimSpace(token), ".") + if len(parts) < 4 { + return "" + } + qid := strings.TrimSpace(parts[3]) + for _, ch := range qid { + if ch < '0' || ch > '9' { + return "" + } + } + return qid +} + +func sanitizeOwnerQID(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" || raw == "0" { + return "" + } + return raw +} + +func parseCookieDownloadSessionFromText(text string) *CookieDownloadSession { + token := extractFirstMatch(text, + `(?i)["']download_token["']\s*[:=]\s*["']([^"'<>]+)["']`, + `(?i)["']token["']\s*[:=]\s*["']([^"'<>]+)["']`, + `(?i)\btoken\s*[:=]\s*["']([^"'<>]+)["']`, + ) + ownerQID := extractFirstMatch(text, + `(?i)["']owner_qid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)["']ownerQid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)["']qid["']\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)\bowner_qid\s*[:=]\s*["']?([0-9]+)["']?`, + `(?i)\bqid\s*[:=]\s*["']?([0-9]+)["']?`, + ) + if ownerQID == "" && token != "" { + ownerQID = ownerQIDFromToken(token) + } + if ownerQID == "" || token == "" { + return nil + } + return &CookieDownloadSession{ + OwnerQID: ownerQID, + Token: token, + } +} + +func extractFirstMatch(text string, patterns ...string) string { + return extractFirstValidatedMatch(nil, text, patterns...) +} + +func extractFirstValidatedMatch(validate func(string) bool, text string, patterns ...string) string { + for _, pattern := range patterns { + re := regexp.MustCompile(pattern) + for _, matches := range re.FindAllStringSubmatch(text, -1) { + if len(matches) < 2 { + continue + } + value := html.UnescapeString(strings.TrimSpace(matches[1])) + value = strings.Trim(value, "\"'") + if value == "" { + continue + } + if validate == nil || validate(value) { + return value + } + } + } + return "" +} + +func (d *Yunpan360) listOpenPage(ctx context.Context, dirPath string, page, pageSize int) (*OpenListResp, error) { + var resp OpenListResp + path := ensureDirAPIPath(dirPath) + params := map[string]string{ + "path": path, + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(pageSize), + } + err := d.openGET(ctx, "File.getList", params, params, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openUserInfo(ctx context.Context) (*OpenUserInfoResp, error) { + var resp OpenUserInfoResp + err := d.openGET(ctx, "User.getUserDetail", nil, nil, &resp, false) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openDownloadURL(ctx context.Context, file model.Obj) (*OpenDownloadResp, error) { + var resp OpenDownloadResp + signParams := map[string]string{} + body := map[string]string{} + + if file.GetPath() != "" { + signParams["fpath"] = normalizeRemotePath(file.GetPath()) + body["fpath"] = signParams["fpath"] + } else if file.GetID() != "" { + signParams["nid"] = file.GetID() + body["nid"] = file.GetID() + } else { + return nil, errors.New("missing file path and id") + } + + err := d.openPOST(ctx, "MCP.getDownLoadUrl", signParams, nil, body, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) cookieMakeDir(ctx context.Context, fullPath string) (*CookieMkdirResp, error) { + var resp CookieMkdirResp + body := map[string]string{ + "path": ensureDirAPIPath(fullPath), + "owner_qid": "0", + } + err := d.cookieRequestForm(ctx, "/file/mkdir", body, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openMakeDir(ctx context.Context, fullPath string) (*OpenMkdirResp, error) { + var resp OpenMkdirResp + body := map[string]string{"fname": ensureDirAPIPath(fullPath)} + err := d.openPOST(ctx, "File.mkdir", body, nil, body, &resp, true) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) openRename(ctx context.Context, srcName, newName string) error { + signParams := map[string]string{ + "src_name": srcName, + "new_name": newName, + } + return d.openPOST(ctx, "File.rename", signParams, nil, signParams, nil, true) +} + +func (d *Yunpan360) cookieMove(ctx context.Context, srcPath, dstPath string) error { + var resp CookieMoveResp + body := map[string]string{ + "path[]": srcPath, + "newpath": ensureDirAPIPath(dstPath), + } + if err := d.cookieRequestForm(ctx, "/file/move", body, &resp); err != nil { + return err + } + if !resp.Data.IsAsync { + return nil + } + return d.waitCookieAsyncTask(ctx, resp.Data.TaskID) +} + +func (d *Yunpan360) cookieRecycle(ctx context.Context, obj model.Obj) error { + path := apiPathForObj(obj) + if path == "" { + return errors.New("missing object path") + } + ownerQID, err := d.resolveCookieOwnerQID(ctx, obj, false) + if err != nil { + return err + } + + var resp CookieRecycleResp + if err := d.cookieRequestForm(ctx, "/file/recycle", map[string]string{ + "path[]": path, + "owner_qid": ownerQID, + }, &resp); err != nil { + return err + } + if !resp.Data.IsAsync { + return nil + } + return d.waitCookieAsyncTask(ctx, resp.Data.TaskID, 3008) +} + +func (d *Yunpan360) cookieAsyncQuery(ctx context.Context, taskID string) (*CookieAsyncQueryResp, error) { + var resp CookieAsyncQueryResp + err := d.cookieRequestForm(ctx, "/async/query", map[string]string{ + "task_id": strings.TrimSpace(taskID), + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Yunpan360) waitCookieAsyncTask(ctx context.Context, taskID string, toleratedErrnos ...int) error { + taskID = strings.TrimSpace(taskID) + if taskID == "" { + return nil + } + + tolerated := map[int]struct{}{} + for _, errno := range toleratedErrnos { + tolerated[errno] = struct{}{} + } + + for attempt := 0; attempt < 15; attempt++ { + resp, err := d.cookieAsyncQuery(ctx, taskID) + if err == nil && resp != nil { + if task, ok := resp.Data[taskID]; ok { + done, taskErr := checkCookieAsyncTask(task, tolerated) + if done { + return taskErr + } + } + } + if attempt == 14 { + break + } + if err := sleepWithContext(ctx, 300*time.Millisecond); err != nil { + return err + } + } + + // Keep prior behavior when the async task is still pending after the probe window. + return nil +} + +func checkCookieAsyncTask(task CookieAsyncTask, toleratedErrnos map[int]struct{}) (bool, error) { + if task.Status != 10 { + return false, nil + } + if task.Errno == 0 { + return true, nil + } + if _, ok := toleratedErrnos[task.Errno]; ok { + return true, nil + } + if strings.TrimSpace(task.Errstr) != "" { + return true, errors.New(task.Errstr) + } + if strings.TrimSpace(task.Action) != "" { + return true, fmt.Errorf("yunpan async task %s failed: errno=%d", task.Action, task.Errno) + } + return true, fmt.Errorf("yunpan async task failed: errno=%d", task.Errno) +} + +func (d *Yunpan360) openMove(ctx context.Context, srcName, dstPath string) error { + signParams := map[string]string{ + "src_name": srcName, + "new_name": dstPath, + } + return d.openPOST(ctx, "File.move", signParams, nil, signParams, nil, true) +} + +func (d *Yunpan360) openDelete(ctx context.Context, targetPath string) error { + return d.openPOST(ctx, "File.delete", nil, nil, map[string]string{ + "fname": targetPath, + }, nil, true) +} + +func apiPathForObj(obj model.Obj) string { + if obj.IsDir() { + return ensureDirAPIPath(obj.GetPath()) + } + return normalizeRemotePath(obj.GetPath()) +} + +func ensureDirSuffix(name string) string { + name = strings.TrimSpace(name) + if name == "" || strings.HasSuffix(name, "/") { + return name + } + return name + "/" +} + +func ensureDirAPIPath(p string) string { + p = normalizeRemotePath(p) + if p == "" || p == "/" { + return "/" + } + return p + "/" +} + +func cloneObj(src model.Obj, newPath, newName string) model.Obj { + obj := model.Object{ + ID: src.GetID(), + Path: normalizeRemotePath(newPath), + Name: newName, + Size: src.GetSize(), + Modified: src.ModTime(), + Ctime: src.CreateTime(), + IsFolder: src.IsDir(), + HashInfo: src.GetHash(), + } + if raw, ok := src.(*YunpanObject); ok { + return &YunpanObject{ + Object: obj, + Thumbnail: raw.Thumbnail, + OwnerQID: raw.OwnerQID, + DownloadToken: raw.DownloadToken, + } + } + return &obj +} + +func absoluteURL(raw string) string { + if raw == "" { + return "" + } + if strings.HasPrefix(raw, "http://") || strings.HasPrefix(raw, "https://") { + return raw + } + if strings.HasPrefix(raw, "/") { + return baseURL + raw + } + return baseURL + "/" + raw +} diff --git a/entrypoint.sh b/entrypoint.sh index 05bbf8d3c83..c24ed6eebf9 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,7 +1,19 @@ -#!/bin/bash - -chown -R ${PUID}:${PGID} /opt/alist/ +#!/bin/sh umask ${UMASK} -exec su-exec ${PUID}:${PGID} ./alist server --no-prefix \ No newline at end of file +if [ "$1" = "version" ]; then + ./alist version +else + if [ "$RUN_ARIA2" = "true" ]; then + chown -R ${PUID}:${PGID} /opt/aria2/ + exec su-exec ${PUID}:${PGID} nohup aria2c \ + --enable-rpc \ + --rpc-allow-origin-all \ + --conf-path=/opt/aria2/.aria2/aria2.conf \ + >/dev/null 2>&1 & + fi + + chown -R ${PUID}:${PGID} /opt/alist/ + exec su-exec ${PUID}:${PGID} ./alist server --no-prefix +fi \ No newline at end of file diff --git a/go.mod b/go.mod index f185aef9b1e..a306287ce69 100644 --- a/go.mod +++ b/go.mod @@ -1,82 +1,198 @@ module github.com/alist-org/alist/v3 -go 1.20 +go 1.25.0 require ( - github.com/SheltonZhu/115driver v1.0.16 - github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 + github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 + github.com/KirCute/sftpd-alist v0.0.12 + github.com/ProtonMail/go-crypto v1.0.0 + github.com/ProtonMail/gopenpgp/v2 v2.7.4 + github.com/SheltonZhu/115driver v1.2.3-1 + github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 - github.com/Xhofe/wopan-sdk-go v0.1.2 - github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible + github.com/alist-org/gofakes3 v0.0.7 + github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.44.327 - github.com/blevesearch/bleve/v2 v2.3.10 + github.com/aws/aws-sdk-go v1.55.5 + github.com/blevesearch/bleve/v2 v2.4.2 github.com/caarlos0/env/v9 v9.0.0 - github.com/charmbracelet/bubbles v0.16.1 - github.com/charmbracelet/bubbletea v0.24.2 - github.com/charmbracelet/lipgloss v0.7.1 + github.com/charmbracelet/bubbles v0.20.0 + github.com/charmbracelet/bubbletea v1.1.0 + github.com/charmbracelet/lipgloss v0.13.0 + github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e github.com/coreos/go-oidc v2.2.1+incompatible - github.com/deckarep/golang-set/v2 v2.3.1 + github.com/deckarep/golang-set/v2 v2.6.0 + github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 github.com/disintegration/imaging v1.6.2 - github.com/djherbis/times v1.5.0 + github.com/dlclark/regexp2 v1.11.4 github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 - github.com/foxxorcat/mopan-sdk-go v0.1.4 - github.com/foxxorcat/weiyun-sdk-go v0.1.2 - github.com/gin-contrib/cors v1.4.0 - github.com/gin-gonic/gin v1.9.1 - github.com/go-resty/resty/v2 v2.9.1 - github.com/go-webauthn/webauthn v0.8.6 - github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/google/uuid v1.3.1 - github.com/gorilla/websocket v1.5.0 + github.com/fatedier/frp v0.68.0 + github.com/foxxorcat/mopan-sdk-go v0.1.6 + github.com/foxxorcat/weiyun-sdk-go v0.1.3 + github.com/gin-contrib/cors v1.7.2 + github.com/gin-gonic/gin v1.10.0 + github.com/go-resty/resty/v2 v2.14.0 + github.com/go-webauthn/webauthn v0.11.1 + github.com/golang-jwt/jwt/v4 v4.5.2 + github.com/google/uuid v1.6.0 + github.com/gorilla/websocket v1.5.3 + github.com/hekmon/transmissionrpc/v3 v3.0.0 + github.com/henrybear327/Proton-API-Bridge v1.0.0 + github.com/henrybear327/go-proton-api v1.0.0 github.com/hirochachacha/go-smb2 v1.1.0 github.com/ipfs/go-ipfs-api v0.7.0 github.com/jlaffaye/ftp v0.2.0 github.com/json-iterator/go v1.1.12 - github.com/maruel/natural v1.1.0 + github.com/kdomanski/iso9660 v0.4.0 + github.com/larksuite/oapi-sdk-go/v3 v3.6.1 + github.com/mark3labs/mcp-go v0.48.0 + github.com/maruel/natural v1.1.1 + github.com/meilisearch/meilisearch-go v0.27.2 + github.com/mholt/archives v0.1.0 + github.com/minio/sio v0.4.0 github.com/natefinch/lumberjack v2.0.0+incompatible - github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 + github.com/ncw/swift/v2 v2.0.3 github.com/pkg/errors v0.9.1 github.com/pkg/sftp v1.13.6 github.com/pquerna/otp v1.4.0 - github.com/rclone/rclone v1.63.1 + github.com/rclone/rclone v1.67.0 + github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.7.0 - github.com/stretchr/testify v1.8.4 - github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca + github.com/spf13/afero v1.11.0 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.11.1 + github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 github.com/u2takey/ffmpeg-go v0.5.0 github.com/upyun/go-sdk/v3 v3.0.4 github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 - golang.org/x/crypto v0.14.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/image v0.11.0 - golang.org/x/net v0.16.0 - golang.org/x/oauth2 v0.12.0 - gorm.io/driver/mysql v1.4.7 - gorm.io/driver/postgres v1.4.8 - gorm.io/driver/sqlite v1.4.4 - gorm.io/gorm v1.24.5 + github.com/xhofe/tache v0.1.5 + github.com/xhofe/wopan-sdk-go v0.1.3 + github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 + github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 + golang.org/x/crypto v0.46.0 + golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e + golang.org/x/image v0.19.0 + golang.org/x/net v0.48.0 + golang.org/x/oauth2 v0.34.0 + golang.org/x/time v0.8.0 + google.golang.org/appengine v1.6.8 + gopkg.in/ldap.v3 v3.1.0 + gorm.io/driver/mysql v1.5.7 + gorm.io/driver/postgres v1.5.9 + gorm.io/driver/sqlite v1.5.6 + gorm.io/gorm v1.25.11 +) + +require ( + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect + github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect + github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect + github.com/ProtonMail/go-srp v0.0.7 // indirect + github.com/PuerkitoBio/goquery v1.8.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect + github.com/bradenaw/juniper v0.15.2 // indirect + github.com/coreos/go-oidc/v3 v3.14.1 // indirect + github.com/cronokirby/saferith v0.33.0 // indirect + github.com/emersion/go-message v0.18.0 // indirect + github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect + github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect + github.com/fatedier/golib v0.5.1 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect + github.com/google/jsonschema-go v0.4.2 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/hashicorp/yamux v0.1.1 // indirect + github.com/klauspost/reedsolomon v1.12.0 // indirect + github.com/pion/dtls/v2 v2.2.7 // indirect + github.com/pion/logging v0.2.2 // indirect + github.com/pion/stun/v2 v2.0.0 // indirect + github.com/pion/transport/v2 v2.2.1 // indirect + github.com/pion/transport/v3 v3.0.1 // indirect + github.com/pires/go-proxyproto v0.7.0 // indirect + github.com/quic-go/quic-go v0.55.0 // indirect + github.com/relvacode/iso8601 v1.3.0 // indirect + github.com/samber/lo v1.47.0 // indirect + github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/templexxx/cpu v0.1.1 // indirect + github.com/templexxx/xorsimd v0.4.3 // indirect + github.com/tjfoc/gmsm v1.4.1 // indirect + github.com/vishvananda/netlink v1.3.0 // indirect + github.com/vishvananda/netns v0.0.4 // indirect + github.com/xtaci/kcp-go/v5 v5.6.13 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect + golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apimachinery v0.28.8 // indirect + k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/yaml v1.3.0 // indirect +) + +require ( + github.com/STARRY-S/zip v0.2.1 // indirect + github.com/aymerick/douceur v0.2.0 // indirect + github.com/blevesearch/go-faiss v1.0.20 // indirect + github.com/blevesearch/zapx/v16 v16.1.5 // indirect + github.com/bodgit/plumbing v1.3.0 // indirect + github.com/bodgit/sevenzip v1.6.0 + github.com/bodgit/windows v1.0.1 // indirect + github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/charmbracelet/x/ansi v0.2.3 // indirect + github.com/charmbracelet/x/term v0.2.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fclairamb/go-log v0.5.0 // indirect + github.com/gorilla/css v1.0.1 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect + github.com/hekmon/cunits/v2 v2.1.0 // indirect + github.com/ipfs/boxo v0.12.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect + github.com/matoous/go-nanoid/v2 v2.1.0 // indirect + github.com/microcosm-cc/bluemonday v1.0.27 + github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 + github.com/sorairolake/lzip-go v0.3.5 // indirect + github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect + github.com/therootcompany/xz v1.0.1 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect + github.com/xhofe/115-sdk-go v0.1.5 + github.com/yuin/goldmark v1.7.8 + go4.org v0.0.0-20230225012048-214862532bf5 + resty.dev/v3 v3.0.0-beta.2 // indirect ) require ( - cloud.google.com/go/compute v1.23.0 // indirect - github.com/BurntSushi/toml v0.3.1 // indirect github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect - github.com/RoaringBitmap/roaring v1.2.3 // indirect + github.com/RoaringBitmap/roaring v1.9.3 // indirect github.com/abbot/go-http-auth v0.4.0 // indirect github.com/aead/ecdh v0.2.0 // indirect - github.com/andreburgaud/crypt2go v1.2.0 // indirect + github.com/andreburgaud/crypt2go v1.8.0 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/benbjohnson/clock v1.3.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/bits-and-blooms/bitset v1.12.0 // indirect github.com/blang/semver/v4 v4.0.0 // indirect - github.com/blevesearch/bleve_index_api v1.0.6 // indirect - github.com/blevesearch/geo v0.1.18 // indirect + github.com/blevesearch/bleve_index_api v1.1.10 // indirect + github.com/blevesearch/geo v0.1.20 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/gtreap v0.1.1 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect - github.com/blevesearch/scorch_segment_api/v2 v2.1.6 // indirect + github.com/blevesearch/scorch_segment_api/v2 v2.2.15 // indirect github.com/blevesearch/segment v0.9.1 // indirect github.com/blevesearch/snowballstem v0.9.0 // indirect github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect @@ -86,71 +202,68 @@ require ( github.com/blevesearch/zapx/v13 v13.3.10 // indirect github.com/blevesearch/zapx/v14 v14.3.10 // indirect github.com/blevesearch/zapx/v15 v15.3.13 // indirect - github.com/bluele/gcache v0.0.2 // indirect github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect - github.com/bytedance/sonic v1.9.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect + github.com/bytedance/sonic v1.11.6 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/fxamacker/cbor/v2 v2.4.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.2 // indirect - github.com/gaoyb7/115drive-webdav v0.1.8 // indirect + github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/geoffgarside/ber v1.1.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-chi/chi/v5 v5.0.10 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-chi/chi/v5 v5.0.12 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.14.0 // indirect + github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect - github.com/go-webauthn/x v0.1.4 // indirect + github.com/go-webauthn/x v0.1.12 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-tpm v0.9.0 // indirect + github.com/google/go-tpm v0.9.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/ipfs/boxo v0.12.0 // indirect - github.com/ipfs/go-cid v0.4.1 // indirect + github.com/ipfs/go-cid v0.4.1 github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect - github.com/jackc/pgx/v5 v5.3.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.9.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/kr/fs v0.1.0 // indirect - github.com/leodido/go-urn v1.2.4 // indirect + github.com/leodido/go-urn v1.4.0 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p v0.27.8 // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect - github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect - github.com/mattn/go-runewidth v0.0.14 // indirect - github.com/mattn/go-sqlite3 v1.14.15 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/mschoch/smat v0.2.0 // indirect - github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect - github.com/muesli/reflow v0.3.0 // indirect - github.com/muesli/termenv v0.15.1 // indirect + github.com/muesli/termenv v0.15.2 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr v0.9.0 // indirect @@ -159,44 +272,55 @@ require ( github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/ncw/swift/v2 v2.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.0.8 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/otiai10/copy v1.14.0 + github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect github.com/pquerna/cachecontrol v0.1.0 // indirect - github.com/prometheus/client_golang v1.16.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/client_golang v1.19.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/rfjakob/eme v1.1.2 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - github.com/shirou/gopsutil/v3 v3.23.7 // indirect - github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect + github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect + github.com/shirou/gopsutil/v3 v3.24.4 // indirect + github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tklauser/go-sysconf v0.3.11 // indirect - github.com/tklauser/numcpus v0.6.1 // indirect + github.com/tklauser/go-sysconf v0.3.13 // indirect + github.com/tklauser/numcpus v0.7.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/u2takey/go-utils v0.3.1 // indirect - github.com/ugorji/go/codec v1.2.11 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d // indirect github.com/x448/float16 v0.8.4 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.etcd.io/bbolt v1.3.7 // indirect - golang.org/x/arch v0.3.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - golang.org/x/time v0.3.0 // indirect - google.golang.org/api v0.134.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.etcd.io/bbolt v1.3.8 // indirect + golang.org/x/arch v0.8.0 // indirect + golang.org/x/sync v0.19.0 + golang.org/x/sys v0.39.0 // indirect + golang.org/x/term v0.38.0 // indirect + golang.org/x/text v0.32.0 + golang.org/x/tools v0.39.0 // indirect + google.golang.org/api v0.169.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/grpc v1.79.3 + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) + +replace github.com/ProtonMail/go-proton-api => github.com/henrybear327/go-proton-api v1.0.0 + +replace github.com/cronokirby/saferith => github.com/Da3zKi7/saferith v0.33.0-fixed + +replace github.com/SheltonZhu/115driver => github.com/okatu-loli/115driver v1.2.3-1 diff --git a/go.sum b/go.sum index 0fb9a131f25..5360822db69 100644 --- a/go.sum +++ b/go.sum @@ -1,77 +1,135 @@ -cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= -cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0 h1:UXT0o77lXQrikd1kgwIPQOUect7EoR/+sbP4wQKdzxM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.0/go.mod h1:cTvi54pg19DoT07ekoeMgE/taAwNtCShVeZqA+Iv2xI= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= +github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Da3zKi7/saferith v0.33.0-fixed h1:fnIWTk7EP9mZAICf7aQjeoAwpfrlCrkOvqmi6CbWdTk= +github.com/Da3zKi7/saferith v0.33.0-fixed/go.mod h1:QKJhjoqUtBsXCAVEjw38mFqoi7DebT7kthcD7UzbnoA= +github.com/KirCute/ftpserverlib-pasvportmap v1.25.0 h1:ikwCzeqoqN6wvBHOB9OI6dde/jbV7EoTMpUcxtYl5Po= +github.com/KirCute/ftpserverlib-pasvportmap v1.25.0/go.mod h1:v0NgMtKDDi/6CM6r4P+daCljCW3eO9yS+Z+pZDTKo1E= +github.com/KirCute/sftpd-alist v0.0.12 h1:GNVM5QLbQLAfXP4wGUlXFA2IO6fVek0n0IsGnOuISdg= +github.com/KirCute/sftpd-alist v0.0.12/go.mod h1:2wNK7yyW2XfjyJq10OY6xB4COLac64hOwfV6clDJn6s= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd h1:nzE1YQBdx1bq9IlZinHa+HVffy+NmVRoKr+wHN8fpLE= github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd/go.mod h1:C8yoIfvESpM3GD07OCHU7fqI7lhwyZ2Td1rbNbTAhnc= -github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVOVmhWBY= -github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= -github.com/SheltonZhu/115driver v1.0.15 h1:RRvgXvXEzvrPwkRno0CUIg7ucEphbsfwct2mQxfNOdQ= -github.com/SheltonZhu/115driver v1.0.15/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= -github.com/SheltonZhu/115driver v1.0.16 h1:XOhqRtKF9huTCobM5rWVd9DtJyBKLJSnBwWPF3+GM+k= -github.com/SheltonZhu/115driver v1.0.16/go.mod h1:e3fPOBANbH/FsTya8FquJwOR3ErhCQgEab3q6CVY2k4= +github.com/ProtonMail/bcrypt v0.0.0-20210511135022-227b4adcab57/go.mod h1:HecWFHognK8GfRDGnFQbW/LiV7A3MX3gZVs45vk5h8I= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf h1:yc9daCCYUefEs69zUkSzubzjBbL+cmOXgnmt9Fyd9ug= +github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf/go.mod h1:o0ESU9p83twszAU8LBeJKFAAMX14tISa0yk4Oo5TOqo= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e h1:lCsqUUACrcMC83lg5rTo9Y0PnPItE61JSfvMyIcANwk= +github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e/go.mod h1:Og5/Dz1MiGpCJn51XujZwxiLG7WzvvjE5PRpZBQmAHo= +github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE= +github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= +github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k= +github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw= +github.com/ProtonMail/go-srp v0.0.7 h1:Sos3Qk+th4tQR64vsxGIxYpN3rdnG9Wf9K4ZloC1JrI= +github.com/ProtonMail/go-srp v0.0.7/go.mod h1:giCp+7qRnMIcCvI6V6U3S1lDDXDQYx2ewJ6F/9wdlJk= +github.com/ProtonMail/gopenpgp/v2 v2.7.4 h1:Vz/8+HViFFnf2A6XX8JOvZMrA6F5puwNvvF21O1mRlo= +github.com/ProtonMail/gopenpgp/v2 v2.7.4/go.mod h1:IhkNEDaxec6NyzSI0PlxapinnwPVIESk8/76da3Ct3g= +github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= +github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/RoaringBitmap/roaring v1.9.3 h1:t4EbC5qQwnisr5PrP9nt0IRhRTb9gMUgQF4t4S2OByM= +github.com/RoaringBitmap/roaring v1.9.3/go.mod h1:6AXUsoIEzDTFFQCe1RbGA6uFONMhvejWj5rqITANK90= +github.com/STARRY-S/zip v0.2.1 h1:pWBd4tuSGm3wtpoqRZZ2EAwOmcHK6XFf7bU9qcJXyFg= +github.com/STARRY-S/zip v0.2.1/go.mod h1:xNvshLODWtC4EJ702g7cTYn13G53o1+X9BWnPFpcWV4= github.com/Unknwon/goconfig v1.0.0 h1:9IAu/BYbSLQi8puFjUQApZTxIHqSwrj5d8vpP8vTq4A= -github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a h1:RenIAa2q4H8UcS/cqmwdT1WCWIAH5aumP8m8RpbqVsE= -github.com/Xhofe/go-cache v0.0.0-20220723083548-714439c8af9a/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= +github.com/Unknwon/goconfig v1.0.0/go.mod h1:wngxua9XCNjvHjDiTiV26DaKDT+0c63QR6H5hjVUUxw= +github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21 h1:h6q5E9aMBhhdqouW81LozVPI1I+Pu6IxL2EKpfm5OjY= +github.com/Xhofe/go-cache v0.0.0-20240804043513-b1a71927bc21/go.mod h1:sSBbaOg90XwWKtpT56kVujF0bIeVITnPlssLclogS04= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4 h1:WnvifFgYyogPz2ZFvaVLk4gI/Co0paF92FmxSR6U1zY= github.com/Xhofe/rateg v0.0.0-20230728072201-251a4e1adad4/go.mod h1:8pWlL2rpusvx7Xa6yYaIWOJ8bR3gPdFBUT7OystyGOY= -github.com/Xhofe/wopan-sdk-go v0.1.1 h1:dSrTxNYclqNuo9libjtC+R6C4RCen/inh/dUXd12vpM= -github.com/Xhofe/wopan-sdk-go v0.1.1/go.mod h1:xWcUS7PoFLDD9gy2BK2VQfilEsZngLMz2Vkx3oF2zJY= -github.com/Xhofe/wopan-sdk-go v0.1.2 h1:6Gh4YTT7b7YHN0OoJ33j7Jm9ru/ckuvcDxPnRmH07jc= -github.com/Xhofe/wopan-sdk-go v0.1.2/go.mod h1:ktLYb4t7rnPFq1AshLaPXq5kZER+DkEagT6/i/in0uo= github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0= github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM= github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= -github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible h1:QoRMR0TCctLDqBCMyOu1eXdZyMw3F7uGA9qPn2J4+R8= -github.com/aliyun/aliyun-oss-go-sdk v2.2.5+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= -github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible h1:Sg/2xHwDrioHpxTN6WMiwbXTpUEinBpHsN7mG21Rc2k= -github.com/aliyun/aliyun-oss-go-sdk v2.2.9+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/andreburgaud/crypt2go v1.1.0 h1:eitZxTPY1krUsxinsng3Qvt/Ud7q/aQmmYRh8p4hyPw= -github.com/andreburgaud/crypt2go v1.1.0/go.mod h1:4qhZPzarj1dCIRmCkpdgCklwp+hBq9yEt0zPe9Ayuhc= -github.com/andreburgaud/crypt2go v1.2.0 h1:oly/ENAodeqTYpUafgd4r3v+VKLQnmOKUyfpj+TxHbE= -github.com/andreburgaud/crypt2go v1.2.0/go.mod h1:kKRqlrX/3Q9Ki7HdUsoh0cX1Urq14/Hcta4l4VrIXrI= +github.com/alist-org/gofakes3 v0.0.7 h1:0cDGI7fLBrqumhCBto9T3ZYCL71AyGZ1l+xxJgjqe8s= +github.com/alist-org/gofakes3 v0.0.7/go.mod h1:6IyGtYGIX29fLvtXo+XZhtwX2P33KVYYj8uTgAHSu58= +github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92 h1:pIEI87zhv8ZzQcu65rTL7kqirrs8dR6HDiXrqWat2Fk= +github.com/alist-org/times v0.0.0-20240721124654-efa0c7d3ad92/go.mod h1:oPJwGY3sLmGgcJamGumz//0A35f4BwQRacyqLNcJTOU= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/andreburgaud/crypt2go v1.8.0 h1:J73vGTb1P6XL69SSuumbKs0DWn3ulbl9L92ZXBjw6pc= +github.com/andreburgaud/crypt2go v1.8.0/go.mod h1:L5nfShQ91W78hOWhUH2tlGRPO+POAPJAF5fKOLB9SXg= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-sdk-go v1.38.20/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= -github.com/aws/aws-sdk-go v1.44.327 h1:ZS8oO4+7MOBLhkdwIhgtVeDzCeWOlTfKJS7EgggbIEY= -github.com/aws/aws-sdk-go v1.44.327/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394 h1:OYA+5W64v3OgClL+IrOD63t4i/RW7RqrAVl9LTZ9UqQ= +github.com/axgle/mahonia v0.0.0-20180208002826-3358181d7394/go.mod h1:Q8n74mJTIgjX4RBBcHnJ05h//6/k6foqmgE45jTQtxg= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA= +github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/blevesearch/bleve/v2 v2.3.9 h1:pUMvK0mxAexqasZcVj8lazmWnEW5XiV0tASIqANiNTQ= -github.com/blevesearch/bleve/v2 v2.3.9/go.mod h1:1PibElcjlQMQHF9uS9mRv58ODQgj4pCWHA1Wfd+qagU= -github.com/blevesearch/bleve/v2 v2.3.10 h1:z8V0wwGoL4rp7nG/O3qVVLYxUqCbEwskMt4iRJsPLgg= -github.com/blevesearch/bleve/v2 v2.3.10/go.mod h1:RJzeoeHC+vNHsoLR54+crS1HmOWpnH87fL70HAUCzIA= -github.com/blevesearch/bleve_index_api v1.0.5 h1:Lc986kpC4Z0/n1g3gg8ul7H+lxgOQPcXb9SxvQGu+tw= -github.com/blevesearch/bleve_index_api v1.0.5/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= -github.com/blevesearch/bleve_index_api v1.0.6 h1:gyUUxdsrvmW3jVhhYdCVL6h9dCjNT/geNU7PxGn37p8= -github.com/blevesearch/bleve_index_api v1.0.6/go.mod h1:YXMDwaXFFXwncRS8UobWs7nvo0DmusriM1nztTlj1ms= -github.com/blevesearch/geo v0.1.17 h1:AguzI6/5mHXapzB0gE9IKWo+wWPHZmXZoscHcjFgAFA= -github.com/blevesearch/geo v0.1.17/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= -github.com/blevesearch/geo v0.1.18 h1:Np8jycHTZ5scFe7VEPLrDoHnnb9C4j636ue/CGrhtDw= -github.com/blevesearch/geo v0.1.18/go.mod h1:uRMGWG0HJYfWfFJpK3zTdnnr1K+ksZTuWKhXeSokfnM= +github.com/blevesearch/bleve/v2 v2.4.2 h1:NooYP1mb3c0StkiY9/xviiq2LGSaE8BQBCc/pirMx0U= +github.com/blevesearch/bleve/v2 v2.4.2/go.mod h1:ATNKj7Yl2oJv/lGuF4kx39bST2dveX6w0th2FFYLkc8= +github.com/blevesearch/bleve_index_api v1.1.10 h1:PDLFhVjrjQWr6jCuU7TwlmByQVCSEURADHdCqVS9+g0= +github.com/blevesearch/bleve_index_api v1.1.10/go.mod h1:PbcwjIcRmjhGbkS/lJCpfgVSMROV6TRubGGAODaK1W8= +github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM= +github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w= +github.com/blevesearch/go-faiss v1.0.20 h1:AIkdTQFWuZ5LQmKQSebgMR4RynGNw8ZseJXaan5kvtI= +github.com/blevesearch/go-faiss v1.0.20/go.mod h1:jrxHrbl42X/RnDPI+wBoZU8joxxuRwedrxqswQ3xfU8= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= github.com/blevesearch/gtreap v0.1.1 h1:2JWigFrzDMR+42WGIN/V2p0cUvn4UP3C4Q5nmaZGW8Y= github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgYICSZ3w0tYk= github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc= github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs= -github.com/blevesearch/scorch_segment_api/v2 v2.1.5 h1:1g713kpCQZ8u4a3stRGBfrwVOuGRnmxOVU5MQkUPrHU= -github.com/blevesearch/scorch_segment_api/v2 v2.1.5/go.mod h1:f2nOkKS1HcjgIWZgDAErgBdxmr2eyt0Kn7IY+FU1Xe4= -github.com/blevesearch/scorch_segment_api/v2 v2.1.6 h1:CdekX/Ob6YCYmeHzD72cKpwzBjvkOGegHOqhAkXp6yA= -github.com/blevesearch/scorch_segment_api/v2 v2.1.6/go.mod h1:nQQYlp51XvoSVxcciBjtvuHPIVjlWrN1hX4qwK2cqdc= +github.com/blevesearch/scorch_segment_api/v2 v2.2.15 h1:prV17iU/o+A8FiZi9MXmqbagd8I0bCqM7OKUYPbnb5Y= +github.com/blevesearch/scorch_segment_api/v2 v2.2.15/go.mod h1:db0cmP03bPNadXrCDuVkKLV6ywFSiRgPFT1YVrestBc= github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU= github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw= github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s= @@ -80,188 +138,297 @@ github.com/blevesearch/upsidedown_store_api v1.0.2 h1:U53Q6YoWEARVLd1OYNc9kvhBMG github.com/blevesearch/upsidedown_store_api v1.0.2/go.mod h1:M01mh3Gpfy56Ps/UXHjEO/knbqyQ1Oamg8If49gRwrQ= github.com/blevesearch/vellum v1.0.10 h1:HGPJDT2bTva12hrHepVT3rOyIKFFF4t7Gf6yMxyMIPI= github.com/blevesearch/vellum v1.0.10/go.mod h1:ul1oT0FhSMDIExNjIxHqJoGpVrBpKCdgDQNxfqgJt7k= -github.com/blevesearch/zapx/v11 v11.3.9 h1:y3ijS4h4MJdmQ07MHASxat4owAixreK2xdo76w9ncrw= -github.com/blevesearch/zapx/v11 v11.3.9/go.mod h1:jcAYnQwlr+LqD2vLjDWjWiZDXDXGFqPbpPDRTd3XmS4= github.com/blevesearch/zapx/v11 v11.3.10 h1:hvjgj9tZ9DeIqBCxKhi70TtSZYMdcFn7gDb71Xo/fvk= github.com/blevesearch/zapx/v11 v11.3.10/go.mod h1:0+gW+FaE48fNxoVtMY5ugtNHHof/PxCqh7CnhYdnMzQ= -github.com/blevesearch/zapx/v12 v12.3.9 h1:MXGLlZ03oxXH3DMJTZaBaRj2xb6t4wQVZeZK/wu1M6w= -github.com/blevesearch/zapx/v12 v12.3.9/go.mod h1:QXCMwmOkdLnMDgTN1P4CcuX5F851iUOtOwXbw0HMBYs= github.com/blevesearch/zapx/v12 v12.3.10 h1:yHfj3vXLSYmmsBleJFROXuO08mS3L1qDCdDK81jDl8s= github.com/blevesearch/zapx/v12 v12.3.10/go.mod h1:0yeZg6JhaGxITlsS5co73aqPtM04+ycnI6D1v0mhbCs= -github.com/blevesearch/zapx/v13 v13.3.9 h1:+VAz9V0VmllHXlZV4DCvfYj0nqaZHgF3MeEHwOyRBwQ= -github.com/blevesearch/zapx/v13 v13.3.9/go.mod h1:s+WjNp4WSDtrBVBpa37DUOd7S/Gr/jTZ7ST/MbCVj/0= github.com/blevesearch/zapx/v13 v13.3.10 h1:0KY9tuxg06rXxOZHg3DwPJBjniSlqEgVpxIqMGahDE8= github.com/blevesearch/zapx/v13 v13.3.10/go.mod h1:w2wjSDQ/WBVeEIvP0fvMJZAzDwqwIEzVPnCPrz93yAk= -github.com/blevesearch/zapx/v14 v14.3.9 h1:wuqxATgsTCNHM9xsOFOeFp8H2heZ/gMX/tsl9lRK8U4= -github.com/blevesearch/zapx/v14 v14.3.9/go.mod h1:MWZ4v8AzFBRurhDzkLvokFW8ljcq9Evm27mkWe8OGbM= github.com/blevesearch/zapx/v14 v14.3.10 h1:SG6xlsL+W6YjhX5N3aEiL/2tcWh3DO75Bnz77pSwwKU= github.com/blevesearch/zapx/v14 v14.3.10/go.mod h1:qqyuR0u230jN1yMmE4FIAuCxmahRQEOehF78m6oTgns= -github.com/blevesearch/zapx/v15 v15.3.12 h1:w/kU9aHyfMDEdwHGZzCiakC3HZ9z5gYlXaALDC4Dct8= -github.com/blevesearch/zapx/v15 v15.3.12/go.mod h1:tx53gDJS/7Oa3Je820cmVurqCuJ4dqdAy1kiDMV/IUo= github.com/blevesearch/zapx/v15 v15.3.13 h1:6EkfaZiPlAxqXz0neniq35my6S48QI94W/wyhnpDHHQ= github.com/blevesearch/zapx/v15 v15.3.13/go.mod h1:Turk/TNRKj9es7ZpKK95PS7f6D44Y7fAFy8F4LXQtGg= -github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= -github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= +github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi/AUHjU= +github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8= +github.com/bodgit/plumbing v1.3.0 h1:pf9Itz1JOQgn7vEOE7v7nlEfBykYqvUYioC61TwWCFU= +github.com/bodgit/plumbing v1.3.0/go.mod h1:JOTb4XiRu5xfnmdnDJo6GmSbSbtSyufrsyZFByMtKEs= +github.com/bodgit/sevenzip v1.6.0 h1:a4R0Wu6/P1o1pP/3VV++aEOcyeBxeO/xE2Y9NSTrr6A= +github.com/bodgit/sevenzip v1.6.0/go.mod h1:zOBh9nJUof7tcrlqJFv1koWRrhz3LbDbUNngkuZxLMc= +github.com/bodgit/windows v1.0.1 h1:tF7K6KOluPYygXa3Z2594zxlkbKPAOvqr97etrGNIz4= +github.com/bodgit/windows v1.0.1/go.mod h1:a6JLwrB4KrTR5hBpp8FI9/9W9jJfeQ2h4XDXU74ZCdM= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= -github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= -github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= -github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= +github.com/bradenaw/juniper v0.15.2 h1:0JdjBGEF2jP1pOxmlNIrPhAoQN7Ng5IMAY5D0PHMW4U= +github.com/bradenaw/juniper v0.15.2/go.mod h1:UX4FX57kVSaDp4TPqvSjkAAewmRFAfXf27BOs5z9dq8= +github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/caarlos0/env/v9 v9.0.0 h1:SI6JNsOA+y5gj9njpgybykATIylrRMklbs5ch6wO6pc= github.com/caarlos0/env/v9 v9.0.0/go.mod h1:ye5mlCVMYh6tZ+vCgrs/B95sj88cg5Tlnc0XIzgZ020= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/charmbracelet/bubbles v0.16.1 h1:6uzpAAaT9ZqKssntbvZMlksWHruQLNxg49H5WdeuYSY= -github.com/charmbracelet/bubbles v0.16.1/go.mod h1:2QCp9LFlEsBQMvIYERr7Ww2H2bA7xen1idUDIzm/+Xc= -github.com/charmbracelet/bubbletea v0.24.2 h1:uaQIKx9Ai6Gdh5zpTbGiWpytMU+CfsPp06RaW2cx/SY= -github.com/charmbracelet/bubbletea v0.24.2/go.mod h1:XdrNrV4J8GiyshTtx3DNuYkR1FDaJmO3l2nejekbsgg= -github.com/charmbracelet/lipgloss v0.7.1 h1:17WMwi7N1b1rVWOjMT+rCh7sQkvDU75B2hbZpc5Kc1E= -github.com/charmbracelet/lipgloss v0.7.1/go.mod h1:yG0k3giv8Qj8edTCbbg6AlQ5e8KNWpFujkNawKNhE2c= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= +github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= +github.com/charmbracelet/bubbletea v1.1.0 h1:FjAl9eAL3HBCHenhz/ZPjkKdScmaS5SK69JAK2YJK9c= +github.com/charmbracelet/bubbletea v1.1.0/go.mod h1:9Ogk0HrdbHolIKHdjfFpyXJmiCzGwy+FesYkZr7hYU4= +github.com/charmbracelet/lipgloss v0.13.0 h1:4X3PPeoWEDCMvzDvGmTajSyYPcZM4+y8sCA/SsA3cjw= +github.com/charmbracelet/lipgloss v0.13.0/go.mod h1:nw4zy0SBX/F/eAO1cWdcvy6qnkDUxr8Lw7dvFrAIbbY= +github.com/charmbracelet/x/ansi v0.2.3 h1:VfFN0NUpcjBRd4DnKfRaIRo53KRgey/nhOoEqosGDEY= +github.com/charmbracelet/x/ansi v0.2.3/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw= +github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q= +github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/term v0.2.0 h1:cNB9Ot9q8I711MyZ7myUR5HFWL/lc3OpU8jZ4hwm0x0= +github.com/charmbracelet/x/term v0.2.0/go.mod h1:GVxgxAbjUrmpvIINHIQnJJKpMlHiZ4cktEQCN6GWyF0= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= -github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= -github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= -github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 h1:q2hJAaP1k2wIvVRd/hEHD7lacgqrCPS+k8g1MndzfWY= -github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e h1:GLC8iDDcbt1H8+RkNao2nRGjyNTIo81e1rAJT9/uWYA= +github.com/city404/v6-public-rpc-proto/go v0.0.0-20240817070657-90f8e24b653e/go.mod h1:ln9Whp+wVY/FTbn2SK0ag+SKD2fC0yQCF/Lqowc1LmU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= +github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/coreos/go-oidc v2.2.1+incompatible h1:mh48q/BqXqgjVHpy2ZY7WnWAbenxRjsz9N1i1YxjHAk= github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= +github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= -github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= +github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8 h1:OtSeLS5y0Uy01jaKK4mA/WVIYtpzVm63vLVAPzJXigg= +github.com/dhowden/tag v0.0.0-20240417053706-3d75831295e8/go.mod h1:apkPC/CR3s48O2D7Y++n1XWEpgPNNCjXYga3PPbJe2E= github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= -github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU= -github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= +github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJLNCEi7YHVMkwwtfSr2k9splgdSM= github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8= -github.com/foxxorcat/mopan-sdk-go v0.1.3 h1:6ww0ulyLDh6neXZBqUM2PDbxQ6lfdkQbr0FCh9BTY0Y= -github.com/foxxorcat/mopan-sdk-go v0.1.3/go.mod h1:iWHA2JFhzmKR28ySp1ON0g6DjLaYtvb5jhTqPVTDW9A= -github.com/foxxorcat/mopan-sdk-go v0.1.4 h1:6utvPiBv8KDRDVKB7A4FERdrVxcHKZd2fBFCNuKcXzU= -github.com/foxxorcat/mopan-sdk-go v0.1.4/go.mod h1:iWHA2JFhzmKR28ySp1ON0g6DjLaYtvb5jhTqPVTDW9A= -github.com/foxxorcat/weiyun-sdk-go v0.1.2 h1:waRWIBmjL9GCcndJ8HvOYrrVB4hhoPYzRrn3I/Cnzqw= -github.com/foxxorcat/weiyun-sdk-go v0.1.2/go.mod h1:AKsLFuWhWlClpGrg1zxTdMejugZEZtmhIuElAk3W83s= +github.com/emersion/go-message v0.18.0 h1:7LxAXHRpSeoO/Wom3ZApVZYG7c3d17yCScYce8WiXA8= +github.com/emersion/go-message v0.18.0/go.mod h1:Zi69ACvzaoV/MBnrxfVBPV3xWEuCmC2nEN39oJF4B8A= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= +github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 h1:ATgqloALX6cHCranzkLb8/zjivwQ9DWWDCQRnxTPfaA= +github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fatedier/frp v0.68.0 h1:woKC31EpgCLQDirRAOAUgHP3IRxLu2mBHS6zGpcw7tw= +github.com/fatedier/frp v0.68.0/go.mod h1:qFdez6Z+RDqoqF1xJhh48+IW91SVQ+pNWnrmcl43Wjs= +github.com/fatedier/golib v0.5.1 h1:hcKAnaw5mdI/1KWRGejxR+i1Hn/NvbY5UsMKDr7o13M= +github.com/fatedier/golib v0.5.1/go.mod h1:W6kIYkIFxHsTzbgqg5piCxIiDo4LzwgTY6R5W8l9NFQ= +github.com/fclairamb/go-log v0.5.0 h1:Gz9wSamEaA6lta4IU2cjJc2xSq5sV5VYSB5w/SUHhVc= +github.com/fclairamb/go-log v0.5.0/go.mod h1:XoRO1dYezpsGmLLkZE9I+sHqpqY65p8JA+Vqblb7k40= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/foxxorcat/mopan-sdk-go v0.1.6 h1:6J37oI4wMZLj8EPgSCcSTTIbnI5D6RCNW/srX8vQd1Y= +github.com/foxxorcat/mopan-sdk-go v0.1.6/go.mod h1:UaY6D88yBXWGrcu/PcyLWyL4lzrk5pSxSABPHftOvxs= +github.com/foxxorcat/weiyun-sdk-go v0.1.3 h1:I5c5nfGErhq9DBumyjCVCggRA74jhgriMqRRFu5jeeY= +github.com/foxxorcat/weiyun-sdk-go v0.1.3/go.mod h1:TPxzN0d2PahweUEHlOBWlwZSA+rELSUlGYMWgXRn9ps= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= -github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gaoyb7/115drive-webdav v0.1.8 h1:EJt4PSmcbvBY4KUh2zSo5p6fN9LZFNkIzuKejipubVw= -github.com/gaoyb7/115drive-webdav v0.1.8/go.mod h1:BKbeY6j8SKs3+rzBFFALznGxbPmefEm3vA+dGhqgOGU= +github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= +github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w= github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc= -github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= -github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= +github.com/gin-contrib/cors v1.7.2 h1:oLDHxdg8W/XDoN/8zamqk/Drgt4oVZDvaV0YmvVICQw= +github.com/gin-contrib/cors v1.7.2/go.mod h1:SUJVARKgQ40dmrzgXEVxj2m7Ig1v1qIboQkPDTQ9t2E= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= -github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= -github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= -github.com/go-chi/chi/v5 v5.0.10 h1:rLz5avzKpjqxrYwXNfmjkrYYXOyLJd37pz53UFHC6vk= -github.com/go-chi/chi/v5 v5.0.10/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-chi/chi/v5 v5.0.12 h1:9euLV5sTrTNTRUU9POmDUvfxyj6LAABLUcEWO+JJb4s= +github.com/go-chi/chi/v5 v5.0.12/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= +github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= -github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= -github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= -github.com/go-resty/resty/v2 v2.8.0 h1:J29d0JFWwSWrDCysnOK/YjsPMLQTx0TvgJEHVGvf2L8= -github.com/go-resty/resty/v2 v2.8.0/go.mod h1:UCui0cMHekLrSntoMyofdSTaPpinlRHFtPpizuyDW2w= -github.com/go-resty/resty/v2 v2.9.1 h1:PIgGx4VrHvag0juCJ4dDv3MiFRlDmP0vicBucwf+gLM= -github.com/go-resty/resty/v2 v2.9.1/go.mod h1:4/GYJVjh9nhkhGR6AUNW3XhpDYNUr+Uvy9gV/VGZIy4= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-resty/resty/v2 v2.14.0 h1:/rhkzsAqGQkozwfKS5aFAbb6TyKd3zyFRWcdRXLPCAU= +github.com/go-resty/resty/v2 v2.14.0/go.mod h1:IW6mekUOsElt9C7oWr0XRt9BNSD6D5rr9mhk6NjmNHg= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= -github.com/go-webauthn/webauthn v0.8.6 h1:bKMtL1qzd2WTFkf1mFTVbreYrwn7dsYmEPjTq6QN90E= -github.com/go-webauthn/webauthn v0.8.6/go.mod h1:emwVLMCI5yx9evTTvr0r+aOZCdWJqMfbRhF0MufyUog= -github.com/go-webauthn/x v0.1.4 h1:sGmIFhcY70l6k7JIDfnjVBiAAFEssga5lXIUXe0GtAs= -github.com/go-webauthn/x v0.1.4/go.mod h1:75Ug0oK6KYpANh5hDOanfDI+dvPWHk788naJVG/37H8= -github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/go-webauthn/webauthn v0.11.1 h1:5G/+dg91/VcaJHTtJUfwIlNJkLwbJCcnUc4W8VtkpzA= +github.com/go-webauthn/webauthn v0.11.1/go.mod h1:YXRm1WG0OtUyDFaVAgB5KG7kVqW+6dYCJ7FTQH4SxEE= +github.com/go-webauthn/x v0.1.12 h1:RjQ5cvApzyU/xLCiP+rub0PE4HBZsLggbxGR5ZpUf/A= +github.com/go-webauthn/x v0.1.12/go.mod h1:XlRcGkNH8PT45TfeJYc6gqpOtiOendHhVmnOxh+5yHs= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 h1:gtexQ/VGyN+VVFRXSFiguSNcXmS6rkKT+X7FdIrTtfo= github.com/golang/geo v0.0.0-20210211234256-740aa86cb551/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-tpm v0.9.0 h1:sQF6YqWMi+SCXpsmS3fd21oPy/vSddwZry4JnmltHVk= -github.com/google/go-tpm v0.9.0/go.mod h1:FkNVkc6C+IsvDI9Jw1OveJmxGZUUaKxtrpOS47QWKfU= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.1 h1:0pGc4X//bAlmZzMKf8iz6IsDo1nYTbYJ6FZN/rg4zdM= +github.com/google/go-tpm v0.9.1/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.12.2 h1:mhN09QQW1jEWeMF74zGR81R30z4VJzjZsfkUhuHF+DA= +github.com/googleapis/gax-go/v2 v2.12.2/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE= +github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= +github.com/hekmon/cunits/v2 v2.1.0 h1:k6wIjc4PlacNOHwKEMBgWV2/c8jyD4eRMs5mR1BBhI0= +github.com/hekmon/cunits/v2 v2.1.0/go.mod h1:9r1TycXYXaTmEWlAIfFV8JT+Xo59U96yUJAYHxzii2M= +github.com/hekmon/transmissionrpc/v3 v3.0.0 h1:0Fb11qE0IBh4V4GlOwHNYpqpjcYDp5GouolwrpmcUDQ= +github.com/hekmon/transmissionrpc/v3 v3.0.0/go.mod h1:38SlNhFzinVUuY87wGj3acOmRxeYZAZfrj6Re7UgCDg= +github.com/henrybear327/Proton-API-Bridge v1.0.0 h1:gjKAaWfKu++77WsZTHg6FUyPC5W0LTKWQciUm8PMZb0= +github.com/henrybear327/Proton-API-Bridge v1.0.0/go.mod h1:gunH16hf6U74W2b9CGDaWRadiLICsoJ6KRkSt53zLts= +github.com/henrybear327/go-proton-api v1.0.0 h1:zYi/IbjLwFAW7ltCeqXneUGJey0TN//Xo851a/BgLXw= +github.com/henrybear327/go-proton-api v1.0.0/go.mod h1:w63MZuzufKcIZ93pwRgiOtxMXYafI8H74D77AxytOBc= github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI= github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs= -github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= -github.com/ipfs/go-ipfs-api v0.6.1 h1:nK5oeFOdMh1ogT+GCOcyBFOOcFGNuudSb1rg9YDyAKE= -github.com/ipfs/go-ipfs-api v0.6.1/go.mod h1:8pl+ZMF2LX42szbqGbpOBEiI1/rYaImvTvJtG0g+rL4= github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2Q= github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.3.0 h1:/NQi8KHMpKWHInxXesC8yD4DhkXPrVhmnwYkjp9AmBA= -github.com/jackc/pgx/v5 v5.3.0/go.mod h1:t3JDKnCBlYIc0ewLF0Q7B8MXmoIaBOZj/ic7iHozM/8= -github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.9.0 h1:T/dI+2TvmI2H8s/KH1/lXIbz1CUFk3gn5oTjr0/mBsE= +github.com/jackc/pgx/v5 v5.9.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jlaffaye/ftp v0.2.0 h1:lXNvW7cBu7R/68bknOX3MrRIIqZ61zELs1P2RAiA3lg= @@ -270,32 +437,49 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 h1:G+9t9cEtnC9jFiTxyptEKuNIAbiN5ZCQzX2a74lj3xg= github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004/go.mod h1:KmHnJWQrgEvbuy0vcvj00gtMqbvNn1L+3YUZLK/B92c= +github.com/kdomanski/iso9660 v0.4.0 h1:BPKKdcINz3m0MdjIMwS0wx1nofsOjxOq8TOr45WGHFg= +github.com/kdomanski/iso9660 v0.4.0/go.mod h1:OxUSupHsO9ceI8lBLPJKWBTphLemjrCQY8LPXM7qSzU= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.12.0 h1:I5FEp3xSwVCcEh3F5A7dofEfhXdF/bWhQWPH+XwBFno= +github.com/klauspost/reedsolomon v1.12.0/go.mod h1:EPLZJeh4l27pUGC3aXOjheaoh1I9yut7xTURiW3LQ9Y= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1 h1:vAdu+sX9yXNkKnKnYQeIv6yBkjP37Q1JEJHmMa2eCjQ= +github.com/larksuite/oapi-sdk-go/v3 v3.6.1/go.mod h1:ZEplY+kwuIrj/nqw5uSCINNATcH3KdxSN7y+UxYY5fI= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= github.com/libp2p/go-buffer-pool v0.1.0/go.mod h1:N+vh8gMqimBzdKkSMVuydVDq+UV5QTWy5HSiZacSbPg= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= @@ -305,27 +489,37 @@ github.com/libp2p/go-libp2p v0.27.8/go.mod h1:eCFFtd0s5i/EVKR7+5Ki8bM7qwkNW3TPTT github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= -github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= -github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= -github.com/maruel/natural v1.1.0 h1:2z1NgP/Vae+gYrtC0VuvrTJ6U35OuyUqDdfluLqMWuQ= -github.com/maruel/natural v1.1.0/go.mod h1:eFVhYCcUOfZFxXoDZam8Ktya72wa79fNC3lc/leA0DQ= +github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed h1:036IscGBfJsFIgJQzlui7nK1Ncm0tp2ktmPj8xO4N/0= +github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mark3labs/mcp-go v0.48.0 h1:o+MXuGW/HCeR2ny5LcAcZQn2bo6I2xaZMEHnpRG+dtw= +github.com/mark3labs/mcp-go v0.48.0/go.mod h1:JKTC7R2LLVagkEWK7Kwu7DbmA6iIvnNAod6yrHiQMag= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= +github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= -github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= -github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= -github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= -github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/meilisearch/meilisearch-go v0.27.2 h1:3G21dJ5i208shnLPDsIEZ0L0Geg/5oeXABFV7nlK94k= +github.com/meilisearch/meilisearch-go v0.27.2/go.mod h1:SxuSqDcPBIykjWz1PX+KzsYzArNLSCadQodWs8extS0= +github.com/mholt/archives v0.1.0 h1:FacgJyrjiuyomTuNA92X5GyRBRZjE43Y/lrzKIlF35Q= +github.com/mholt/archives v0.1.0/go.mod h1:j/Ire/jm42GN7h90F5kzj6hf6ZFzEH66de+hmjEKu+I= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/minio/sio v0.4.0 h1:u4SWVEm5lXSqU42ZWawV0D9I5AZ5YMmo2RXpEQ/kRhc= +github.com/minio/sio v0.4.0/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -341,14 +535,12 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= -github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b h1:1XF24mVaiu7u+CFywTdcDo2ie1pzzhwjt6RHqzpMU34= -github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b/go.mod h1:fQuZ0gauxyBcmsdE3ZT4NasjaRdxmbCS0jRHsrWu3Ho= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= -github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= -github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= -github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs= -github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= @@ -357,46 +549,55 @@ github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sgg github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= -github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= -github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= -github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= -github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= -github.com/orzogc/fake115uploader v0.3.3-0.20221009101310-08b764073b77 h1:dg/EaaJLPIg4xn2kaZil7Ax3wfoxcFXaBwyOTlcz5AI= -github.com/orzogc/fake115uploader v0.3.3-0.20221009101310-08b764073b77/go.mod h1:FD9a09Vw07CSMTdT0Y7ttStOa1WZsnPBslliMw2DkeM= -github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831 h1:K3T3eu4h5aYIOzUtLjN08L4Qt4WGaJONMgcaD0ayBJQ= -github.com/orzogc/fake115uploader v0.3.3-0.20230715111618-58f9eb76f831/go.mod h1:lSHD4lC4zlMl+zcoysdJcd5KFzsWwOD8BJbyg1Ws9Ng= +github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= +github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= +github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78 h1:MYzLheyVx1tJVDqfu3YnN4jtnyALNzLvwl+f58TcvQY= +github.com/nwaples/rardecode/v2 v2.0.0-beta.4.0.20241112120701-034e449c6e78/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/okatu-loli/115driver v1.2.3-1 h1:UoBEREqh6RD6WlxiJ2Z29JxNZ/UcoChvdHn9r9Tx7nI= +github.com/okatu-loli/115driver v1.2.3-1/go.mod h1:Zk7Qz7SYO1QU0SJIne6DnUD2k36S3wx/KbsQpxcfY/Y= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= +github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= +github.com/otiai10/mint v1.5.1/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= -github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= +github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= +github.com/pires/go-proxyproto v0.7.0 h1:IukmRewDQFWC7kfnb66CSomk2q/seBuilHBYFwyq0Hs= +github.com/pires/go-proxyproto v0.7.0/go.mod h1:Vz/1JPY/OACxWGQNIRY2BeyDmpoaWmEP40O9LbuiFR4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 h1:5TvW1dv00Y13njmQ1AWkxSWtPkwE7ZEF6yDuv9q+Als= -github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= github.com/pkg/xattr v0.4.9 h1:5883YPCtkSd8LFbs13nXplj9g9tlrwoJRjgpgMu1/fE= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pkg/xattr v0.4.9/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= @@ -404,244 +605,499 @@ github.com/pquerna/cachecontrol v0.1.0 h1:yJMy84ti9h/+OEWa752kBTKv4XC30OtVVHYv/8 github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/rclone/rclone v1.63.1 h1:iITCUNBfAXnguHjRPFq+w/gGIW0L0las78h4H5CH2Ms= -github.com/rclone/rclone v1.63.1/go.mod h1:eUQaKsf1wJfHKB0RDoM8RaPAeRB2eI/Qw+Vc9Ho5FGM= +github.com/prometheus/client_golang v1.19.1 h1:wZWJDwK+NameRJuPGDhlnFgx8e8HN3XHQeLaYJFJBOE= +github.com/prometheus/client_golang v1.19.1/go.mod h1:mP78NwGzrVks5S2H6ab8+ZZGJLZUq1hoULYBAYBw1Ho= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk= +github.com/quic-go/quic-go v0.55.0/go.mod h1:DR51ilwU1uE164KuWXhinFcKWGlEjzys2l8zUl5Ss1U= +github.com/rclone/rclone v1.67.0 h1:yLRNgHEG2vQ60HCuzFqd0hYwKCRuWuvPUhvhMJ2jI5E= +github.com/rclone/rclone v1.67.0/go.mod h1:Cb3Ar47M/SvwfhAjZTbVXdtrP/JLtPFCq2tkdtBVC6w= +github.com/relvacode/iso8601 v1.3.0 h1:HguUjsGpIMh/zsTczGN3DVJFxTU/GX+MMmzcKoMO7ko= +github.com/relvacode/iso8601 v1.3.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= github.com/rfjakob/eme v1.1.2 h1:SxziR8msSOElPayZNFfQw4Tjx/Sbaeeh3eRvrHVMUs4= github.com/rfjakob/eme v1.1.2/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k= -github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= -github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4= -github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= +github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/samber/lo v1.47.0 h1:z7RynLwP5nbyRscyvcD043DWYoOcYRv3mV8lBeqOCLc= +github.com/samber/lo v1.47.0/go.mod h1:RmDH9Ct32Qy3gduHQuKJ3gW1fMHAnE/fAzQuf6He5cU= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4 h1:PT+ElG/UUFMfqy5HrxJxNzj3QBOf7dZwupeVC+mG1Lo= +github.com/secsy/goftp v0.0.0-20200609142545-aa2de14babf4/go.mod h1:MnkX001NG75g3p8bhFycnyIjeQoOjGL6CEIsdE/nKSY= +github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= +github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= +github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= +github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/go-m1cpu v0.2.1 h1:yqRB4fvOge2+FyRXFkXqsyMoqPazv14Yyy+iyccT2E4= +github.com/shoenig/go-m1cpu v0.2.1/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M= github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EEf9cgbU6AtGPK4CTG3Zf6CKMNqf0MHTggAUA= +github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8 h1:TG/diQgUe0pntT/2D9tmUCz4VNwm9MfrtPr0SU2qSX8= +github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8/go.mod h1:P5HUIBuIWKbyjl083/loAegFkfbFNx5i2qEP4CNbm7E= +github.com/sorairolake/lzip-go v0.3.5 h1:ms5Xri9o1JBIWvOFAorYtUNik6HI3HgBTkISiqu0Cwg= +github.com/sorairolake/lzip-go v0.3.5/go.mod h1:N0KYq5iWrMXI0ZEXKXaS9hCyOjZUQdBDEIbXfoUwbdk= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca h1:I9rVnNXdIkij4UvMT7OmKhH9sOIvS8iXkxfPdnn9wQA= -github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= -github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 h1:Jtcrb09q0AVWe3BGe8qtuuGxNSHWGkTWr43kHTJ+CpA= +github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7/go.mod h1:suDIky6yrK07NnaBadCB4sS0CqFOvUK91lH7CR+JlDA= +github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 h1:6Y51mutOvRGRx6KqyMNo//xk8B8o6zW9/RVmy1VamOs= +github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543/go.mod h1:jpwqYA8KUVEvSUJHkCXsnBRJCSKP1BMa81QZ6kvRpow= +github.com/templexxx/cpu v0.1.1 h1:isxHaxBXpYFWnk2DReuKkigaZyrjs2+9ypIdGP4h+HI= +github.com/templexxx/cpu v0.1.1/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= +github.com/templexxx/xorsimd v0.4.3 h1:9AQTFHd7Bhk3dIT7Al2XeBX5DWOvsUPZCuhyAtNbHjU= +github.com/templexxx/xorsimd v0.4.3/go.mod h1:oZQcD6RFDisW2Am58dSAGwwL6rHjbzrlu25VDqfWkQg= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= +github.com/tjfoc/gmsm v1.4.1 h1:aMe1GlZb+0bLjn+cKTPEvvn9oUEBlJitaZiiBwsbgho= +github.com/tjfoc/gmsm v1.4.1/go.mod h1:j4INPkHWMrhJb38G+J6W4Tw0AbuN8Thu3PbdVYhVcTE= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4= +github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4= +github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/u2takey/ffmpeg-go v0.4.1 h1:l5ClIwL3N2LaH1zF3xivb3kP2HW95eyG5xhHE1JdZ9Y= -github.com/u2takey/ffmpeg-go v0.4.1/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= github.com/u2takey/ffmpeg-go v0.5.0 h1:r7d86XuL7uLWJ5mzSeQ03uvjfIhiJYvsRAJFCW4uklU= github.com/u2takey/ffmpeg-go v0.5.0/go.mod h1:ruZWkvC1FEiUNjmROowOAps3ZcWxEiOpFoHCvk97kGc= github.com/u2takey/go-utils v0.3.1 h1:TaQTgmEZZeDHQFYfd+AdUT1cT4QJgJn/XVPELhHw4ys= github.com/u2takey/go-utils v0.3.1/go.mod h1:6e+v5vEZ/6gu12w/DC2ixZdZtCrNokVxD0JUklcqdCs= -github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= -github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= -github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q= github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E= -github.com/valyala/fastjson v1.6.3 h1:tAKFnnwmeMGPbwJ7IwxcTPCNr3uIzoIj3/Fh90ra4xc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d h1:xS9QTPgKl9ewGsAOPc+xW7DeStJDqYPfisDmeSCcbco= +github.com/valyala/fasthttp v1.37.1-0.20220607072126-8a320890c08d/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk= +github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs= +github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= +github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5 h1:jxZvjx8Ve5sOXorZG0KzTxbp0Cr1n3FEegfmyd9br1k= github.com/winfsp/cgofuse v1.5.1-0.20230130140708-f87f5db493b5/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xhofe/115-sdk-go v0.1.5 h1:2+E92l6AX0+ABAkrdmDa9PE5ONN7wVLCaKkK80zETOg= +github.com/xhofe/115-sdk-go v0.1.5/go.mod h1:MIdpe/4Kw4ODrPld7E11bANc4JsCuXcm5ZZBHSiOI0U= +github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25 h1:eDfebW/yfq9DtG9RO3KP7BT2dot2CvJGIvrB0NEoDXI= +github.com/xhofe/gsync v0.0.0-20230917091818-2111ceb38a25/go.mod h1:fH4oNm5F9NfI5dLi0oIMtsLNKQOirUDbEMCIBb/7SU0= +github.com/xhofe/tache v0.1.5 h1:ezDcgim7tj7KNMXliQsmf8BJQbaZtitfyQA9Nt+B4WM= +github.com/xhofe/tache v0.1.5/go.mod h1:PYt6I/XUKliSg1uHlgsk6ha+le/f6PAvjUtFZAVl3a8= +github.com/xhofe/wopan-sdk-go v0.1.3 h1:J58X6v+n25ewBZjb05pKOr7AWGohb+Rdll4CThGh6+A= +github.com/xhofe/wopan-sdk-go v0.1.3/go.mod h1:dcY9yA28fnaoZPnXZiVTFSkcd7GnIPTpTIIlfSI5z5Q= +github.com/xtaci/kcp-go/v5 v5.6.13 h1:FEjtz9+D4p8t2x4WjciGt/jsIuhlWjjgPCCWjrVR4Hk= +github.com/xtaci/kcp-go/v5 v5.6.13/go.mod h1:75S1AKYYzNUSXIv30h+jPKJYZUwqpfvLshu63nCNSOM= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37 h1:EWU6Pktpas0n8lLQwDsRyZfmkPeRbdgPtW609es+/9E= +github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37/go.mod h1:HpMP7DB2CyokmAh4lp0EQnnWhmycP/TvwBGzvuie+H0= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9 h1:K8gF0eekWPEX+57l30ixxzGhHH/qscI3JCnuhbN6V4M= +github.com/yeka/zip v0.0.0-20231116150916-03d6312748a9/go.mod h1:9BnoKCcgJ/+SLhfAXj15352hTOuVmG5Gzo8xNRINfqI= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= -go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= +github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22 h1:X+lHsNTlbatQ1cErXIbtyrh+3MTWxqQFS+sBP/wpFXo= +github.com/zzzhr1990/go-common-entity v0.0.0-20221216044934-fd1c571e3a22/go.mod h1:1zGRDJd8zlG6P8azG96+uywfh6udYWwhOmUivw+xsuM= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= +go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go4.org v0.0.0-20230225012048-214862532bf5 h1:nifaUDeh+rPaBCMPMQHZmvJf+QdpLFnuQPwx+LxVmtc= +go4.org v0.0.0-20230225012048-214862532bf5/go.mod h1:F57wTi5Lrj6WLyswp5EYV1ncrEbFGHD4hhz6S1ZYeaU= gocv.io/x/gocv v0.25.0/go.mod h1:Rar2PS6DV+T4FL+PM535EImD/h13hGVaHhnCu1xarBs= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= -golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= -golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI= -golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e h1:I88y4caeGeuDQxgdoFPUq097j7kNfw6uvuiNxUBfcBk= +golang.org/x/exp v0.0.0-20240904232852-e7e105dedf7e/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.10.0 h1:gXjUUtwtx5yOE0VKWq1CH4IJAClq4UGgUA3i+rpON9M= -golang.org/x/image v0.10.0/go.mod h1:jtrku+n79PfroUbvDdeUWMAI+heR786BofxrbiSF+J0= -golang.org/x/image v0.11.0 h1:ds2RoQvBvYTiJkwpSFDwCcDFNX7DqjL2WsUgTNk0Ooo= -golang.org/x/image v0.11.0/go.mod h1:bglhjqbqVuEb9e9+eNR45Jfu7D+T4Qan+NhQk8Ck2P8= +golang.org/x/image v0.19.0 h1:D9FX4QWkLfkeqaC62SonffIIuYdOk/UE2XKUBgRIBIQ= +golang.org/x/image v0.19.0/go.mod h1:y0zrRqlQRWQ5PXaYCOMLTW2fpsxZ8Qh9I/ohnInJEys= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= -golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.16.0 h1:7eBu7KsSvFDtSXUIDbh3aqlK4DPsZ1rByC8PFfBThos= -golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= +golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220702020025-31831981b65f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= +golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= +golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= +golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.134.0 h1:ktL4Goua+UBgoP1eL1/60LwZJqa1sIzkLmvoR3hR6Gw= -google.golang.org/api v0.134.0/go.mod h1:sjRL3UnjTx5UqNQS9EWr9N8p7xbHpy1k0XGRLCf3Spk= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5 h1:eSaPbMR4T7WfH9FvABk36NBMacoTUKdWCvV0dx+KfOg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230803162519-f966b187b2e5/go.mod h1:zBEcrKX2ZOcEkHWxBPAIvYUWOKKMIhYcmNiUIu2ji3I= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.169.0 h1:QwWPy71FgMWqJN/l6jVlFHUa29a7dcUy02I8o799nPY= +google.golang.org/api v0.169.0/go.mod h1:gpNOiMA2tZ4mf5R9Iwf4rK/Dcz0fbdIgWYWVoxmsyLg= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= +gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= @@ -652,21 +1108,39 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.4.7 h1:rY46lkCspzGHn7+IYsNpSfEv9tA+SU4SkkB+GFX125Y= -gorm.io/driver/mysql v1.4.7/go.mod h1:SxzItlnT1cb6e1e4ZRpgJN2VYtcqJgqnHxWr4wsP8oc= -gorm.io/driver/postgres v1.4.8 h1:NDWizaclb7Q2aupT0jkwK8jx1HVCNzt+PQ8v/VnxviA= -gorm.io/driver/postgres v1.4.8/go.mod h1:O9MruWGNLUBUWVYfWuBClpf3HeGjOoybY0SNmCs3wsw= -gorm.io/driver/sqlite v1.4.4 h1:gIufGoR0dQzjkyqDyYSCvsYR6fba1Gw5YKDqKeChxFc= -gorm.io/driver/sqlite v1.4.4/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= -gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= -gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= -gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= -gorm.io/gorm v1.24.5 h1:g6OPREKqqlWq4kh/3MCQbZKImeB9e6Xgc4zD+JgNZGE= -gorm.io/gorm v1.24.5/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= +gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= +gorm.io/driver/postgres v1.5.9 h1:DkegyItji119OlcaLjqN11kHoUgZ/j13E0jkJZgD6A8= +gorm.io/driver/postgres v1.5.9/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/driver/sqlite v1.5.6 h1:fO/X46qn5NUEEOZtnjJRWRzZMe8nqJiQ9E+0hi+hKQE= +gorm.io/driver/sqlite v1.5.6/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.11 h1:/Wfyg1B/je1hnDx3sMkX+gAlxrlZpn6X0BXRlwXlvHg= +gorm.io/gorm v1.25.11/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/apimachinery v0.28.8 h1:hi/nrxHwk4QLV+W/SHve1bypTE59HCDorLY1stBIxKQ= +k8s.io/apimachinery v0.28.8/go.mod h1:cBnwIM3fXoRo28SqbV/Ihxf/iviw85KyXOrzxvZQ83U= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= +k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= +resty.dev/v3 v3.0.0-beta.2 h1:xu4mGAdbCLuc3kbk7eddWfWm4JfhwDtdapwss5nCjnQ= +resty.dev/v3 v3.0.0-beta.2/go.mod h1:OgkqiPvTDtOuV4MGZuUDhwOpkY8enjOsjjMzeOHefy4= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/archive/all.go b/internal/archive/all.go new file mode 100644 index 00000000000..63206cb89ed --- /dev/null +++ b/internal/archive/all.go @@ -0,0 +1,9 @@ +package archive + +import ( + _ "github.com/alist-org/alist/v3/internal/archive/archives" + _ "github.com/alist-org/alist/v3/internal/archive/iso9660" + _ "github.com/alist-org/alist/v3/internal/archive/rardecode" + _ "github.com/alist-org/alist/v3/internal/archive/sevenzip" + _ "github.com/alist-org/alist/v3/internal/archive/zip" +) diff --git a/internal/archive/archives/archives.go b/internal/archive/archives/archives.go new file mode 100644 index 00000000000..e3a48b15757 --- /dev/null +++ b/internal/archive/archives/archives.go @@ -0,0 +1,171 @@ +package archives + +import ( + "fmt" + "io" + "io/fs" + "os" + stdpath "path" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type Archives struct { +} + +func (Archives) AcceptedExtensions() []string { + return []string{ + ".br", ".bz2", ".gz", ".lz4", ".lz", ".sz", ".s2", ".xz", ".zz", ".zst", ".tar", + } +} + +func (Archives) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{} +} + +func (Archives) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + fsys, err := getFs(ss[0], args) + if err != nil { + return nil, err + } + files, err := fsys.ReadDir(".") + if err != nil { + return nil, filterPassword(err) + } + + tree := make([]model.ObjTree, 0, len(files)) + for _, file := range files { + info, err := file.Info() + if err != nil { + continue + } + tree = append(tree, &model.ObjectTree{Object: *toModelObj(info)}) + } + return &model.ArchiveMetaInfo{ + Comment: "", + Encrypted: false, + Tree: tree, + }, nil +} + +func (Archives) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + fsys, err := getFs(ss[0], args.ArchiveArgs) + if err != nil { + return nil, err + } + innerPath := strings.TrimPrefix(args.InnerPath, "/") + if innerPath == "" { + innerPath = "." + } + obj, err := fsys.ReadDir(innerPath) + if err != nil { + return nil, filterPassword(err) + } + return utils.SliceConvert(obj, func(src os.DirEntry) (model.Obj, error) { + info, err := src.Info() + if err != nil { + return nil, err + } + return toModelObj(info), nil + }) +} + +func (Archives) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + fsys, err := getFs(ss[0], args.ArchiveArgs) + if err != nil { + return nil, 0, err + } + file, err := fsys.Open(strings.TrimPrefix(args.InnerPath, "/")) + if err != nil { + return nil, 0, filterPassword(err) + } + stat, err := file.Stat() + if err != nil { + return nil, 0, filterPassword(err) + } + return file, stat.Size(), nil +} + +func (Archives) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + fsys, err := getFs(ss[0], args.ArchiveArgs) + if err != nil { + return err + } + isDir := false + path := strings.TrimPrefix(args.InnerPath, "/") + if path == "" { + isDir = true + path = "." + } else { + stat, err := fsys.Stat(path) + if err != nil { + return filterPassword(err) + } + if stat.IsDir() { + isDir = true + outputPath = filepath.Join(outputPath, stat.Name()) + err = os.Mkdir(outputPath, 0700) + if err != nil { + return filterPassword(err) + } + } + } + if isDir { + err = fs.WalkDir(fsys, path, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if p == path { + if d.IsDir() { + return nil + } + } + relPath := strings.TrimPrefix(p, path+"/") + if relPath == "" || relPath == "." { + if d.IsDir() { + return nil + } + } + dstPath, err := tool.SecureJoin(outputPath, relPath) + if err != nil { + return err + } + if d.IsDir() { + return os.MkdirAll(dstPath, 0700) + } + info, err := d.Info() + if err != nil { + return err + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, p) + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + return decompress(fsys, p, dstPath, func(_ float64) {}) + }) + } else { + entryName := stdpath.Base(path) + dstPath, e := tool.SecureJoin(outputPath, entryName) + if e != nil { + return e + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = decompress(fsys, path, dstPath, up) + } + return filterPassword(err) +} + +var _ tool.Tool = (*Archives)(nil) + +func init() { + tool.RegisterTool(Archives{}) +} diff --git a/internal/archive/archives/utils.go b/internal/archive/archives/utils.go new file mode 100644 index 00000000000..5249862ce4b --- /dev/null +++ b/internal/archive/archives/utils.go @@ -0,0 +1,89 @@ +package archives + +import ( + "fmt" + "io" + fs2 "io/fs" + "os" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/mholt/archives" +) + +func getFs(ss *stream.SeekableStream, args model.ArchiveArgs) (*archives.ArchiveFS, error) { + reader, err := stream.NewReadAtSeeker(ss, 0) + if err != nil { + return nil, err + } + if r, ok := reader.(*stream.RangeReadReadAtSeeker); ok { + r.InitHeadCache() + } + format, _, err := archives.Identify(ss.Ctx, ss.GetName(), reader) + if err != nil { + return nil, errs.UnknownArchiveFormat + } + extractor, ok := format.(archives.Extractor) + if !ok { + return nil, errs.UnknownArchiveFormat + } + switch f := format.(type) { + case archives.SevenZip: + f.Password = args.Password + case archives.Rar: + f.Password = args.Password + } + return &archives.ArchiveFS{ + Stream: io.NewSectionReader(reader, 0, ss.GetSize()), + Format: extractor, + Context: ss.Ctx, + }, nil +} + +func toModelObj(file os.FileInfo) *model.Object { + return &model.Object{ + Name: file.Name(), + Size: file.Size(), + Modified: file.ModTime(), + IsFolder: file.IsDir(), + } +} + +func filterPassword(err error) error { + if err != nil && strings.Contains(err.Error(), "password") { + return errs.WrongArchivePassword + } + return err +} + +func decompress(fsys fs2.FS, filePath, dstPath string, up model.UpdateProgress) error { + rc, err := fsys.Open(filePath) + if err != nil { + return err + } + defer rc.Close() + stat, err := rc.Stat() + if err != nil { + return err + } + if !stat.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, filePath) + } + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer f.Close() + _, err = utils.CopyWithBuffer(f, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: rc, + Size: stat.Size(), + }, + UpdateProgress: up, + }) + return err +} diff --git a/internal/archive/iso9660/iso9660.go b/internal/archive/iso9660/iso9660.go new file mode 100644 index 00000000000..7de8da6f393 --- /dev/null +++ b/internal/archive/iso9660/iso9660.go @@ -0,0 +1,103 @@ +package iso9660 + +import ( + "io" + "os" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/kdomanski/iso9660" +) + +type ISO9660 struct { +} + +func (ISO9660) AcceptedExtensions() []string { + return []string{".iso"} +} + +func (ISO9660) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{} +} + +func (ISO9660) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + return &model.ArchiveMetaInfo{ + Comment: "", + Encrypted: false, + }, nil +} + +func (ISO9660) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + img, err := getImage(ss[0]) + if err != nil { + return nil, err + } + dir, err := getObj(img, args.InnerPath) + if err != nil { + return nil, err + } + if !dir.IsDir() { + return nil, errs.NotFolder + } + children, err := dir.GetChildren() + if err != nil { + return nil, err + } + ret := make([]model.Obj, 0, len(children)) + for _, child := range children { + ret = append(ret, toModelObj(child)) + } + return ret, nil +} + +func (ISO9660) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + img, err := getImage(ss[0]) + if err != nil { + return nil, 0, err + } + obj, err := getObj(img, args.InnerPath) + if err != nil { + return nil, 0, err + } + if obj.IsDir() { + return nil, 0, errs.NotFile + } + return io.NopCloser(obj.Reader()), obj.Size(), nil +} + +func (ISO9660) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + img, err := getImage(ss[0]) + if err != nil { + return err + } + obj, err := getObj(img, args.InnerPath) + if err != nil { + return err + } + if obj.IsDir() { + if args.InnerPath != "/" { + outputPath, err = tool.SecureJoin(outputPath, obj.Name()) + if err != nil { + return err + } + if err = os.MkdirAll(outputPath, 0700); err != nil { + return err + } + } + var children []*iso9660.File + if children, err = obj.GetChildren(); err == nil { + err = decompressAll(children, outputPath) + } + } else { + err = decompress(obj, outputPath, up) + } + return err +} + +var _ tool.Tool = (*ISO9660)(nil) + +func init() { + tool.RegisterTool(ISO9660{}) +} diff --git a/internal/archive/iso9660/utils.go b/internal/archive/iso9660/utils.go new file mode 100644 index 00000000000..dd2e00354f8 --- /dev/null +++ b/internal/archive/iso9660/utils.go @@ -0,0 +1,117 @@ +package iso9660 + +import ( + "io" + "os" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/kdomanski/iso9660" +) + +func getImage(ss *stream.SeekableStream) (*iso9660.Image, error) { + reader, err := stream.NewReadAtSeeker(ss, 0) + if err != nil { + return nil, err + } + return iso9660.OpenImage(reader) +} + +func getObj(img *iso9660.Image, path string) (*iso9660.File, error) { + obj, err := img.RootDir() + if err != nil { + return nil, err + } + if path == "/" { + return obj, nil + } + paths := strings.Split(strings.TrimPrefix(path, "/"), "/") + for _, p := range paths { + if !obj.IsDir() { + return nil, errs.ObjectNotFound + } + children, err := obj.GetChildren() + if err != nil { + return nil, err + } + exist := false + for _, child := range children { + if child.Name() == p { + obj = child + exist = true + break + } + } + if !exist { + return nil, errs.ObjectNotFound + } + } + return obj, nil +} + +func toModelObj(file *iso9660.File) model.Obj { + return &model.Object{ + Name: file.Name(), + Size: file.Size(), + Modified: file.ModTime(), + IsFolder: file.IsDir(), + } +} + +func decompress(f *iso9660.File, path string, up model.UpdateProgress) error { + return decompressEntry(f.Reader(), f.Size(), path, f.Name(), up) +} + +func decompressEntry(reader io.Reader, size int64, path, entryName string, up model.UpdateProgress) error { + dstPath, err := tool.SecureJoin(path, entryName) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + file, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer file.Close() + _, err = utils.CopyWithBuffer(file, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: reader, + Size: size, + }, + UpdateProgress: up, + }) + return err +} + +func decompressAll(children []*iso9660.File, path string) error { + for _, child := range children { + if child.IsDir() { + nextChildren, err := child.GetChildren() + if err != nil { + return err + } + nextPath, err := tool.SecureJoin(path, child.Name()) + if err != nil { + return err + } + if err = os.MkdirAll(nextPath, 0700); err != nil { + return err + } + if err = decompressAll(nextChildren, nextPath); err != nil { + return err + } + } else { + if err := decompress(child, path, func(_ float64) {}); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/archive/rardecode/rardecode.go b/internal/archive/rardecode/rardecode.go new file mode 100644 index 00000000000..2848c704bee --- /dev/null +++ b/internal/archive/rardecode/rardecode.go @@ -0,0 +1,181 @@ +package rardecode + +import ( + "fmt" + "io" + "os" + stdpath "path" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/nwaples/rardecode/v2" +) + +type RarDecoder struct{} + +func (RarDecoder) AcceptedExtensions() []string { + return []string{".rar"} +} + +func (RarDecoder) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{ + ".part1.rar": {".part%d.rar", 2}, + } +} + +func (RarDecoder) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + l, err := list(ss, args.Password) + if err != nil { + return nil, err + } + _, tree := tool.GenerateMetaTreeFromFolderTraversal(l) + return &model.ArchiveMetaInfo{ + Comment: "", + Encrypted: false, + Tree: tree, + }, nil +} + +func (RarDecoder) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +func (RarDecoder) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + reader, err := getReader(ss, args.Password) + if err != nil { + return nil, 0, err + } + innerPath := strings.TrimPrefix(args.InnerPath, "/") + for { + var header *rardecode.FileHeader + header, err = reader.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, 0, err + } + if header.Name == innerPath { + if header.IsDir { + break + } + return io.NopCloser(reader), header.UnPackedSize, nil + } + } + return nil, 0, errs.ObjectNotFound +} + +func (RarDecoder) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + reader, err := getReader(ss, args.Password) + if err != nil { + return err + } + if args.InnerPath == "/" { + for { + var header *rardecode.FileHeader + header, err = reader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + name := header.Name + if header.IsDir { + name = name + "/" + } + dstPath, e := tool.SecureJoin(outputPath, name) + if e != nil { + return e + } + err = decompress(reader, header, dstPath) + if err != nil { + return err + } + } + } else { + innerPath := strings.TrimPrefix(args.InnerPath, "/") + innerBase := stdpath.Base(innerPath) + createdBaseDir := false + var baseDirPath string + for { + var header *rardecode.FileHeader + header, err = reader.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + name := header.Name + if header.IsDir { + name = name + "/" + } + if name == innerPath { + if header.IsDir { + if !createdBaseDir { + baseDirPath, err = tool.SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + if err = os.MkdirAll(baseDirPath, 0700); err != nil { + return err + } + createdBaseDir = true + } + continue + } + if !header.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, header.Name) + } + dstPath, e := tool.SecureJoin(outputPath, stdpath.Base(innerPath)) + if e != nil { + return e + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(reader, header, dstPath, up) + if err != nil { + return err + } + break + } else if strings.HasPrefix(name, innerPath+"/") { + if !createdBaseDir { + baseDirPath, err = tool.SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + err = os.MkdirAll(baseDirPath, 0700) + if err != nil { + return err + } + createdBaseDir = true + } + restPath := strings.TrimPrefix(name, innerPath+"/") + if restPath == "" || restPath == "." { + continue + } + dstPath, e := tool.SecureJoin(baseDirPath, restPath) + if e != nil { + return e + } + err = decompress(reader, header, dstPath) + if err != nil { + return err + } + } + } + } + return nil +} + +var _ tool.Tool = (*RarDecoder)(nil) + +func init() { + tool.RegisterTool(RarDecoder{}) +} diff --git a/internal/archive/rardecode/utils.go b/internal/archive/rardecode/utils.go new file mode 100644 index 00000000000..e3612b363df --- /dev/null +++ b/internal/archive/rardecode/utils.go @@ -0,0 +1,221 @@ +package rardecode + +import ( + "fmt" + "io" + "io/fs" + "os" + stdpath "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/nwaples/rardecode/v2" +) + +type VolumeFile struct { + stream.SStreamReadAtSeeker + name string +} + +func (v *VolumeFile) Name() string { + return v.name +} + +func (v *VolumeFile) Size() int64 { + return v.SStreamReadAtSeeker.GetRawStream().GetSize() +} + +func (v *VolumeFile) Mode() fs.FileMode { + return 0644 +} + +func (v *VolumeFile) ModTime() time.Time { + return v.SStreamReadAtSeeker.GetRawStream().ModTime() +} + +func (v *VolumeFile) IsDir() bool { + return false +} + +func (v *VolumeFile) Sys() any { + return nil +} + +func (v *VolumeFile) Stat() (fs.FileInfo, error) { + return v, nil +} + +func (v *VolumeFile) Close() error { + return nil +} + +type VolumeFs struct { + parts map[string]*VolumeFile +} + +func (v *VolumeFs) Open(name string) (fs.File, error) { + file, ok := v.parts[name] + if !ok { + return nil, fs.ErrNotExist + } + return file, nil +} + +func makeOpts(ss []*stream.SeekableStream) (string, rardecode.Option, error) { + if len(ss) == 1 { + reader, err := stream.NewReadAtSeeker(ss[0], 0) + if err != nil { + return "", nil, err + } + fileName := "file.rar" + fsys := &VolumeFs{parts: map[string]*VolumeFile{ + fileName: {SStreamReadAtSeeker: reader, name: fileName}, + }} + return fileName, rardecode.FileSystem(fsys), nil + } else { + parts := make(map[string]*VolumeFile, len(ss)) + for i, s := range ss { + reader, err := stream.NewReadAtSeeker(s, 0) + if err != nil { + return "", nil, err + } + fileName := fmt.Sprintf("file.part%d.rar", i+1) + parts[fileName] = &VolumeFile{SStreamReadAtSeeker: reader, name: fileName} + } + return "file.part1.rar", rardecode.FileSystem(&VolumeFs{parts: parts}), nil + } +} + +type WrapReader struct { + files []*rardecode.File +} + +func (r *WrapReader) Files() []tool.SubFile { + ret := make([]tool.SubFile, 0, len(r.files)) + for _, f := range r.files { + ret = append(ret, &WrapFile{File: f}) + } + return ret +} + +type WrapFile struct { + *rardecode.File +} + +func (f *WrapFile) Name() string { + if f.File.IsDir { + return f.File.Name + "/" + } + return f.File.Name +} + +func (f *WrapFile) FileInfo() fs.FileInfo { + return &WrapFileInfo{File: f.File} +} + +type WrapFileInfo struct { + *rardecode.File +} + +func (f *WrapFileInfo) Name() string { + return stdpath.Base(f.File.Name) +} + +func (f *WrapFileInfo) Size() int64 { + return f.File.UnPackedSize +} + +func (f *WrapFileInfo) ModTime() time.Time { + return f.File.ModificationTime +} + +func (f *WrapFileInfo) IsDir() bool { + return f.File.IsDir +} + +func (f *WrapFileInfo) Sys() any { + return nil +} + +func list(ss []*stream.SeekableStream, password string) (*WrapReader, error) { + fileName, fsOpt, err := makeOpts(ss) + if err != nil { + return nil, err + } + opts := []rardecode.Option{fsOpt} + if password != "" { + opts = append(opts, rardecode.Password(password)) + } + files, err := rardecode.List(fileName, opts...) + // rardecode输出文件列表的顺序不一定是父目录在前,子目录在后 + // 父路径的长度一定比子路径短,排序后的files可保证父路径在前 + sort.Slice(files, func(i, j int) bool { + return len(files[i].Name) < len(files[j].Name) + }) + if err != nil { + return nil, filterPassword(err) + } + return &WrapReader{files: files}, nil +} + +func getReader(ss []*stream.SeekableStream, password string) (*rardecode.Reader, error) { + fileName, fsOpt, err := makeOpts(ss) + if err != nil { + return nil, err + } + opts := []rardecode.Option{fsOpt} + if password != "" { + opts = append(opts, rardecode.Password(password)) + } + rc, err := rardecode.OpenReader(fileName, opts...) + if err != nil { + return nil, filterPassword(err) + } + ss[0].Closers.Add(rc) + return &rc.Reader, nil +} + +func decompress(reader *rardecode.Reader, header *rardecode.FileHeader, dstPath string) error { + if header.IsDir { + return os.MkdirAll(dstPath, 0700) + } + if !header.Mode().IsRegular() { + return fmt.Errorf("%w: %s", tool.ErrArchiveIllegalPath, header.Name) + } + if err := os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + return _decompress(reader, header, dstPath, func(_ float64) {}) +} + +func _decompress(reader *rardecode.Reader, header *rardecode.FileHeader, dstPath string, up model.UpdateProgress) error { + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: reader, + Size: header.UnPackedSize, + }, + UpdateProgress: up, + }) + if err != nil { + return err + } + return nil +} + +func filterPassword(err error) error { + if err != nil && strings.Contains(err.Error(), "password") { + return errs.WrongArchivePassword + } + return err +} diff --git a/internal/archive/sevenzip/sevenzip.go b/internal/archive/sevenzip/sevenzip.go new file mode 100644 index 00000000000..281699664f8 --- /dev/null +++ b/internal/archive/sevenzip/sevenzip.go @@ -0,0 +1,72 @@ +package sevenzip + +import ( + "io" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" +) + +type SevenZip struct{} + +func (SevenZip) AcceptedExtensions() []string { + return []string{".7z"} +} + +func (SevenZip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{ + ".7z.001": {".7z.%.3d", 2}, + } +} + +func (SevenZip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + reader, err := getReader(ss, args.Password) + if err != nil { + return nil, err + } + _, tree := tool.GenerateMetaTreeFromFolderTraversal(&WrapReader{Reader: reader}) + return &model.ArchiveMetaInfo{ + Comment: "", + Encrypted: args.Password != "", + Tree: tree, + }, nil +} + +func (SevenZip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + return nil, errs.NotSupport +} + +func (SevenZip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + reader, err := getReader(ss, args.Password) + if err != nil { + return nil, 0, err + } + innerPath := strings.TrimPrefix(args.InnerPath, "/") + for _, file := range reader.File { + if file.Name == innerPath { + r, e := file.Open() + if e != nil { + return nil, 0, e + } + return r, file.FileInfo().Size(), nil + } + } + return nil, 0, errs.ObjectNotFound +} + +func (SevenZip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + reader, err := getReader(ss, args.Password) + if err != nil { + return err + } + return tool.DecompressFromFolderTraversal(&WrapReader{Reader: reader}, outputPath, args, up) +} + +var _ tool.Tool = (*SevenZip)(nil) + +func init() { + tool.RegisterTool(SevenZip{}) +} diff --git a/internal/archive/sevenzip/utils.go b/internal/archive/sevenzip/utils.go new file mode 100644 index 00000000000..624ba1879c8 --- /dev/null +++ b/internal/archive/sevenzip/utils.go @@ -0,0 +1,61 @@ +package sevenzip + +import ( + "errors" + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/bodgit/sevenzip" + "io" + "io/fs" +) + +type WrapReader struct { + Reader *sevenzip.Reader +} + +func (r *WrapReader) Files() []tool.SubFile { + ret := make([]tool.SubFile, 0, len(r.Reader.File)) + for _, f := range r.Reader.File { + ret = append(ret, &WrapFile{f: f}) + } + return ret +} + +type WrapFile struct { + f *sevenzip.File +} + +func (f *WrapFile) Name() string { + return f.f.Name +} + +func (f *WrapFile) FileInfo() fs.FileInfo { + return f.f.FileInfo() +} + +func (f *WrapFile) Open() (io.ReadCloser, error) { + return f.f.Open() +} + +func getReader(ss []*stream.SeekableStream, password string) (*sevenzip.Reader, error) { + readerAt, err := stream.NewMultiReaderAt(ss) + if err != nil { + return nil, err + } + sr, err := sevenzip.NewReaderWithPassword(readerAt, readerAt.Size(), password) + if err != nil { + return nil, filterPassword(err) + } + return sr, nil +} + +func filterPassword(err error) error { + if err != nil { + var e *sevenzip.ReadError + if errors.As(err, &e) && e.Encrypted { + return errs.WrongArchivePassword + } + } + return err +} diff --git a/internal/archive/tool/base.go b/internal/archive/tool/base.go new file mode 100644 index 00000000000..8f5b10d96b7 --- /dev/null +++ b/internal/archive/tool/base.go @@ -0,0 +1,21 @@ +package tool + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "io" +) + +type MultipartExtension struct { + PartFileFormat string + SecondPartIndex int +} + +type Tool interface { + AcceptedExtensions() []string + AcceptedMultipartExtensions() map[string]MultipartExtension + GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) + List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) + Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) + Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error +} diff --git a/internal/archive/tool/helper.go b/internal/archive/tool/helper.go new file mode 100644 index 00000000000..80254b8fd04 --- /dev/null +++ b/internal/archive/tool/helper.go @@ -0,0 +1,265 @@ +package tool + +import ( + "fmt" + "io" + "io/fs" + "os" + stdpath "path" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" +) + +type SubFile interface { + Name() string + FileInfo() fs.FileInfo + Open() (io.ReadCloser, error) +} + +type CanEncryptSubFile interface { + IsEncrypted() bool + SetPassword(password string) +} + +type ArchiveReader interface { + Files() []SubFile +} + +func GenerateMetaTreeFromFolderTraversal(r ArchiveReader) (bool, []model.ObjTree) { + encrypted := false + dirMap := make(map[string]*model.ObjectTree) + for _, file := range r.Files() { + if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { + encrypted = true + } + + name := strings.TrimPrefix(file.Name(), "/") + var dir string + var dirObj *model.ObjectTree + isNewFolder := false + if !file.FileInfo().IsDir() { + // 先将 文件 添加到 所在的文件夹 + dir = stdpath.Dir(name) + dirObj = dirMap[dir] + if dirObj == nil { + isNewFolder = dir != "." + dirObj = &model.ObjectTree{} + dirObj.IsFolder = true + dirObj.Name = stdpath.Base(dir) + dirObj.Modified = file.FileInfo().ModTime() + dirMap[dir] = dirObj + } + dirObj.Children = append( + dirObj.Children, &model.ObjectTree{ + Object: *MakeModelObj(file.FileInfo()), + }, + ) + } else { + dir = strings.TrimSuffix(name, "/") + dirObj = dirMap[dir] + if dirObj == nil { + isNewFolder = dir != "." + dirObj = &model.ObjectTree{} + dirMap[dir] = dirObj + } + dirObj.IsFolder = true + dirObj.Name = stdpath.Base(dir) + dirObj.Modified = file.FileInfo().ModTime() + } + if isNewFolder { + // 将 文件夹 添加到 父文件夹 + // 考虑压缩包仅记录文件的路径,不记录文件夹 + // 循环创建所有父文件夹 + parentDir := stdpath.Dir(dir) + for { + parentDirObj := dirMap[parentDir] + if parentDirObj == nil { + parentDirObj = &model.ObjectTree{} + if parentDir != "." { + parentDirObj.IsFolder = true + parentDirObj.Name = stdpath.Base(parentDir) + parentDirObj.Modified = file.FileInfo().ModTime() + } + dirMap[parentDir] = parentDirObj + } + parentDirObj.Children = append(parentDirObj.Children, dirObj) + + parentDir = stdpath.Dir(parentDir) + if dirMap[parentDir] != nil { + break + } + dirObj = parentDirObj + } + } + } + if len(dirMap) > 0 { + return encrypted, dirMap["."].GetChildren() + } else { + return encrypted, nil + } +} + +func MakeModelObj(file os.FileInfo) *model.Object { + return &model.Object{ + Name: file.Name(), + Size: file.Size(), + Modified: file.ModTime(), + IsFolder: file.IsDir(), + } +} + +type WrapFileInfo struct { + model.Obj +} + +func DecompressFromFolderTraversal(r ArchiveReader, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + var err error + files := r.Files() + if args.InnerPath == "/" { + for i, file := range files { + name := file.Name() + info := file.FileInfo() + if info.IsDir() { + var dirPath string + dirPath, err = SecureJoin(outputPath, name) + if err != nil { + return err + } + if err = os.MkdirAll(dirPath, 0700); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(outputPath, name) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, func(_ float64) {}) + if err != nil { + return err + } + up(float64(i+1) * 100.0 / float64(len(files))) + } + } else { + innerPath := strings.TrimPrefix(args.InnerPath, "/") + innerBase := stdpath.Base(innerPath) + createdBaseDir := false + var baseDirPath string + for _, file := range files { + name := file.Name() + if name == innerPath { + info := file.FileInfo() + if info.IsDir() { + if !createdBaseDir { + baseDirPath, err = SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + if err = os.MkdirAll(baseDirPath, 0700); err != nil { + return err + } + createdBaseDir = true + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(outputPath, stdpath.Base(innerPath)) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, up) + if err != nil { + return err + } + break + } else if strings.HasPrefix(name, innerPath+"/") { + if !createdBaseDir { + baseDirPath, err = SecureJoin(outputPath, innerBase) + if err != nil { + return err + } + err = os.MkdirAll(baseDirPath, 0700) + if err != nil { + return err + } + createdBaseDir = true + } + restPath := strings.TrimPrefix(name, innerPath+"/") + if restPath == "" || restPath == "." { + continue + } + info := file.FileInfo() + if info.IsDir() { + var dirPath string + dirPath, err = SecureJoin(baseDirPath, restPath) + if err != nil { + return err + } + if err = os.MkdirAll(dirPath, 0700); err != nil { + return err + } + continue + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%w: %s", ErrArchiveIllegalPath, name) + } + var dstPath string + dstPath, err = SecureJoin(baseDirPath, restPath) + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(dstPath), 0700); err != nil { + return err + } + err = _decompress(file, dstPath, args.Password, func(_ float64) {}) + if err != nil { + return err + } + } + } + } + return nil +} + +func _decompress(file SubFile, dstPath, password string, up model.UpdateProgress) error { + if encrypt, ok := file.(CanEncryptSubFile); ok && encrypt.IsEncrypted() { + encrypt.SetPassword(password) + } + rc, err := file.Open() + if err != nil { + return err + } + defer func() { _ = rc.Close() }() + f, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + _, err = io.Copy(f, &stream.ReaderUpdatingProgress{ + Reader: &stream.SimpleReaderWithSize{ + Reader: rc, + Size: file.FileInfo().Size(), + }, + UpdateProgress: up, + }) + if err != nil { + return err + } + return nil +} diff --git a/internal/archive/tool/securepath.go b/internal/archive/tool/securepath.go new file mode 100644 index 00000000000..f9bd89a914a --- /dev/null +++ b/internal/archive/tool/securepath.go @@ -0,0 +1,63 @@ +package tool + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" +) + +// ErrArchiveIllegalPath indicates an archive entry path is unsafe for extraction. +var ErrArchiveIllegalPath = errors.New("archive entry has illegal path") + +// SecureJoin returns a safe extraction path for an archive entry. +// It rejects absolute paths, traversal, Windows drive/UNC paths, and NUL bytes. +func SecureJoin(baseDir, entryName string) (string, error) { + if strings.Contains(entryName, "\x00") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + normalized := strings.ReplaceAll(entryName, "\\", "/") + if strings.HasPrefix(normalized, "//") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + cleaned := path.Clean(normalized) + + if cleaned == "." || cleaned == ".." || strings.HasPrefix(cleaned, "../") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + if strings.HasPrefix(cleaned, "/") { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + rel := filepath.FromSlash(cleaned) + if filepath.IsAbs(rel) || filepath.VolumeName(rel) != "" { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + if strings.HasPrefix(rel, `\\`) { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + + base := filepath.Clean(baseDir) + dst := filepath.Join(base, rel) + + baseAbs, err := filepath.Abs(base) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + dstAbs, err := filepath.Abs(dst) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + + relCheck, err := filepath.Rel(baseAbs, dstAbs) + if err != nil { + return "", fmt.Errorf("%w: %s (%v)", ErrArchiveIllegalPath, entryName, err) + } + if relCheck == ".." || strings.HasPrefix(relCheck, ".."+string(os.PathSeparator)) { + return "", fmt.Errorf("%w: %s", ErrArchiveIllegalPath, entryName) + } + return dst, nil +} diff --git a/internal/archive/tool/securepath_test.go b/internal/archive/tool/securepath_test.go new file mode 100644 index 00000000000..78be52ee5d5 --- /dev/null +++ b/internal/archive/tool/securepath_test.go @@ -0,0 +1,48 @@ +package tool + +import ( + "path/filepath" + "strings" + "testing" +) + +func TestSecureJoin(t *testing.T) { + baseDir := t.TempDir() + tests := []struct { + name string + entry string + wantErr bool + }{ + {name: "ok", entry: "a/b/c.txt", wantErr: false}, + {name: "parent", entry: "../evil.txt", wantErr: true}, + {name: "parent-backslash", entry: "..\\evil.txt", wantErr: true}, + {name: "abs", entry: "/tmp/evil.txt", wantErr: true}, + {name: "drive", entry: "C:\\evil.txt", wantErr: true}, + {name: "unc", entry: "\\\\server\\share\\evil.txt", wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dst, err := SecureJoin(baseDir, tc.entry) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %q, got nil", tc.entry) + } + if !strings.Contains(err.Error(), tc.entry) { + t.Fatalf("error should include entry name %q, got %q", tc.entry, err.Error()) + } + return + } + if err != nil { + t.Fatalf("unexpected error for %q: %v", tc.entry, err) + } + rel, err := filepath.Rel(baseDir, dst) + if err != nil { + t.Fatalf("Rel failed: %v", err) + } + if rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + t.Fatalf("path escaped baseDir: %q", dst) + } + }) + } +} diff --git a/internal/archive/tool/utils.go b/internal/archive/tool/utils.go new file mode 100644 index 00000000000..aa92cb1d792 --- /dev/null +++ b/internal/archive/tool/utils.go @@ -0,0 +1,32 @@ +package tool + +import ( + "github.com/alist-org/alist/v3/internal/errs" +) + +var ( + Tools = make(map[string]Tool) + MultipartExtensions = make(map[string]MultipartExtension) +) + +func RegisterTool(tool Tool) { + for _, ext := range tool.AcceptedExtensions() { + Tools[ext] = tool + } + for mainFile, ext := range tool.AcceptedMultipartExtensions() { + MultipartExtensions[mainFile] = ext + Tools[mainFile] = tool + } +} + +func GetArchiveTool(ext string) (*MultipartExtension, Tool, error) { + t, ok := Tools[ext] + if !ok { + return nil, nil, errs.UnknownArchiveFormat + } + partExt, ok := MultipartExtensions[ext] + if !ok { + return nil, t, nil + } + return &partExt, t, nil +} diff --git a/internal/archive/zip/utils.go b/internal/archive/zip/utils.go new file mode 100644 index 00000000000..59f4ed51378 --- /dev/null +++ b/internal/archive/zip/utils.go @@ -0,0 +1,195 @@ +package zip + +import ( + "bytes" + "io" + "io/fs" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/saintfish/chardet" + "github.com/yeka/zip" + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/charmap" + "golang.org/x/text/encoding/japanese" + "golang.org/x/text/encoding/korean" + "golang.org/x/text/encoding/simplifiedchinese" + "golang.org/x/text/encoding/traditionalchinese" + "golang.org/x/text/encoding/unicode" + "golang.org/x/text/encoding/unicode/utf32" + "golang.org/x/text/transform" +) + +type WrapReader struct { + Reader *zip.Reader +} + +func (r *WrapReader) Files() []tool.SubFile { + ret := make([]tool.SubFile, 0, len(r.Reader.File)) + for _, f := range r.Reader.File { + ret = append(ret, &WrapFile{f: f}) + } + return ret +} + +type WrapFileInfo struct { + fs.FileInfo +} + +func (f *WrapFileInfo) Name() string { + return decodeName(f.FileInfo.Name()) +} + +type WrapFile struct { + f *zip.File +} + +func (f *WrapFile) Name() string { + return decodeName(f.f.Name) +} + +func (f *WrapFile) FileInfo() fs.FileInfo { + return &WrapFileInfo{FileInfo: f.f.FileInfo()} +} + +func (f *WrapFile) Open() (io.ReadCloser, error) { + return f.f.Open() +} + +func (f *WrapFile) IsEncrypted() bool { + return f.f.IsEncrypted() +} + +func (f *WrapFile) SetPassword(password string) { + f.f.SetPassword(password) +} + +func getReader(ss []*stream.SeekableStream) (*zip.Reader, error) { + if len(ss) > 1 && stdpath.Ext(ss[1].GetName()) == ".z01" { + // FIXME: Incorrect parsing method for standard multipart zip format + ss = append(ss[1:], ss[0]) + } + reader, err := stream.NewMultiReaderAt(ss) + if err != nil { + return nil, err + } + return zip.NewReader(reader, reader.Size()) +} + +func filterPassword(err error) error { + if err != nil && strings.Contains(err.Error(), "password") { + return errs.WrongArchivePassword + } + return err +} + +func decodeName(name string) string { + b := []byte(name) + detector := chardet.NewTextDetector() + results, err := detector.DetectAll(b) + if err != nil { + return name + } + var ce, re, enc encoding.Encoding + for _, r := range results { + if r.Confidence > 30 { + ce = getCommonEncoding(r.Charset) + if ce != nil { + break + } + } + if re == nil { + re = getEncoding(r.Charset) + } + } + if ce != nil { + enc = ce + } else if re != nil { + enc = re + } else { + return name + } + i := bytes.NewReader(b) + decoder := transform.NewReader(i, enc.NewDecoder()) + content, _ := io.ReadAll(decoder) + return string(content) +} + +func getCommonEncoding(name string) (enc encoding.Encoding) { + switch name { + case "UTF-8": + enc = unicode.UTF8 + case "UTF-16LE": + enc = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) + case "Shift_JIS": + enc = japanese.ShiftJIS + case "GB-18030": + enc = simplifiedchinese.GB18030 + case "EUC-KR": + enc = korean.EUCKR + case "Big5": + enc = traditionalchinese.Big5 + default: + enc = nil + } + return +} + +func getEncoding(name string) (enc encoding.Encoding) { + switch name { + case "UTF-8": + enc = unicode.UTF8 + case "UTF-16BE": + enc = unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM) + case "UTF-16LE": + enc = unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM) + case "UTF-32BE": + enc = utf32.UTF32(utf32.BigEndian, utf32.IgnoreBOM) + case "UTF-32LE": + enc = utf32.UTF32(utf32.LittleEndian, utf32.IgnoreBOM) + case "ISO-8859-1": + enc = charmap.ISO8859_1 + case "ISO-8859-2": + enc = charmap.ISO8859_2 + case "ISO-8859-3": + enc = charmap.ISO8859_3 + case "ISO-8859-4": + enc = charmap.ISO8859_4 + case "ISO-8859-5": + enc = charmap.ISO8859_5 + case "ISO-8859-6": + enc = charmap.ISO8859_6 + case "ISO-8859-7": + enc = charmap.ISO8859_7 + case "ISO-8859-8": + enc = charmap.ISO8859_8 + case "ISO-8859-8-I": + enc = charmap.ISO8859_8I + case "ISO-8859-9": + enc = charmap.ISO8859_9 + case "windows-1251": + enc = charmap.Windows1251 + case "windows-1256": + enc = charmap.Windows1256 + case "KOI8-R": + enc = charmap.KOI8R + case "Shift_JIS": + enc = japanese.ShiftJIS + case "GB-18030": + enc = simplifiedchinese.GB18030 + case "EUC-JP": + enc = japanese.EUCJP + case "EUC-KR": + enc = korean.EUCKR + case "Big5": + enc = traditionalchinese.Big5 + case "ISO-2022-JP": + enc = japanese.ISO2022JP + default: + enc = nil + } + return +} diff --git a/internal/archive/zip/zip.go b/internal/archive/zip/zip.go new file mode 100644 index 00000000000..6e23570c73c --- /dev/null +++ b/internal/archive/zip/zip.go @@ -0,0 +1,132 @@ +package zip + +import ( + "io" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" +) + +type Zip struct { +} + +func (Zip) AcceptedExtensions() []string { + return []string{} +} + +func (Zip) AcceptedMultipartExtensions() map[string]tool.MultipartExtension { + return map[string]tool.MultipartExtension{ + ".zip": {".z%.2d", 1}, + ".zip.001": {".zip.%.3d", 2}, + } +} + +func (Zip) GetMeta(ss []*stream.SeekableStream, args model.ArchiveArgs) (model.ArchiveMeta, error) { + zipReader, err := getReader(ss) + if err != nil { + return nil, err + } + encrypted, tree := tool.GenerateMetaTreeFromFolderTraversal(&WrapReader{Reader: zipReader}) + return &model.ArchiveMetaInfo{ + Comment: zipReader.Comment, + Encrypted: encrypted, + Tree: tree, + }, nil +} + +func (Zip) List(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) ([]model.Obj, error) { + zipReader, err := getReader(ss) + if err != nil { + return nil, err + } + if args.InnerPath == "/" { + ret := make([]model.Obj, 0) + passVerified := false + var dir *model.Object + for _, file := range zipReader.File { + if !passVerified && file.IsEncrypted() { + file.SetPassword(args.Password) + rc, e := file.Open() + if e != nil { + return nil, filterPassword(e) + } + _ = rc.Close() + passVerified = true + } + name := strings.TrimSuffix(decodeName(file.Name), "/") + if strings.Contains(name, "/") { + // 有些压缩包不压缩第一个文件夹 + strs := strings.Split(name, "/") + if dir == nil && len(strs) == 2 { + dir = &model.Object{ + Name: strs[0], + Modified: ss[0].ModTime(), + IsFolder: true, + } + } + continue + } + ret = append(ret, tool.MakeModelObj(&WrapFileInfo{FileInfo: file.FileInfo()})) + } + if len(ret) == 0 && dir != nil { + ret = append(ret, dir) + } + return ret, nil + } else { + innerPath := strings.TrimPrefix(args.InnerPath, "/") + "/" + ret := make([]model.Obj, 0) + exist := false + for _, file := range zipReader.File { + name := decodeName(file.Name) + dir := stdpath.Dir(strings.TrimSuffix(name, "/")) + "/" + if dir != innerPath { + continue + } + exist = true + ret = append(ret, tool.MakeModelObj(&WrapFileInfo{file.FileInfo()})) + } + if !exist { + return nil, errs.ObjectNotFound + } + return ret, nil + } +} + +func (Zip) Extract(ss []*stream.SeekableStream, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + zipReader, err := getReader(ss) + if err != nil { + return nil, 0, err + } + innerPath := strings.TrimPrefix(args.InnerPath, "/") + for _, file := range zipReader.File { + if decodeName(file.Name) == innerPath { + if file.IsEncrypted() { + file.SetPassword(args.Password) + } + r, e := file.Open() + if e != nil { + return nil, 0, e + } + return r, file.FileInfo().Size(), nil + } + } + return nil, 0, errs.ObjectNotFound +} + +func (Zip) Decompress(ss []*stream.SeekableStream, outputPath string, args model.ArchiveInnerArgs, up model.UpdateProgress) error { + zipReader, err := getReader(ss) + if err != nil { + return err + } + return tool.DecompressFromFolderTraversal(&WrapReader{Reader: zipReader}, outputPath, args, up) +} + +var _ tool.Tool = (*Zip)(nil) + +func init() { + tool.RegisterTool(Zip{}) +} diff --git a/internal/aria2/add.go b/internal/aria2/add.go deleted file mode 100644 index 4eb83f3dd35..00000000000 --- a/internal/aria2/add.go +++ /dev/null @@ -1,61 +0,0 @@ -package aria2 - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/google/uuid" - "github.com/pkg/errors" -) - -func AddURI(ctx context.Context, uri string, dstDirPath string) error { - // check storage - storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - // check is it could upload - if storage.Config().NoUpload { - return errors.WithStack(errs.UploadNotSupported) - } - // check path is valid - obj, err := op.Get(ctx, storage, dstDirActualPath) - if err != nil { - if !errs.IsObjectNotFound(err) { - return errors.WithMessage(err, "failed get object") - } - } else { - if !obj.IsDir() { - // can't add to a file - return errors.WithStack(errs.NotFolder) - } - } - // call aria2 rpc - tempDir := filepath.Join(conf.Conf.TempDir, "aria2", uuid.NewString()) - options := map[string]interface{}{ - "dir": tempDir, - } - gid, err := client.AddURI([]string{uri}, options) - if err != nil { - return errors.Wrapf(err, "failed to add uri %s", uri) - } - DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ - ID: gid, - Name: fmt.Sprintf("download %s to [%s](%s)", uri, storage.GetStorage().MountPath, dstDirActualPath), - Func: func(tsk *task.Task[string]) error { - m := &Monitor{ - tsk: tsk, - tempDir: tempDir, - retried: 0, - dstDirPath: dstDirPath, - } - return m.Loop() - }, - })) - return nil -} diff --git a/internal/aria2/aria2.go b/internal/aria2/aria2.go deleted file mode 100644 index 7250afabc16..00000000000 --- a/internal/aria2/aria2.go +++ /dev/null @@ -1,42 +0,0 @@ -package aria2 - -import ( - "context" - "time" - - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/pkg/aria2/rpc" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -var DownTaskManager = task.NewTaskManager[string](3) -var notify = NewNotify() -var client rpc.Client - -func InitClient(timeout int) (string, error) { - client = nil - uri := setting.GetStr(conf.Aria2Uri) - secret := setting.GetStr(conf.Aria2Secret) - return InitAria2Client(uri, secret, timeout) -} - -func InitAria2Client(uri string, secret string, timeout int) (string, error) { - c, err := rpc.New(context.Background(), uri, secret, time.Duration(timeout)*time.Second, notify) - if err != nil { - return "", errors.Wrap(err, "failed to init aria2 client") - } - version, err := c.GetVersion() - if err != nil { - return "", errors.Wrapf(err, "failed get aria2 version") - } - client = c - log.Infof("using aria2 version: %s", version.Version) - return version.Version, nil -} - -func IsAria2Ready() bool { - return client != nil -} diff --git a/internal/aria2/aria2_test.go b/internal/aria2/aria2_test.go deleted file mode 100644 index 1e1b296b24a..00000000000 --- a/internal/aria2/aria2_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package aria2 - -import ( - "context" - "path/filepath" - "testing" - "time" - - _ "github.com/alist-org/alist/v3/drivers" - conf2 "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/db" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" - "gorm.io/driver/sqlite" - "gorm.io/gorm" -) - -func init() { - conf2.Conf = conf2.DefaultConfig() - absPath, err := filepath.Abs("../../data/temp") - if err != nil { - panic(err) - } - conf2.Conf.TempDir = absPath - dB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{}) - if err != nil { - panic("failed to connect database") - } - db.Init(dB) -} - -func TestConnect(t *testing.T) { - _, err := InitAria2Client("http://localhost:16800/jsonrpc", "secret", 3) - if err != nil { - t.Errorf("failed to init aria2: %+v", err) - } -} - -func TestDown(t *testing.T) { - TestConnect(t) - _, err := op.CreateStorage(context.Background(), model.Storage{ - ID: 0, - MountPath: "/", - Order: 0, - Driver: "Local", - Status: "", - Addition: `{"root_folder":"../../data"}`, - Remark: "", - }) - if err != nil { - t.Fatalf("failed to create storage: %+v", err) - } - err = AddURI(context.Background(), "https://nodejs.org/dist/index.json", "/test") - if err != nil { - t.Errorf("failed to add uri: %+v", err) - } - tasks := DownTaskManager.GetAll() - if len(tasks) != 1 { - t.Errorf("failed to get tasks: %+v", tasks) - } - for { - tsk := tasks[0] - t.Logf("task: %+v", tsk) - if tsk.GetState() == task.SUCCEEDED { - break - } - if tsk.GetState() == task.ERRORED { - t.Fatalf("failed to download: %+v", tsk) - } - time.Sleep(time.Second) - } - for { - if len(TransferTaskManager.GetAll()) == 0 { - continue - } - tsk := TransferTaskManager.GetAll()[0] - t.Logf("task: %+v", tsk) - if tsk.GetState() == task.SUCCEEDED { - break - } - if tsk.GetState() == task.ERRORED { - t.Fatalf("failed to download: %+v", tsk) - } - time.Sleep(time.Second) - } -} diff --git a/internal/aria2/monitor.go b/internal/aria2/monitor.go deleted file mode 100644 index 77265b372b1..00000000000 --- a/internal/aria2/monitor.go +++ /dev/null @@ -1,191 +0,0 @@ -package aria2 - -import ( - "fmt" - "github.com/alist-org/alist/v3/internal/stream" - "os" - "path" - "path/filepath" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -type Monitor struct { - tsk *task.Task[string] - tempDir string - retried int - c chan int - dstDirPath string - finish chan struct{} -} - -func (m *Monitor) Loop() error { - defer func() { - notify.Signals.Delete(m.tsk.ID) - // clear temp dir, should do while complete - //_ = os.RemoveAll(m.tempDir) - }() - m.c = make(chan int) - m.finish = make(chan struct{}) - notify.Signals.Store(m.tsk.ID, m.c) - var ( - err error - ok bool - ) -outer: - for { - select { - case <-m.tsk.Ctx.Done(): - _, err := client.Remove(m.tsk.ID) - return err - case <-m.c: - ok, err = m.Update() - if ok { - break outer - } - case <-time.After(time.Second * 2): - ok, err = m.Update() - if ok { - break outer - } - } - } - if err != nil { - return err - } - m.tsk.SetStatus("aria2 download completed, transferring") - <-m.finish - m.tsk.SetStatus("completed") - return nil -} - -func (m *Monitor) Update() (bool, error) { - info, err := client.TellStatus(m.tsk.ID) - if err != nil { - m.retried++ - log.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) - return false, nil - } - if m.retried > 5 { - return true, errors.Errorf("failed to get status of %s, retried %d times", m.tsk.ID, m.retried) - } - m.retried = 0 - if len(info.FollowedBy) != 0 { - log.Debugf("followen by: %+v", info.FollowedBy) - gid := info.FollowedBy[0] - notify.Signals.Delete(m.tsk.ID) - oldId := m.tsk.ID - m.tsk.ID = gid - DownTaskManager.RawTasks().Delete(oldId) - DownTaskManager.RawTasks().Store(m.tsk.ID, m.tsk) - notify.Signals.Store(gid, m.c) - return false, nil - } - // update download status - total, err := strconv.ParseUint(info.TotalLength, 10, 64) - if err != nil { - total = 0 - } - downloaded, err := strconv.ParseUint(info.CompletedLength, 10, 64) - if err != nil { - downloaded = 0 - } - progress := float64(downloaded) / float64(total) * 100 - m.tsk.SetProgress(int(progress)) - switch info.Status { - case "complete": - err := m.Complete() - return true, errors.WithMessage(err, "failed to transfer file") - case "error": - return true, errors.Errorf("failed to download %s, error: %s", m.tsk.ID, info.ErrorMessage) - case "active": - m.tsk.SetStatus("aria2: " + info.Status) - if info.Seeder == "true" { - err := m.Complete() - return true, errors.WithMessage(err, "failed to transfer file") - } - return false, nil - case "waiting", "paused": - m.tsk.SetStatus("aria2: " + info.Status) - return false, nil - case "removed": - return true, errors.Errorf("failed to download %s, removed", m.tsk.ID) - default: - return true, errors.Errorf("failed to download %s, unknown status %s", m.tsk.ID, info.Status) - } -} - -var TransferTaskManager = task.NewTaskManager(3, func(k *uint64) { - atomic.AddUint64(k, 1) -}) - -func (m *Monitor) Complete() error { - // check dstDir again - storage, dstDirActualPath, err := op.GetStorageAndActualPath(m.dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - // get files - files, err := client.GetFiles(m.tsk.ID) - log.Debugf("files len: %d", len(files)) - if err != nil { - return errors.Wrapf(err, "failed to get files of %s", m.tsk.ID) - } - // upload files - var wg sync.WaitGroup - wg.Add(len(files)) - go func() { - wg.Wait() - err := os.RemoveAll(m.tempDir) - m.finish <- struct{}{} - if err != nil { - log.Errorf("failed to remove aria2 temp dir: %+v", err.Error()) - } - }() - for i, _ := range files { - file := files[i] - TransferTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("transfer %s to [%s](%s)", file.Path, storage.GetStorage().MountPath, dstDirActualPath), - Func: func(tsk *task.Task[uint64]) error { - defer wg.Done() - size, _ := strconv.ParseInt(file.Length, 10, 64) - mimetype := utils.GetMimeType(file.Path) - f, err := os.Open(file.Path) - if err != nil { - return errors.Wrapf(err, "failed to open file %s", file.Path) - } - s := stream.FileStream{ - Obj: &model.Object{ - Name: path.Base(file.Path), - Size: size, - Modified: time.Now(), - IsFolder: false, - }, - Reader: f, - Closers: utils.NewClosers(f), - Mimetype: mimetype, - } - ss, err := stream.NewSeekableStream(s, nil) - if err != nil { - return err - } - relDir, err := filepath.Rel(m.tempDir, filepath.Dir(file.Path)) - if err != nil { - log.Errorf("find relation directory error: %v", err) - } - newDistDir := filepath.Join(dstDirActualPath, relDir) - return op.Put(tsk.Ctx, storage, newDistDir, ss, tsk.SetProgress) - }, - })) - } - return nil -} diff --git a/internal/authn/authn.go b/internal/authn/authn.go index df1d1fc6ff2..ea621d048af 100644 --- a/internal/authn/authn.go +++ b/internal/authn/authn.go @@ -1,6 +1,7 @@ package authn import ( + "fmt" "net/http" "net/url" @@ -19,7 +20,7 @@ func NewAuthnInstance(r *http.Request) (*webauthn.WebAuthn, error) { RPDisplayName: setting.GetStr(conf.SiteTitle), RPID: siteUrl.Hostname(), //RPOrigin: siteUrl.String(), - RPOrigins: []string{siteUrl.String()}, + RPOrigins: []string{fmt.Sprintf("%s://%s", siteUrl.Scheme, siteUrl.Host)}, // RPOrigin: "http://localhost:5173" }) } diff --git a/internal/bootstrap/aria2.go b/internal/bootstrap/aria2.go deleted file mode 100644 index 60017ddaa6f..00000000000 --- a/internal/bootstrap/aria2.go +++ /dev/null @@ -1,16 +0,0 @@ -package bootstrap - -import ( - "github.com/alist-org/alist/v3/internal/aria2" - "github.com/alist-org/alist/v3/pkg/utils" -) - -func InitAria2() { - go func() { - _, err := aria2.InitClient(2) - if err != nil { - //utils.Log.Errorf("failed to init aria2 client: %+v", err) - utils.Log.Infof("Aria2 not ready.") - } - }() -} diff --git a/internal/bootstrap/config.go b/internal/bootstrap/config.go index 2b7e9e13506..ac36059a076 100644 --- a/internal/bootstrap/config.go +++ b/internal/bootstrap/config.go @@ -9,6 +9,7 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/pkg/utils" "github.com/caarlos0/env/v9" log "github.com/sirupsen/logrus" @@ -34,6 +35,8 @@ func InitConfig() { log.Fatalf("failed to create config file: %+v", err) } conf.Conf = conf.DefaultConfig() + LastLaunchedVersion = conf.Version + conf.Conf.LastLaunchedVersion = conf.Version if !utils.WriteJsonToFile(configPath, conf.Conf) { log.Fatalf("failed to create default config file") } @@ -47,6 +50,10 @@ func InitConfig() { if err != nil { log.Fatalf("load config error: %+v", err) } + LastLaunchedVersion = conf.Conf.LastLaunchedVersion + if strings.HasPrefix(conf.Version, "v") || LastLaunchedVersion == "" { + conf.Conf.LastLaunchedVersion = conf.Version + } // update config.json struct confBody, err := utils.Json.MarshalIndent(conf.Conf, "", " ") if err != nil { @@ -57,9 +64,21 @@ func InitConfig() { log.Fatalf("update config struct error: %+v", err) } } + if conf.Conf.MaxConcurrency > 0 { + net.DefaultConcurrencyLimit = &net.ConcurrencyLimit{Limit: conf.Conf.MaxConcurrency} + } if !conf.Conf.Force { confFromEnv() } + if conf.Conf.TlsInsecureSkipVerify { + log.Warn("SECURITY WARNING / 安全警告:") + log.Warn("TLS certificate verification is disabled.") + log.Warn("TLS 证书校验已被禁用。") + log.Warn("This exposes all storage traffic to MitM attacks and may leak credentials or allow data tampering.") + log.Warn("这会使所有存储通信暴露于中间人攻击(MitM),可能导致凭据泄露和数据被篡改。") + log.Warn("Only use this setting if you fully understand the risks.") + log.Warn("仅在你完全理解风险的情况下使用该配置。") + } // convert abs path if !filepath.IsAbs(conf.Conf.TempDir) { absPath, err := filepath.Abs(conf.Conf.TempDir) @@ -68,11 +87,7 @@ func InitConfig() { } conf.Conf.TempDir = absPath } - err := os.RemoveAll(filepath.Join(conf.Conf.TempDir)) - if err != nil { - log.Errorln("failed delete temp file:", err) - } - err = os.MkdirAll(conf.Conf.TempDir, 0o777) + err := os.MkdirAll(conf.Conf.TempDir, 0o777) if err != nil { log.Fatalf("create temp dir error: %+v", err) } @@ -104,3 +119,15 @@ func initURL() { } conf.URL = u } + +func CleanTempDir() { + files, err := os.ReadDir(conf.Conf.TempDir) + if err != nil { + log.Errorln("failed list temp file: ", err) + } + for _, file := range files { + if err := os.RemoveAll(filepath.Join(conf.Conf.TempDir, file.Name())); err != nil { + log.Errorln("failed delete temp file: ", err) + } + } +} diff --git a/internal/bootstrap/data/data.go b/internal/bootstrap/data/data.go index 6c77ebf2385..1f0a5909a58 100644 --- a/internal/bootstrap/data/data.go +++ b/internal/bootstrap/data/data.go @@ -3,8 +3,10 @@ package data import "github.com/alist-org/alist/v3/cmd/flags" func InitData() { + initRoles() initUser() initSettings() + initTasks() if flags.Dev { initDevData() initDevDo() diff --git a/internal/bootstrap/data/dev.go b/internal/bootstrap/data/dev.go index f6296c9e96a..74097dbd8b7 100644 --- a/internal/bootstrap/data/dev.go +++ b/internal/bootstrap/data/dev.go @@ -26,7 +26,7 @@ func initDevData() { Username: "Noah", Password: "hsu", BasePath: "/data", - Role: 0, + Role: nil, Permission: 512, }) if err != nil { diff --git a/internal/bootstrap/data/role.go b/internal/bootstrap/data/role.go new file mode 100644 index 00000000000..a82fa2afcc3 --- /dev/null +++ b/internal/bootstrap/data/role.go @@ -0,0 +1,52 @@ +package data + +// initRoles creates the default admin and guest roles if missing. +// These roles are essential and must not be modified or removed. + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +func initRoles() { + guestRole, err := op.GetRoleByName("guest") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + guestRole = &model.Role{ + ID: uint(model.GUEST), + Name: "guest", + Description: "Guest", + PermissionScopes: []model.PermissionEntry{ + {Path: "/", Permission: 0}, + }, + } + if err := op.CreateRole(guestRole); err != nil { + utils.Log.Fatalf("[init role] Failed to create guest role: %v", err) + } + } else { + utils.Log.Fatalf("[init role] Failed to get guest role: %v", err) + } + } + + _, err = op.GetRoleByName("admin") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + adminRole := &model.Role{ + ID: uint(model.ADMIN), + Name: "admin", + Description: "Administrator", + PermissionScopes: []model.PermissionEntry{ + {Path: "/", Permission: 0xFFFF}, + }, + } + if err := op.CreateRole(adminRole); err != nil { + utils.Log.Fatalf("[init role] Failed to create admin role: %v", err) + } + } else { + utils.Log.Fatalf("[init role] Failed to get admin role: %v", err) + } + } +} diff --git a/internal/bootstrap/data/setting.go b/internal/bootstrap/data/setting.go index ca17b6b148f..71486335240 100644 --- a/internal/bootstrap/data/setting.go +++ b/internal/bootstrap/data/setting.go @@ -1,9 +1,13 @@ package data import ( + "strconv" + "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/pkg/utils/random" @@ -20,41 +24,53 @@ func initSettings() { if err != nil { utils.Log.Fatalf("failed get settings: %+v", err) } - - for i := range settings { - if !isActive(settings[i].Key) && settings[i].Flag != model.DEPRECATED { - settings[i].Flag = model.DEPRECATED - err = op.SaveSettingItem(&settings[i]) + settingMap := map[string]*model.SettingItem{} + for _, v := range settings { + if !isActive(v.Key) && v.Flag != model.DEPRECATED { + v.Flag = model.DEPRECATED + err = op.SaveSettingItem(&v) if err != nil { utils.Log.Fatalf("failed save setting: %+v", err) } } + settingMap[v.Key] = &v } - // create or save setting + save := false for i := range initialSettingItems { item := &initialSettingItems[i] + item.Index = uint(i) + if item.PreDefault == "" { + item.PreDefault = item.Value + } // err - stored, err := op.GetSettingItemByKey(item.Key) - if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { - utils.Log.Fatalf("failed get setting: %+v", err) - continue + stored, ok := settingMap[item.Key] + if !ok { + stored, err = op.GetSettingItemByKey(item.Key) + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + utils.Log.Fatalf("failed get setting: %+v", err) + continue + } } - // save - if stored != nil && item.Key != conf.VERSION { + if stored != nil && item.Key != conf.VERSION && stored.Value != item.PreDefault { item.Value = stored.Value } + _, err = op.HandleSettingItemHook(item) + if err != nil { + utils.Log.Errorf("failed to execute hook on %s: %+v", item.Key, err) + continue + } + // save if stored == nil || *item != *stored { - err = op.SaveSettingItem(item) - if err != nil { - utils.Log.Fatalf("failed save setting: %+v", err) - } + save = true + } + } + if save { + err = db.SaveSettingItems(initialSettingItems) + if err != nil { + utils.Log.Fatalf("failed save setting: %+v", err) } else { - // Not save so needs to execute hook - _, err = op.HandleSettingItemHook(item) - if err != nil { - utils.Log.Errorf("failed to execute hook on %s: %+v", item.Key, err) - } + op.SettingCacheUpdate() } } } @@ -75,6 +91,7 @@ func InitialSettings() []model.SettingItem { } else { token = random.Token() } + defaultRoleID := strconv.Itoa(model.GUEST) initialSettingItems = []model.SettingItem{ // site settings {Key: conf.VERSION, Value: conf.Version, Type: conf.TypeString, Group: model.SITE, Flag: model.READONLY}, @@ -83,10 +100,15 @@ func InitialSettings() []model.SettingItem { {Key: conf.SiteTitle, Value: "AList", Type: conf.TypeString, Group: model.SITE}, {Key: conf.Announcement, Value: "### repo\nhttps://github.com/alist-org/alist", Type: conf.TypeText, Group: model.SITE}, {Key: "pagination_type", Value: "all", Type: conf.TypeSelect, Options: "all,pagination,load_more,auto_load_more", Group: model.SITE}, - {Key: "default_page_size", Value: "30", Type: conf.TypeNumber, Group: model.SITE}, + {Key: "default_page_size", Value: "50", Type: conf.TypeNumber, Group: model.SITE}, {Key: conf.AllowIndexed, Value: "false", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.AllowMounted, Value: "true", Type: conf.TypeBool, Group: model.SITE}, {Key: conf.RobotsTxt, Value: "User-agent: *\nAllow: /", Type: conf.TypeText, Group: model.SITE}, + {Key: conf.AllowRegister, Value: "false", Type: conf.TypeBool, Group: model.SITE}, + {Key: conf.DefaultRole, Value: defaultRoleID, Type: conf.TypeSelect, Group: model.SITE}, + // newui settings + {Key: conf.UseNewui, Value: "false", Type: conf.TypeBool, Group: model.SITE}, + {Key: conf.FrontendRememberSort, Value: "false", Type: conf.TypeBool, Group: model.SITE, Help: "Persist frontend list sorting in the browser. When disabled, backend/driver order is used until the user sorts manually in the current session."}, // style settings {Key: conf.Logo, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeText, Group: model.STYLE}, {Key: conf.Favicon, Value: "https://cdn.jsdelivr.net/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.STYLE}, @@ -97,10 +119,10 @@ func InitialSettings() []model.SettingItem { // preview settings {Key: conf.TextTypes, Value: "txt,htm,html,xml,java,properties,sql,js,md,json,conf,ini,vue,php,py,bat,gitignore,yml,go,sh,c,cpp,h,hpp,tsx,vtt,srt,ass,rs,lrc", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.AudioTypes, Value: "mp3,flac,ogg,m4a,wav,opus,wma", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, - {Key: conf.VideoTypes, Value: "mp4,mkv,avi,mov,rmvb,webm,flv", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, + {Key: conf.VideoTypes, Value: "mp4,mkv,avi,mov,rmvb,webm,flv,m3u8", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ImageTypes, Value: "jpg,tiff,jpeg,png,gif,bmp,svg,ico,swf,webp", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, //{Key: conf.OfficeTypes, Value: "doc,docx,xls,xlsx,ppt,pptx", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, - {Key: conf.ProxyTypes, Value: "m3u8", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, + {Key: conf.ProxyTypes, Value: "m3u8,url", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: conf.ProxyIgnoreHeaders, Value: "authorization,referer", Type: conf.TypeText, Group: model.PREVIEW, Flag: model.PRIVATE}, {Key: "external_previews", Value: `{}`, Type: conf.TypeText, Group: model.PREVIEW}, {Key: "iframe_previews", Value: `{ @@ -115,6 +137,7 @@ func InitialSettings() []model.SettingItem { "EPUB.js":"https://alist-org.github.io/static/epub.js/viewer.html?url=$e_url" } }`, Type: conf.TypeText, Group: model.PREVIEW}, + {Key: "preview_settings", Value: `{}`, Type: conf.TypeText, Group: model.PREVIEW}, // {Key: conf.OfficeViewers, Value: `{ // "Microsoft":"https://view.officeapps.live.com/op/view.aspx?src=$url", // "Google":"https://docs.google.com/gview?url=$url&embedded=true", @@ -125,29 +148,34 @@ func InitialSettings() []model.SettingItem { {Key: "audio_cover", Value: "https://jsd.nn.ci/gh/alist-org/logo@main/logo.svg", Type: conf.TypeString, Group: model.PREVIEW}, {Key: conf.AudioAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, {Key: conf.VideoAutoplay, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.ThumbnailSize, Value: "144", Type: conf.TypeNumber, Group: model.PREVIEW, Help: "Thumbnail width in pixels. Height is scaled proportionally."}, + {Key: conf.PreviewArchivesByDefault, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.ReadMeAutoRender, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, + {Key: conf.FilterReadMeScripts, Value: "true", Type: conf.TypeBool, Group: model.PREVIEW}, // global settings {Key: conf.HideFiles, Value: "/\\/README.md/i", Type: conf.TypeText, Group: model.GLOBAL}, {Key: "package_download", Value: "true", Type: conf.TypeBool, Group: model.GLOBAL}, - {Key: conf.CustomizeHead, Value: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.CustomizeHead, PreDefault: ``, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.CustomizeBody, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.LinkExpiration, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE}, - {Key: conf.SignAll, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, + {Key: conf.SignAll, Value: "true", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE}, {Key: conf.PrivacyRegs, Value: `(?:(?:\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5])\.){3}(?:\d|[1-9]\d|1\d\d|2[0-4]\d|25[0-5]) ([[:xdigit:]]{1,4}(?::[[:xdigit:]]{1,4}){7}|::|:(?::[[:xdigit:]]{1,4}){1,6}|[[:xdigit:]]{1,4}:(?::[[:xdigit:]]{1,4}){1,5}|(?:[[:xdigit:]]{1,4}:){2}(?::[[:xdigit:]]{1,4}){1,4}|(?:[[:xdigit:]]{1,4}:){3}(?::[[:xdigit:]]{1,4}){1,3}|(?:[[:xdigit:]]{1,4}:){4}(?::[[:xdigit:]]{1,4}){1,2}|(?:[[:xdigit:]]{1,4}:){5}:[[:xdigit:]]{1,4}|(?:[[:xdigit:]]{1,4}:){1,6}:) (?U)access_token=(.*)&`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE}, - {Key: conf.OcrApi, Value: "https://api.nn.ci/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, + {Key: conf.OcrApi, Value: "https://api.alistgo.com/ocr/file/json", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.FilenameCharMapping, Value: `{"/": "|"}`, Type: conf.TypeText, Group: model.GLOBAL}, {Key: conf.ForwardDirectLinkParams, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL}, + {Key: conf.IgnoreDirectLinkParams, Value: "sign,alist_ts", Type: conf.TypeString, Group: model.GLOBAL}, {Key: conf.WebauthnLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC}, - - // aria2 settings - {Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - {Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, + {Key: conf.MaxDevices, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL}, + {Key: conf.DeviceEvictPolicy, Value: "deny", Type: conf.TypeSelect, Options: "deny,evict_oldest", Group: model.GLOBAL}, + {Key: conf.DeviceSessionTTL, Value: "86400", Type: conf.TypeNumber, Group: model.GLOBAL}, + {Key: conf.MetaNotFoundCacheExpire, Value: "60", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE, Help: "Negative cache expiration for missing meta records, in seconds. Set 0 to disable."}, // single settings {Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,none", Group: model.INDEX}, + {Key: conf.SearchIndex, Value: "none", Type: conf.TypeSelect, Options: "database,database_non_full_text,bleve,meilisearch,none", Group: model.INDEX}, {Key: conf.AutoUpdateIndex, Value: "false", Type: conf.TypeBool, Group: model.INDEX}, {Key: conf.IgnorePaths, Value: "", Type: conf.TypeText, Group: model.INDEX, Flag: model.PRIVATE, Help: `one path per line`}, {Key: conf.MaxIndexDepth, Value: "20", Type: conf.TypeNumber, Group: model.INDEX, Flag: model.PRIVATE, Help: `max depth of index`}, @@ -163,15 +191,66 @@ func InitialSettings() []model.SettingItem { {Key: conf.SSOApplicationName, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOEndpointName, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOJwtPublicKey, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, + {Key: conf.SSOExtraScopes, Value: "", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOAutoRegister, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultDir, Value: "/", Type: conf.TypeString, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSODefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.SSO, Flag: model.PRIVATE}, {Key: conf.SSOCompatibilityMode, Value: "false", Type: conf.TypeBool, Group: model.SSO, Flag: model.PUBLIC}, - // qbittorrent settings - {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, + // ldap settings + {Key: conf.LdapLoginEnabled, Value: "false", Type: conf.TypeBool, Group: model.LDAP, Flag: model.PUBLIC}, + {Key: conf.LdapServer, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapManagerDN, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapManagerPassword, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapUserSearchBase, Value: "", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapUserSearchFilter, Value: "(uid=%s)", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapDefaultDir, Value: "/", Type: conf.TypeString, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapDefaultPermission, Value: "0", Type: conf.TypeNumber, Group: model.LDAP, Flag: model.PRIVATE}, + {Key: conf.LdapLoginTips, Value: "login with ldap", Type: conf.TypeString, Group: model.LDAP, Flag: model.PUBLIC}, + + // s3 settings + {Key: conf.S3AccessKeyId, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, + {Key: conf.S3SecretAccessKey, Value: "", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, + {Key: conf.S3Buckets, Value: "[]", Type: conf.TypeString, Group: model.S3, Flag: model.PRIVATE}, + + // ftp settings + {Key: conf.FTPPublicHost, Value: "127.0.0.1", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPPasvPortMap, Value: "", Type: conf.TypeText, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPProxyUserAgent, Value: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + + "Chrome/87.0.4280.88 Safari/537.36", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPMandatoryTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPImplicitTLS, Value: "false", Type: conf.TypeBool, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPTLSPrivateKeyPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + {Key: conf.FTPTLSPublicCertPath, Value: "", Type: conf.TypeString, Group: model.FTP, Flag: model.PRIVATE}, + + // frp settings + {Key: conf.FRPEnabled, Value: "false", Type: conf.TypeBool, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPServerAddr, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPServerPort, Value: "7000", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPAuthToken, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPProxyName, Value: "alist", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPProxyType, Value: "http", Type: conf.TypeSelect, Options: "http,https,tcp,stcp", Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPCustomDomain, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPSubdomain, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPRemotePort, Value: "0", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPLocalPort, Value: "5244", Type: conf.TypeNumber, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPTLSEnable, Value: "false", Type: conf.TypeBool, Group: model.FRP, Flag: model.PRIVATE}, + {Key: conf.FRPSTCPSecretKey, Value: "", Type: conf.TypeString, Group: model.FRP, Flag: model.PRIVATE, Help: "Required for stcp proxy type"}, + {Key: conf.FRPStatus, Value: "stopped", Type: conf.TypeString, Group: model.FRP, Flag: model.READONLY}, + + // traffic settings + {Key: conf.TaskOfflineDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Download.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskOfflineDownloadTransferThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Transfer.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Upload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskCopyThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Copy.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskDecompressDownloadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.Decompress.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.TaskDecompressUploadThreadsNum, Value: strconv.Itoa(conf.Conf.Tasks.DecompressUpload.Workers), Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.StreamMaxClientDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.StreamMaxClientUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.StreamMaxServerDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, + {Key: conf.StreamMaxServerUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE}, } + initialSettingItems = append(initialSettingItems, tool.Tools.Items()...) if flags.Dev { initialSettingItems = append(initialSettingItems, []model.SettingItem{ {Key: "test_deprecated", Value: "test_value", Type: conf.TypeString, Flag: model.DEPRECATED}, diff --git a/internal/bootstrap/data/task.go b/internal/bootstrap/data/task.go new file mode 100644 index 00000000000..7100e2e25c1 --- /dev/null +++ b/internal/bootstrap/data/task.go @@ -0,0 +1,29 @@ +package data + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" +) + +var initialTaskItems []model.TaskItem + +func initTasks() { + InitialTasks() + + for i := range initialTaskItems { + item := &initialTaskItems[i] + taskitem, _ := db.GetTaskDataByType(item.Key) + if taskitem == nil { + db.CreateTaskData(item) + } + } +} + +func InitialTasks() []model.TaskItem { + initialTaskItems = []model.TaskItem{ + {Key: "copy", PersistData: "[]"}, + {Key: "download", PersistData: "[]"}, + {Key: "transfer", PersistData: "[]"}, + } + return initialTaskItems +} diff --git a/internal/bootstrap/data/user.go b/internal/bootstrap/data/user.go index 9ac62fe841f..6851118668b 100644 --- a/internal/bootstrap/data/user.go +++ b/internal/bootstrap/data/user.go @@ -1,10 +1,10 @@ package data import ( + "github.com/alist-org/alist/v3/internal/db" "os" "github.com/alist-org/alist/v3/cmd/flags" - "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" @@ -14,6 +14,28 @@ import ( ) func initUser() { + guest, err := op.GetGuest() + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + salt := random.String(16) + guestRole, _ := op.GetRoleByName("guest") + guest = &model.User{ + Username: "guest", + PwdHash: model.TwoHashPwd("guest", salt), + Salt: salt, + Role: model.Roles{int(guestRole.ID)}, + BasePath: "/", + Permission: 0, + Disabled: true, + Authn: "[]", + } + if err := db.CreateUser(guest); err != nil { + utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) + } + } else { + utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) + } + } admin, err := op.GetAdmin() adminPassword := random.String(8) envpass := os.Getenv("ALIST_ADMIN_PASSWORD") @@ -25,12 +47,16 @@ func initUser() { if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { salt := random.String(16) + adminRole, _ := op.GetRoleByName("admin") admin = &model.User{ Username: "admin", Salt: salt, PwdHash: model.TwoHashPwd(adminPassword, salt), - Role: model.ADMIN, + Role: model.Roles{int(adminRole.ID)}, BasePath: "/", + Authn: "[]", + // 0(can see hidden) - 7(can remove) & 12(can read archives) - 13(can decompress archives) + Permission: 0xFFFF, } if err := op.CreateUser(admin); err != nil { panic(err) @@ -41,41 +67,4 @@ func initUser() { utils.Log.Fatalf("[init user] Failed to get admin user: %v", err) } } - guest, err := op.GetGuest() - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - salt := random.String(16) - guest = &model.User{ - Username: "guest", - PwdHash: model.TwoHashPwd("guest", salt), - Role: model.GUEST, - BasePath: "/", - Permission: 0, - Disabled: true, - } - if err := db.CreateUser(guest); err != nil { - utils.Log.Fatalf("[init user] Failed to create guest user: %v", err) - } - } else { - utils.Log.Fatalf("[init user] Failed to get guest user: %v", err) - } - } - hashPwdForOldVersion() -} - -func hashPwdForOldVersion() { - users, _, err := op.GetUsers(1, -1) - if err != nil { - utils.Log.Fatalf("[hash pwd for old version] failed get users: %v", err) - } - for i := range users { - user := users[i] - if user.PwdHash == "" { - user.SetPassword(user.Password) - user.Password = "" - if err := db.UpdateUser(&user); err != nil { - utils.Log.Fatalf("[hash pwd for old version] failed update user: %v", err) - } - } - } } diff --git a/internal/bootstrap/db.go b/internal/bootstrap/db.go index 4c4044f19e3..5f5f6fcef3e 100644 --- a/internal/bootstrap/db.go +++ b/internal/bootstrap/db.go @@ -56,14 +56,26 @@ func InitDB() { } case "mysql": { - dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local&tls=%s", - database.User, database.Password, database.Host, database.Port, database.Name, database.SSLMode) + dsn := database.DSN + if dsn == "" { + //[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN] + dsn = fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local&tls=%s", + database.User, database.Password, database.Host, database.Port, database.Name, database.SSLMode) + } dB, err = gorm.Open(mysql.Open(dsn), gormConfig) } case "postgres": { - dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", - database.Host, database.User, database.Password, database.Name, database.Port, database.SSLMode) + dsn := database.DSN + if dsn == "" { + if database.Password != "" { + dsn = fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", + database.Host, database.User, database.Password, database.Name, database.Port, database.SSLMode) + } else { + dsn = fmt.Sprintf("host=%s user=%s dbname=%s port=%d sslmode=%s TimeZone=Asia/Shanghai", + database.Host, database.User, database.Name, database.Port, database.SSLMode) + } + } dB, err = gorm.Open(postgres.Open(dsn), gormConfig) } default: diff --git a/internal/bootstrap/frp.go b/internal/bootstrap/frp.go new file mode 100644 index 00000000000..b8417843c30 --- /dev/null +++ b/internal/bootstrap/frp.go @@ -0,0 +1,19 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func InitFRP() { + frp.Instance = frp.Init() + if setting.GetBool(conf.FRPEnabled) { + if err := frp.Instance.Start(); err != nil { + utils.Log.Warnf("failed to start frp client: %v", err) + } else { + utils.Log.Info("frp client started") + } + } +} diff --git a/internal/bootstrap/log.go b/internal/bootstrap/log.go index 00411e5e189..b4f4af08f1b 100644 --- a/internal/bootstrap/log.go +++ b/internal/bootstrap/log.go @@ -14,10 +14,14 @@ import ( func init() { formatter := logrus.TextFormatter{ - ForceColors: true, - EnvironmentOverrideColors: true, - TimestampFormat: "2006-01-02 15:04:05", - FullTimestamp: true, + TimestampFormat: "2006-01-02 15:04:05", + FullTimestamp: true, + } + if os.Getenv("NO_COLOR") != "" || os.Getenv("ALIST_NO_COLOR") == "1" { + formatter.DisableColors = true + } else { + formatter.ForceColors = true + formatter.EnvironmentOverrideColors = true } logrus.SetFormatter(&formatter) utils.Log.SetFormatter(&formatter) diff --git a/internal/bootstrap/offline_download.go b/internal/bootstrap/offline_download.go new file mode 100644 index 00000000000..26e04071b10 --- /dev/null +++ b/internal/bootstrap/offline_download.go @@ -0,0 +1,17 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/pkg/utils" +) + +func InitOfflineDownloadTools() { + for k, v := range tool.Tools { + res, err := v.Init() + if err != nil { + utils.Log.Warnf("init tool %s failed: %s", k, err) + } else { + utils.Log.Infof("init tool %s success: %s", k, res) + } + } +} diff --git a/internal/bootstrap/patch.go b/internal/bootstrap/patch.go new file mode 100644 index 00000000000..5c7ca7583b6 --- /dev/null +++ b/internal/bootstrap/patch.go @@ -0,0 +1,74 @@ +package bootstrap + +import ( + "fmt" + + "github.com/alist-org/alist/v3/internal/bootstrap/patch" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/pkg/utils" + "strings" +) + +var LastLaunchedVersion = "" + +func safeCall(v string, i int, f func()) { + defer func() { + if r := recover(); r != nil { + utils.Log.Errorf("Recovered from patch (version: %s, index: %d) panic: %v", v, i, r) + } + }() + + f() +} + +func getVersion(v string) (major, minor, patchNum int, err error) { + _, err = fmt.Sscanf(v, "v%d.%d.%d", &major, &minor, &patchNum) + return major, minor, patchNum, err +} + +func compareVersion(majorA, minorA, patchNumA, majorB, minorB, patchNumB int) bool { + if majorA != majorB { + return majorA > majorB + } + if minorA != minorB { + return minorA > minorB + } + if patchNumA != patchNumB { + return patchNumA > patchNumB + } + return true +} + +func InitUpgradePatch() { + if !strings.HasPrefix(conf.Version, "v") { + for _, vp := range patch.UpgradePatches { + for i, p := range vp.Patches { + safeCall(vp.Version, i, p) + } + } + return + } + if LastLaunchedVersion == conf.Version { + return + } + if LastLaunchedVersion == "" { + LastLaunchedVersion = "v0.0.0" + } + major, minor, patchNum, err := getVersion(LastLaunchedVersion) + if err != nil { + utils.Log.Warnf("Failed to parse last launched version %s: %v, skipping all patches and rewrite last launched version", LastLaunchedVersion, err) + return + } + for _, vp := range patch.UpgradePatches { + ma, mi, pn, err := getVersion(vp.Version) + if err != nil { + utils.Log.Errorf("Skip invalid version %s patches: %v", vp.Version, err) + continue + } + if compareVersion(ma, mi, pn, major, minor, patchNum) { + for i, p := range vp.Patches { + safeCall(vp.Version, i, p) + } + } + } +} diff --git a/internal/bootstrap/patch/all.go b/internal/bootstrap/patch/all.go new file mode 100644 index 00000000000..eb679147ec9 --- /dev/null +++ b/internal/bootstrap/patch/all.go @@ -0,0 +1,42 @@ +package patch + +import ( + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_24_0" + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_32_0" + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_41_0" + "github.com/alist-org/alist/v3/internal/bootstrap/patch/v3_46_0" +) + +type VersionPatches struct { + // Version means if the system is upgraded from Version or an earlier one + // to the current version, all patches in Patches will be executed. + Version string + Patches []func() +} + +var UpgradePatches = []VersionPatches{ + { + Version: "v3.24.0", + Patches: []func(){ + v3_24_0.HashPwdForOldVersion, + }, + }, + { + Version: "v3.32.0", + Patches: []func(){ + v3_32_0.UpdateAuthnForOldVersion, + }, + }, + { + Version: "v3.41.0", + Patches: []func(){ + v3_41_0.GrantAdminPermissions, + }, + }, + { + Version: "v3.46.0", + Patches: []func(){ + v3_46_0.ConvertLegacyRoles, + }, + }, +} diff --git a/internal/bootstrap/patch/v3_24_0/hash_password.go b/internal/bootstrap/patch/v3_24_0/hash_password.go new file mode 100644 index 00000000000..2adb640d8c9 --- /dev/null +++ b/internal/bootstrap/patch/v3_24_0/hash_password.go @@ -0,0 +1,26 @@ +package v3_24_0 + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// HashPwdForOldVersion encode passwords using SHA256 +// First published: 75acbcc perf: sha256 for user's password (close #3552) by Andy Hsu +func HashPwdForOldVersion() { + users, _, err := op.GetUsers(1, -1) + if err != nil { + utils.Log.Fatalf("[hash pwd for old version] failed get users: %v", err) + } + for i := range users { + user := users[i] + if user.PwdHash == "" { + user.SetPassword(user.Password) + user.Password = "" + if err := db.UpdateUser(&user); err != nil { + utils.Log.Fatalf("[hash pwd for old version] failed update user: %v", err) + } + } + } +} diff --git a/internal/bootstrap/patch/v3_32_0/update_authn.go b/internal/bootstrap/patch/v3_32_0/update_authn.go new file mode 100644 index 00000000000..92a594fda78 --- /dev/null +++ b/internal/bootstrap/patch/v3_32_0/update_authn.go @@ -0,0 +1,25 @@ +package v3_32_0 + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// UpdateAuthnForOldVersion updates users' authn +// First published: bdfc159 fix: webauthn logspam (#6181) by itsHenry +func UpdateAuthnForOldVersion() { + users, _, err := op.GetUsers(1, -1) + if err != nil { + utils.Log.Fatalf("[update authn for old version] failed get users: %v", err) + } + for i := range users { + user := users[i] + if user.Authn == "" { + user.Authn = "[]" + if err := db.UpdateUser(&user); err != nil { + utils.Log.Fatalf("[update authn for old version] failed update user: %v", err) + } + } + } +} diff --git a/internal/bootstrap/patch/v3_41_0/grant_permission.go b/internal/bootstrap/patch/v3_41_0/grant_permission.go new file mode 100644 index 00000000000..60d8ab4fa3b --- /dev/null +++ b/internal/bootstrap/patch/v3_41_0/grant_permission.go @@ -0,0 +1,21 @@ +package v3_41_0 + +import ( + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// GrantAdminPermissions gives admin Permission 0(can see hidden) - 9(webdav manage) and +// 12(can read archives) - 13(can decompress archives) +// This patch is written to help users upgrading from older version better adapt to PR AlistGo/alist#7705 and +// PR AlistGo/alist#7817. +func GrantAdminPermissions() { + admin, err := op.GetAdmin() + if err == nil && (admin.Permission & 0x33FF) == 0 { + admin.Permission |= 0x33FF + err = op.UpdateUser(admin) + } + if err != nil { + utils.Log.Errorf("Cannot grant permissions to admin: %v", err) + } +} diff --git a/internal/bootstrap/patch/v3_46_0/convert_role.go b/internal/bootstrap/patch/v3_46_0/convert_role.go new file mode 100644 index 00000000000..3aac95b691c --- /dev/null +++ b/internal/bootstrap/patch/v3_46_0/convert_role.go @@ -0,0 +1,186 @@ +package v3_46_0 + +import ( + "database/sql" + "encoding/json" + "errors" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "gorm.io/gorm" +) + +// ConvertLegacyRoles migrates old integer role values to a new role model with permission scopes. +func ConvertLegacyRoles() { + guestRole, err := op.GetRoleByName("guest") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + guestRole = &model.Role{ + ID: uint(model.GUEST), + Name: "guest", + Description: "Guest", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0, + }, + }, + } + if err = op.CreateRole(guestRole); err != nil { + utils.Log.Errorf("[convert roles] failed to create guest role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed to get guest role: %v", err) + return + } + } + + adminRole, err := op.GetRoleByName("admin") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + adminRole = &model.Role{ + ID: uint(model.ADMIN), + Name: "admin", + Description: "Administrator", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0x33FF, + }, + }, + } + if err = op.CreateRole(adminRole); err != nil { + utils.Log.Errorf("[convert roles] failed to create admin role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed to get admin role: %v", err) + return + } + } + + generalRole, err := op.GetRoleByName("general") + if err != nil { + if errors.Is(err, gorm.ErrRecordNotFound) { + generalRole = &model.Role{ + ID: uint(model.NEWGENERAL), + Name: "general", + Description: "General User", + PermissionScopes: []model.PermissionEntry{ + { + Path: "/", + Permission: 0, + }, + }, + } + if err = op.CreateRole(generalRole); err != nil { + utils.Log.Errorf("[convert roles] failed create general role: %v", err) + return + } + } else { + utils.Log.Errorf("[convert roles] failed get general role: %v", err) + return + } + } + + rawDb := db.GetDb() + table := conf.Conf.Database.TablePrefix + "users" + rows, err := rawDb.Table(table).Select("id, username, role").Rows() + if err != nil { + utils.Log.Errorf("[convert roles] failed to get users: %v", err) + return + } + defer rows.Close() + + var updatedCount int + for rows.Next() { + var id uint + var username string + var rawRole []byte + + if err := rows.Scan(&id, &username, &rawRole); err != nil { + utils.Log.Warnf("[convert roles] skip user scan err: %v", err) + continue + } + + utils.Log.Debugf("[convert roles] user: %s raw role: %s", username, string(rawRole)) + + if len(rawRole) == 0 { + continue + } + + var oldRoles []int + wasSingleInt := false + if err := json.Unmarshal(rawRole, &oldRoles); err != nil { + var single int + if err := json.Unmarshal(rawRole, &single); err != nil { + utils.Log.Warnf("[convert roles] user %s has invalid role: %s", username, string(rawRole)) + continue + } + oldRoles = []int{single} + wasSingleInt = true + } + + var newRoles model.Roles + for _, r := range oldRoles { + switch r { + case model.ADMIN: + newRoles = append(newRoles, int(adminRole.ID)) + case model.GUEST: + newRoles = append(newRoles, int(guestRole.ID)) + case model.GENERAL: + newRoles = append(newRoles, int(generalRole.ID)) + default: + newRoles = append(newRoles, r) + } + } + + if wasSingleInt { + err := rawDb.Table(table).Where("id = ?", id).Update("role", newRoles).Error + if err != nil { + utils.Log.Errorf("[convert roles] failed to update user %s: %v", username, err) + } else { + updatedCount++ + utils.Log.Infof("[convert roles] updated user %s: %v → %v", username, oldRoles, newRoles) + } + } + } + + utils.Log.Infof("[convert roles] completed role conversion for %d users", updatedCount) +} + +func IsLegacyRoleDetected() bool { + rawDb := db.GetDb() + table := conf.Conf.Database.TablePrefix + "users" + rows, err := rawDb.Table(table).Select("role").Rows() + if err != nil { + utils.Log.Errorf("[role check] failed to scan user roles: %v", err) + return false + } + defer rows.Close() + + for rows.Next() { + var raw sql.RawBytes + if err := rows.Scan(&raw); err != nil { + continue + } + if len(raw) == 0 { + continue + } + + var roles []int + if err := json.Unmarshal(raw, &roles); err == nil { + continue + } + + var single int + if err := json.Unmarshal(raw, &single); err == nil { + utils.Log.Infof("[role check] detected legacy int role: %d", single) + return true + } + } + return false +} diff --git a/internal/bootstrap/qbittorrent.go b/internal/bootstrap/qbittorrent.go deleted file mode 100644 index 315977ebe3f..00000000000 --- a/internal/bootstrap/qbittorrent.go +++ /dev/null @@ -1,15 +0,0 @@ -package bootstrap - -import ( - "github.com/alist-org/alist/v3/internal/qbittorrent" - "github.com/alist-org/alist/v3/pkg/utils" -) - -func InitQbittorrent() { - go func() { - err := qbittorrent.InitClient() - if err != nil { - utils.Log.Infof("qbittorrent not ready.") - } - }() -} diff --git a/internal/bootstrap/storage.go b/internal/bootstrap/storage.go index 44af99c56e1..86288edcb16 100644 --- a/internal/bootstrap/storage.go +++ b/internal/bootstrap/storage.go @@ -21,8 +21,8 @@ func LoadStorages() { if err != nil { utils.Log.Errorf("failed get enabled storages: %+v", err) } else { - utils.Log.Infof("success load storage: [%s], driver: [%s]", - storages[i].MountPath, storages[i].Driver) + utils.Log.Infof("success load storage: [%s], driver: [%s], order: [%d]", + storages[i].MountPath, storages[i].Driver, storages[i].Order) } } conf.StoragesLoaded = true diff --git a/internal/bootstrap/stream_limit.go b/internal/bootstrap/stream_limit.go new file mode 100644 index 00000000000..5ece71e4beb --- /dev/null +++ b/internal/bootstrap/stream_limit.go @@ -0,0 +1,53 @@ +package bootstrap + +import ( + "context" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/stream" + "golang.org/x/time/rate" +) + +type blockBurstLimiter struct { + *rate.Limiter +} + +func (l blockBurstLimiter) WaitN(ctx context.Context, total int) error { + for total > 0 { + n := l.Burst() + if l.Limiter.Limit() == rate.Inf || n > total { + n = total + } + err := l.Limiter.WaitN(ctx, n) + if err != nil { + return err + } + total -= n + } + return nil +} + +func streamFilterNegative(limit int) (rate.Limit, int) { + if limit < 0 { + return rate.Inf, 0 + } + return rate.Limit(limit) * 1024.0, limit * 1024 +} + +func initLimiter(limiter *stream.Limiter, s string) { + clientDownLimit, burst := streamFilterNegative(setting.GetInt(s, -1)) + *limiter = blockBurstLimiter{Limiter: rate.NewLimiter(clientDownLimit, burst)} + op.RegisterSettingChangingCallback(func() { + newLimit, newBurst := streamFilterNegative(setting.GetInt(s, -1)) + (*limiter).SetLimit(newLimit) + (*limiter).SetBurst(newBurst) + }) +} + +func InitStreamLimit() { + initLimiter(&stream.ClientDownloadLimit, conf.StreamMaxClientDownloadSpeed) + initLimiter(&stream.ClientUploadLimit, conf.StreamMaxClientUploadSpeed) + initLimiter(&stream.ServerDownloadLimit, conf.StreamMaxServerDownloadSpeed) + initLimiter(&stream.ServerUploadLimit, conf.StreamMaxServerUploadSpeed) +} diff --git a/internal/bootstrap/task.go b/internal/bootstrap/task.go new file mode 100644 index 00000000000..8f05b84a347 --- /dev/null +++ b/internal/bootstrap/task.go @@ -0,0 +1,60 @@ +package bootstrap + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/xhofe/tache" +) + +func taskFilterNegative(num int) int64 { + if num < 0 { + num = 0 + } + return int64(num) +} + +func InitTaskManager() { + fs.UploadTaskManager = tache.NewManager[*fs.UploadTask](tache.WithWorks(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //upload will not support persist + op.RegisterSettingChangingCallback(func() { + fs.UploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers))) + }) + fs.CopyTaskManager = tache.NewManager[*fs.CopyTask](tache.WithWorks(setting.GetInt(conf.TaskCopyThreadsNum, conf.Conf.Tasks.Copy.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant), db.UpdateTaskDataFunc("copy", conf.Conf.Tasks.Copy.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Copy.MaxRetry)) + op.RegisterSettingChangingCallback(func() { + fs.CopyTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskCopyThreadsNum, conf.Conf.Tasks.Copy.Workers))) + }) + tool.DownloadTaskManager = tache.NewManager[*tool.DownloadTask](tache.WithWorks(setting.GetInt(conf.TaskOfflineDownloadThreadsNum, conf.Conf.Tasks.Download.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant), db.UpdateTaskDataFunc("download", conf.Conf.Tasks.Download.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Download.MaxRetry)) + op.RegisterSettingChangingCallback(func() { + tool.DownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskOfflineDownloadThreadsNum, conf.Conf.Tasks.Download.Workers))) + }) + tool.TransferTaskManager = tache.NewManager[*tool.TransferTask](tache.WithWorks(setting.GetInt(conf.TaskOfflineDownloadTransferThreadsNum, conf.Conf.Tasks.Transfer.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant), db.UpdateTaskDataFunc("transfer", conf.Conf.Tasks.Transfer.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Transfer.MaxRetry)) + op.RegisterSettingChangingCallback(func() { + tool.TransferTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskOfflineDownloadTransferThreadsNum, conf.Conf.Tasks.Transfer.Workers))) + }) + if len(tool.TransferTaskManager.GetAll()) == 0 { //prevent offline downloaded files from being deleted + CleanTempDir() + } + workers := conf.Conf.Tasks.S3Transition.Workers + if workers < 0 { + workers = 0 + } + fs.S3TransitionTaskManager = tache.NewManager[*fs.S3TransitionTask]( + tache.WithWorks(workers), + tache.WithPersistFunction( + db.GetTaskDataFunc("s3_transition", conf.Conf.Tasks.S3Transition.TaskPersistant), + db.UpdateTaskDataFunc("s3_transition", conf.Conf.Tasks.S3Transition.TaskPersistant), + ), + tache.WithMaxRetry(conf.Conf.Tasks.S3Transition.MaxRetry), + ) + fs.ArchiveDownloadTaskManager = tache.NewManager[*fs.ArchiveDownloadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers)), tache.WithPersistFunction(db.GetTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant), db.UpdateTaskDataFunc("decompress", conf.Conf.Tasks.Decompress.TaskPersistant)), tache.WithMaxRetry(conf.Conf.Tasks.Decompress.MaxRetry)) + op.RegisterSettingChangingCallback(func() { + fs.ArchiveDownloadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressDownloadThreadsNum, conf.Conf.Tasks.Decompress.Workers))) + }) + fs.ArchiveContentUploadTaskManager.Manager = tache.NewManager[*fs.ArchiveContentUploadTask](tache.WithWorks(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.DecompressUpload.MaxRetry)) //decompress upload will not support persist + op.RegisterSettingChangingCallback(func() { + fs.ArchiveContentUploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers))) + }) +} diff --git a/internal/conf/config.go b/internal/conf/config.go index 67c2dc0fa56..383a5a4e52f 100644 --- a/internal/conf/config.go +++ b/internal/conf/config.go @@ -8,15 +8,22 @@ import ( ) type Database struct { - Type string `json:"type" env:"DB_TYPE"` - Host string `json:"host" env:"DB_HOST"` - Port int `json:"port" env:"DB_PORT"` - User string `json:"user" env:"DB_USER"` - Password string `json:"password" env:"DB_PASS"` - Name string `json:"name" env:"DB_NAME"` - DBFile string `json:"db_file" env:"DB_FILE"` - TablePrefix string `json:"table_prefix" env:"DB_TABLE_PREFIX"` - SSLMode string `json:"ssl_mode" env:"DB_SSL_MODE"` + Type string `json:"type" env:"TYPE"` + Host string `json:"host" env:"HOST"` + Port int `json:"port" env:"PORT"` + User string `json:"user" env:"USER"` + Password string `json:"password" env:"PASS"` + Name string `json:"name" env:"NAME"` + DBFile string `json:"db_file" env:"FILE"` + TablePrefix string `json:"table_prefix" env:"TABLE_PREFIX"` + SSLMode string `json:"ssl_mode" env:"SSL_MODE"` + DSN string `json:"dsn" env:"DSN"` +} + +type Meilisearch struct { + Host string `json:"host" env:"HOST"` + APIKey string `json:"api_key" env:"API_KEY"` + IndexPrefix string `json:"index_prefix" env:"INDEX_PREFIX"` } type Scheme struct { @@ -28,6 +35,7 @@ type Scheme struct { KeyFile string `json:"key_file" env:"KEY_FILE"` UnixFile string `json:"unix_file" env:"UNIX_FILE"` UnixFilePerm string `json:"unix_file_perm" env:"UNIX_FILE_PERM"` + EnableH2c bool `json:"enable_h2c" env:"ENABLE_H2C"` } type LogConfig struct { @@ -39,20 +47,82 @@ type LogConfig struct { Compress bool `json:"compress" env:"COMPRESS"` } +type TaskConfig struct { + Workers int `json:"workers" env:"WORKERS"` + MaxRetry int `json:"max_retry" env:"MAX_RETRY"` + TaskPersistant bool `json:"task_persistant" env:"TASK_PERSISTANT"` +} + +type TasksConfig struct { + Download TaskConfig `json:"download" envPrefix:"DOWNLOAD_"` + Transfer TaskConfig `json:"transfer" envPrefix:"TRANSFER_"` + Upload TaskConfig `json:"upload" envPrefix:"UPLOAD_"` + Copy TaskConfig `json:"copy" envPrefix:"COPY_"` + Decompress TaskConfig `json:"decompress" envPrefix:"DECOMPRESS_"` + DecompressUpload TaskConfig `json:"decompress_upload" envPrefix:"DECOMPRESS_UPLOAD_"` + S3Transition TaskConfig `json:"s3_transition" envPrefix:"S3_TRANSITION_"` + AllowRetryCanceled bool `json:"allow_retry_canceled" env:"ALLOW_RETRY_CANCELED"` +} + +type Cors struct { + AllowOrigins []string `json:"allow_origins" env:"ALLOW_ORIGINS"` + AllowMethods []string `json:"allow_methods" env:"ALLOW_METHODS"` + AllowHeaders []string `json:"allow_headers" env:"ALLOW_HEADERS"` +} + +type S3 struct { + Enable bool `json:"enable" env:"ENABLE"` + Port int `json:"port" env:"PORT"` + SSL bool `json:"ssl" env:"SSL"` +} + +type FTP struct { + Enable bool `json:"enable" env:"ENABLE"` + Listen string `json:"listen" env:"LISTEN"` + FindPasvPortAttempts int `json:"find_pasv_port_attempts" env:"FIND_PASV_PORT_ATTEMPTS"` + ActiveTransferPortNon20 bool `json:"active_transfer_port_non_20" env:"ACTIVE_TRANSFER_PORT_NON_20"` + IdleTimeout int `json:"idle_timeout" env:"IDLE_TIMEOUT"` + ConnectionTimeout int `json:"connection_timeout" env:"CONNECTION_TIMEOUT"` + DisableActiveMode bool `json:"disable_active_mode" env:"DISABLE_ACTIVE_MODE"` + DefaultTransferBinary bool `json:"default_transfer_binary" env:"DEFAULT_TRANSFER_BINARY"` + EnableActiveConnIPCheck bool `json:"enable_active_conn_ip_check" env:"ENABLE_ACTIVE_CONN_IP_CHECK"` + EnablePasvConnIPCheck bool `json:"enable_pasv_conn_ip_check" env:"ENABLE_PASV_CONN_IP_CHECK"` +} + +type SFTP struct { + Enable bool `json:"enable" env:"ENABLE"` + Listen string `json:"listen" env:"LISTEN"` +} + +type MCP struct { + Enable bool `json:"enable" env:"ENABLE"` + Port int `json:"port" env:"PORT"` +} + type Config struct { - Force bool `json:"force" env:"FORCE"` - SiteURL string `json:"site_url" env:"SITE_URL"` - Cdn string `json:"cdn" env:"CDN"` - JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` - TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` - Database Database `json:"database"` - Scheme Scheme `json:"scheme"` - TempDir string `json:"temp_dir" env:"TEMP_DIR"` - BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` - Log LogConfig `json:"log"` - DelayedStart int `json:"delayed_start" env:"DELAYED_START"` - MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` - TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` + Force bool `json:"force" env:"FORCE"` + SiteURL string `json:"site_url" env:"SITE_URL"` + Cdn string `json:"cdn" env:"CDN"` + JwtSecret string `json:"jwt_secret" env:"JWT_SECRET"` + TokenExpiresIn int `json:"token_expires_in" env:"TOKEN_EXPIRES_IN"` + Database Database `json:"database" envPrefix:"DB_"` + Meilisearch Meilisearch `json:"meilisearch" envPrefix:"MEILISEARCH_"` + Scheme Scheme `json:"scheme"` + TempDir string `json:"temp_dir" env:"TEMP_DIR"` + BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"` + DistDir string `json:"dist_dir"` + Log LogConfig `json:"log"` + DelayedStart int `json:"delayed_start" env:"DELAYED_START"` + MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"` + MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"` + TlsInsecureSkipVerify bool `json:"tls_insecure_skip_verify" env:"TLS_INSECURE_SKIP_VERIFY"` + Tasks TasksConfig `json:"tasks" envPrefix:"TASKS_"` + Cors Cors `json:"cors" envPrefix:"CORS_"` + S3 S3 `json:"s3" envPrefix:"S3_"` + FTP FTP `json:"ftp" envPrefix:"FTP_"` + SFTP SFTP `json:"sftp" envPrefix:"SFTP_"` + MCP MCP `json:"mcp" envPrefix:"MCP_"` + LastLaunchedVersion string `json:"last_launched_version"` } func DefaultConfig() *Config { @@ -79,6 +149,9 @@ func DefaultConfig() *Config { TablePrefix: "x_", DBFile: dbPath, }, + Meilisearch: Meilisearch{ + Host: "http://localhost:7700", + }, BleveDir: indexDir, Log: LogConfig{ Enable: true, @@ -88,6 +161,73 @@ func DefaultConfig() *Config { MaxAge: 28, }, MaxConnections: 0, - TlsInsecureSkipVerify: true, + MaxConcurrency: 64, + TlsInsecureSkipVerify: false, + Tasks: TasksConfig{ + Download: TaskConfig{ + Workers: 5, + MaxRetry: 1, + // TaskPersistant: true, + }, + Transfer: TaskConfig{ + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, + }, + Upload: TaskConfig{ + Workers: 5, + }, + Copy: TaskConfig{ + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, + }, + Decompress: TaskConfig{ + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, + }, + DecompressUpload: TaskConfig{ + Workers: 5, + MaxRetry: 2, + }, + S3Transition: TaskConfig{ + Workers: 5, + MaxRetry: 2, + // TaskPersistant: true, + }, + AllowRetryCanceled: false, + }, + Cors: Cors{ + AllowOrigins: []string{"*"}, + AllowMethods: []string{"*"}, + AllowHeaders: []string{"*"}, + }, + S3: S3{ + Enable: false, + Port: 5246, + SSL: false, + }, + FTP: FTP{ + Enable: false, + Listen: ":5221", + FindPasvPortAttempts: 50, + ActiveTransferPortNon20: false, + IdleTimeout: 900, + ConnectionTimeout: 30, + DisableActiveMode: false, + DefaultTransferBinary: false, + EnableActiveConnIPCheck: true, + EnablePasvConnIPCheck: true, + }, + SFTP: SFTP{ + Enable: false, + Listen: ":5222", + }, + MCP: MCP{ + Enable: false, + Port: 5248, + }, + LastLaunchedVersion: "", } } diff --git a/internal/conf/const.go b/internal/conf/const.go index 02a00060a68..79480a4589e 100644 --- a/internal/conf/const.go +++ b/internal/conf/const.go @@ -10,27 +10,34 @@ const ( const ( // site - VERSION = "version" - SiteTitle = "site_title" - Announcement = "announcement" - AllowIndexed = "allow_indexed" - AllowMounted = "allow_mounted" - RobotsTxt = "robots_txt" + VERSION = "version" + SiteTitle = "site_title" + Announcement = "announcement" + AllowIndexed = "allow_indexed" + AllowMounted = "allow_mounted" + RobotsTxt = "robots_txt" + AllowRegister = "allow_register" + DefaultRole = "default_role" + UseNewui = "use_newui" + FrontendRememberSort = "frontend_remember_sort" Logo = "logo" Favicon = "favicon" MainColor = "main_color" // preview - TextTypes = "text_types" - AudioTypes = "audio_types" - VideoTypes = "video_types" - ImageTypes = "image_types" - ProxyTypes = "proxy_types" - ProxyIgnoreHeaders = "proxy_ignore_headers" - AudioAutoplay = "audio_autoplay" - VideoAutoplay = "video_autoplay" - + TextTypes = "text_types" + AudioTypes = "audio_types" + VideoTypes = "video_types" + ImageTypes = "image_types" + ProxyTypes = "proxy_types" + ProxyIgnoreHeaders = "proxy_ignore_headers" + AudioAutoplay = "audio_autoplay" + VideoAutoplay = "video_autoplay" + ThumbnailSize = "thumbnail_size" + PreviewArchivesByDefault = "preview_archives_by_default" + ReadMeAutoRender = "readme_autorender" + FilterReadMeScripts = "filter_readme_scripts" // global HideFiles = "hide_files" CustomizeHead = "customize_head" @@ -41,7 +48,12 @@ const ( OcrApi = "ocr_api" FilenameCharMapping = "filename_char_mapping" ForwardDirectLinkParams = "forward_direct_link_params" + IgnoreDirectLinkParams = "ignore_direct_link_params" WebauthnLoginEnabled = "webauthn_login_enabled" + MaxDevices = "max_devices" + DeviceEvictPolicy = "device_evict_policy" + DeviceSessionTTL = "device_session_ttl" + MetaNotFoundCacheExpire = "meta_not_found_cache_expire" // index SearchIndex = "search_index" @@ -53,11 +65,27 @@ const ( Aria2Uri = "aria2_uri" Aria2Secret = "aria2_secret" + // transmission + TransmissionUri = "transmission_uri" + TransmissionSeedtime = "transmission_seedtime" + + // 115 + Pan115TempDir = "115_temp_dir" + + // pikpak + PikPakTempDir = "pikpak_temp_dir" + + // thunder + ThunderTempDir = "thunder_temp_dir" + + // guangyapan + GuangYaPanTempDir = "guangyapan_temp_dir" + // single Token = "token" IndexProgress = "index_progress" - //SSO + // SSO SSOClientId = "sso_client_id" SSOClientSecret = "sso_client_secret" SSOLoginEnabled = "sso_login_enabled" @@ -67,20 +95,73 @@ const ( SSOApplicationName = "sso_application_name" SSOEndpointName = "sso_endpoint_name" SSOJwtPublicKey = "sso_jwt_public_key" + SSOExtraScopes = "sso_extra_scopes" SSOAutoRegister = "sso_auto_register" SSODefaultDir = "sso_default_dir" SSODefaultPermission = "sso_default_permission" SSOCompatibilityMode = "sso_compatibility_mode" + // ldap + LdapLoginEnabled = "ldap_login_enabled" + LdapServer = "ldap_server" + LdapManagerDN = "ldap_manager_dn" + LdapManagerPassword = "ldap_manager_password" + LdapUserSearchBase = "ldap_user_search_base" + LdapUserSearchFilter = "ldap_user_search_filter" + LdapDefaultPermission = "ldap_default_permission" + LdapDefaultDir = "ldap_default_dir" + LdapLoginTips = "ldap_login_tips" + + // s3 + S3Buckets = "s3_buckets" + S3AccessKeyId = "s3_access_key_id" + S3SecretAccessKey = "s3_secret_access_key" + // qbittorrent QbittorrentUrl = "qbittorrent_url" QbittorrentSeedtime = "qbittorrent_seedtime" + + // ftp + FTPPublicHost = "ftp_public_host" + FTPPasvPortMap = "ftp_pasv_port_map" + FTPProxyUserAgent = "ftp_proxy_user_agent" + FTPMandatoryTLS = "ftp_mandatory_tls" + FTPImplicitTLS = "ftp_implicit_tls" + FTPTLSPrivateKeyPath = "ftp_tls_private_key_path" + FTPTLSPublicCertPath = "ftp_tls_public_cert_path" + + // frp + FRPEnabled = "frp_enabled" + FRPServerAddr = "frp_server_addr" + FRPServerPort = "frp_server_port" + FRPAuthToken = "frp_auth_token" + FRPProxyName = "frp_proxy_name" + FRPProxyType = "frp_proxy_type" + FRPCustomDomain = "frp_custom_domain" + FRPSubdomain = "frp_subdomain" + FRPRemotePort = "frp_remote_port" + FRPLocalPort = "frp_local_port" + FRPTLSEnable = "frp_tls_enable" + FRPSTCPSecretKey = "frp_stcp_secret_key" + FRPStatus = "frp_status" + + // traffic + TaskOfflineDownloadThreadsNum = "offline_download_task_threads_num" + TaskOfflineDownloadTransferThreadsNum = "offline_download_transfer_task_threads_num" + TaskUploadThreadsNum = "upload_task_threads_num" + TaskCopyThreadsNum = "copy_task_threads_num" + TaskDecompressDownloadThreadsNum = "decompress_download_task_threads_num" + TaskDecompressUploadThreadsNum = "decompress_upload_task_threads_num" + StreamMaxClientDownloadSpeed = "max_client_download_speed" + StreamMaxClientUploadSpeed = "max_client_upload_speed" + StreamMaxServerDownloadSpeed = "max_server_download_speed" + StreamMaxServerUploadSpeed = "max_server_upload_speed" ) const ( UNKNOWN = iota FOLDER - //OFFICE + // OFFICE VIDEO AUDIO TEXT diff --git a/internal/conf/var.go b/internal/conf/var.go index 0a8eb16fcd1..7ae1a5abfb9 100644 --- a/internal/conf/var.go +++ b/internal/conf/var.go @@ -7,7 +7,6 @@ import ( var ( BuiltAt string - GoVersion string GitAuthor string GitCommit string Version string = "dev" diff --git a/internal/db/db.go b/internal/db/db.go index cd3905ffc92..f33927be0ff 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -12,7 +12,7 @@ var db *gorm.DB func Init(d *gorm.DB) { db = d - err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode)) + err := AutoMigrate(new(model.Storage), new(model.User), new(model.Meta), new(model.SettingItem), new(model.SearchNode), new(model.TaskItem), new(model.SSHPublicKey), new(model.Role), new(model.Label), new(model.LabelFileBinding), new(model.ObjFile), new(model.Session), new(model.Share)) if err != nil { log.Fatalf("failed migrate database: %s", err.Error()) } diff --git a/internal/db/label.go b/internal/db/label.go new file mode 100644 index 00000000000..fd9842d680c --- /dev/null +++ b/internal/db/label.go @@ -0,0 +1,79 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" + "time" +) + +// GetLabels Get all label from database order by id +func GetLabels(pageIndex, pageSize int) ([]model.Label, int64, error) { + labelDB := db.Model(&model.Label{}) + var count int64 + if err := labelDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get label count") + } + var labels []model.Label + if err := labelDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&labels).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return labels, count, nil +} + +// GetLabelById Get Label by id, used to update label usually +func GetLabelById(id uint) (*model.Label, error) { + var label model.Label + label.ID = id + if err := db.First(&label).Error; err != nil { + return nil, errors.WithStack(err) + } + return &label, nil +} + +// CreateLabel just insert label to database +func CreateLabel(label model.Label) (uint, error) { + label.CreateTime = time.Now() + err := errors.WithStack(db.Create(&label).Error) + if err != nil { + return label.ID, errors.WithMessage(err, "failed create label in database") + } + return label.ID, nil +} + +// UpdateLabel just update storage in database +func UpdateLabel(label *model.Label) (*model.Label, error) { + label.CreateTime = time.Now() + _, err := GetLabelById(label.ID) + if err != nil { + return nil, errors.WithMessage(err, "failed get old label") + } + err = errors.WithStack(db.Save(label).Error) + if err != nil { + return nil, errors.WithMessage(err, "failed create label in database") + } + return label, nil +} + +// DeleteLabelById just delete label from database by id +func DeleteLabelById(id uint) error { + return errors.WithStack(db.Delete(&model.Label{}, id).Error) +} + +// GetLabelByIds Get label from database order by ids +func GetLabelByIds(ids []uint) ([]model.Label, error) { + labelDB := db.Model(&model.Label{}) + var labels []model.Label + if err := labelDB.Where(ids).Find(&labels).Error; err != nil { + return nil, errors.WithStack(err) + } + return labels, nil +} + +// GetLabelByName Get Label by name +func GetLabelByName(name string) bool { + var label model.Label + result := db.Where("name = ?", name).First(&label) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} diff --git a/internal/db/label_file_binding.go b/internal/db/label_file_binding.go new file mode 100644 index 00000000000..4dda80f2c42 --- /dev/null +++ b/internal/db/label_file_binding.go @@ -0,0 +1,192 @@ +package db + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" + "gorm.io/gorm/clause" + "time" +) + +// GetLabelIds Get all label_ids from database order by file_name +func GetLabelIds(userId uint, fileName string) ([]uint, error) { + //fmt.Printf(">>> [GetLabelIds] userId: %d, fileName: %s\n", userId, fileName) + labelFileBinDingDB := db.Model(&model.LabelFileBinding{}) + var labelIds []uint + if err := labelFileBinDingDB.Where("file_name = ?", fileName).Where("user_id = ?", userId).Pluck("label_id", &labelIds).Error; err != nil { + return nil, errors.WithStack(err) + } + return labelIds, nil +} + +func CreateLabelFileBinDing(fileName string, labelId, userId uint) error { + var labelFileBinDing model.LabelFileBinding + labelFileBinDing.UserId = userId + labelFileBinDing.LabelId = labelId + labelFileBinDing.FileName = fileName + labelFileBinDing.CreateTime = time.Now() + err := errors.WithStack(db.Create(&labelFileBinDing).Error) + if err != nil { + return errors.WithMessage(err, "failed create label in database") + } + return nil +} + +// GetLabelFileBinDingByLabelIdExists Get Label by label_id, used to del label usually +func GetLabelFileBinDingByLabelIdExists(labelId, userId uint) bool { + var labelFileBinDing model.LabelFileBinding + result := db.Where("label_id = ?", labelId).Where("user_id = ?", userId).First(&labelFileBinDing) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} + +// DelLabelFileBinDingByFileName used to del usually +func DelLabelFileBinDingByFileName(userId uint, fileName string) error { + return errors.WithStack(db.Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinding{}).Error) +} + +// DelLabelFileBinDingById used to del usually +func DelLabelFileBinDingById(labelId, userId uint, fileName string) error { + return errors.WithStack(db.Where("label_id = ?", labelId).Where("file_name = ?", fileName).Where("user_id = ?", userId).Delete(model.LabelFileBinding{}).Error) +} + +func GetLabelFileBinDingByLabelId(labelIds []uint, userId uint) (result []model.LabelFileBinding, err error) { + if err := db.Where("label_id in (?)", labelIds).Where("user_id = ?", userId).Find(&result).Error; err != nil { + return nil, errors.WithStack(err) + } + return result, nil +} + +func GetLabelBindingsByFileNamesPublic(fileNames []string) (map[string][]uint, error) { + var binds []model.LabelFileBinding + if err := db.Where("file_name IN ?", fileNames).Find(&binds).Error; err != nil { + return nil, errors.WithStack(err) + } + out := make(map[string][]uint, len(fileNames)) + seen := make(map[string]struct{}, len(binds)) + for _, b := range binds { + key := fmt.Sprintf("%s-%d", b.FileName, b.LabelId) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out[b.FileName] = append(out[b.FileName], b.LabelId) + } + return out, nil +} + +func GetLabelsByFileNamesPublic(fileNames []string) (map[string][]model.Label, error) { + bindMap, err := GetLabelBindingsByFileNamesPublic(fileNames) + if err != nil { + return nil, err + } + + idSet := make(map[uint]struct{}) + for _, ids := range bindMap { + for _, id := range ids { + idSet[id] = struct{}{} + } + } + if len(idSet) == 0 { + return make(map[string][]model.Label, 0), nil + } + allIDs := make([]uint, 0, len(idSet)) + for id := range idSet { + allIDs = append(allIDs, id) + } + labels, err := GetLabelByIds(allIDs) // 你已有的函数 + if err != nil { + return nil, err + } + + labelByID := make(map[uint]model.Label, len(labels)) + for _, l := range labels { + labelByID[l.ID] = l + } + + out := make(map[string][]model.Label, len(bindMap)) + for fname, ids := range bindMap { + for _, id := range ids { + if lab, ok := labelByID[id]; ok { + out[fname] = append(out[fname], lab) + } + } + } + return out, nil +} + +func ListLabelFileBinDing(userId uint, labelIDs []uint, fileName string, page, pageSize int) ([]model.LabelFileBinding, int64, error) { + q := db.Model(&model.LabelFileBinding{}).Where("user_id = ?", userId) + + if len(labelIDs) > 0 { + q = q.Where("label_id IN ?", labelIDs) + } + if fileName != "" { + q = q.Where("file_name LIKE ?", "%"+fileName+"%") + } + + var total int64 + if err := q.Count(&total).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + + var rows []model.LabelFileBinding + if err := q. + Order("id DESC"). + Offset((page - 1) * pageSize). + Limit(pageSize). + Find(&rows).Error; err != nil { + return nil, 0, errors.WithStack(err) + } + return rows, total, nil +} + +func RestoreLabelFileBindings(bindings []model.LabelFileBinding, keepIDs bool, override bool) error { + if len(bindings) == 0 { + return nil + } + tx := db.Begin() + + if override { + type key struct { + uid uint + name string + } + toDel := make(map[key]struct{}, len(bindings)) + for i := range bindings { + k := key{uid: bindings[i].UserId, name: bindings[i].FileName} + toDel[k] = struct{}{} + } + for k := range toDel { + if err := tx.Where("user_id = ? AND file_name = ?", k.uid, k.name). + Delete(&model.LabelFileBinding{}).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } + } + + for i := range bindings { + b := bindings[i] + if !keepIDs { + b.ID = 0 + } + if b.CreateTime.IsZero() { + b.CreateTime = time.Now() + } + if override { + if err := tx.Create(&b).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } else { + if err := tx.Clauses(clause.OnConflict{DoNothing: true}).Create(&b).Error; err != nil { + tx.Rollback() + return errors.WithStack(err) + } + } + } + + return errors.WithStack(tx.Commit().Error) +} diff --git a/internal/db/meta.go b/internal/db/meta.go index 8b6a605e809..14532637a7b 100644 --- a/internal/db/meta.go +++ b/internal/db/meta.go @@ -34,7 +34,7 @@ func GetMetas(pageIndex, pageSize int) (metas []model.Meta, count int64, err err if err = metaDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get metas count") } - if err = metaDB.Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&metas).Error; err != nil { + if err = metaDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&metas).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find metas") } return metas, count, nil diff --git a/internal/db/obj_file.go b/internal/db/obj_file.go new file mode 100644 index 00000000000..2bbce9e6dd6 --- /dev/null +++ b/internal/db/obj_file.go @@ -0,0 +1,31 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// GetFileByNameExists Get file by name +func GetFileByNameExists(name string) bool { + var label model.ObjFile + result := db.Where("name = ?", name).First(&label) + exists := !errors.Is(result.Error, gorm.ErrRecordNotFound) + return exists +} + +// GetFileByName Get file by name +func GetFileByName(name string, userId uint) (objFile model.ObjFile, err error) { + if err = db.Where("name = ?", name).Where("user_id = ?", userId).First(&objFile).Error; err != nil { + return objFile, errors.WithStack(err) + } + return objFile, nil +} + +func CreateObjFile(obj model.ObjFile) error { + err := errors.WithStack(db.Create(&obj).Error) + if err != nil { + return errors.WithMessage(err, "failed create file in database") + } + return nil +} diff --git a/internal/db/role.go b/internal/db/role.go new file mode 100644 index 00000000000..d0b776b391d --- /dev/null +++ b/internal/db/role.go @@ -0,0 +1,106 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "path" + "strings" +) + +func GetRole(id uint) (*model.Role, error) { + var r model.Role + if err := db.First(&r, id).Error; err != nil { + return nil, errors.Wrapf(err, "failed get role") + } + return &r, nil +} + +func GetRoleByName(name string) (*model.Role, error) { + r := model.Role{Name: name} + if err := db.Where(r).First(&r).Error; err != nil { + return nil, errors.Wrapf(err, "failed get role") + } + return &r, nil +} + +func GetRoles(pageIndex, pageSize int) (roles []model.Role, count int64, err error) { + roleDB := db.Model(&model.Role{}) + if err = roleDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get roles count") + } + if err = roleDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&roles).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get find roles") + } + return roles, count, nil +} + +func GetAllRoles() ([]model.Role, error) { + var roles []model.Role + if err := db.Find(&roles).Error; err != nil { + return nil, errors.WithStack(err) + } + return roles, nil +} + +func CreateRole(r *model.Role) error { + if err := db.Create(r).Error; err != nil { + return errors.WithStack(err) + } + if r.Default { + if err := db.Model(&model.Role{}).Where("id <> ?", r.ID).Update("default", false).Error; err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func UpdateRole(r *model.Role) error { + if err := db.Save(r).Error; err != nil { + return errors.WithStack(err) + } + if r.Default { + if err := db.Model(&model.Role{}).Where("id <> ?", r.ID).Update("default", false).Error; err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func DeleteRole(id uint) error { + return errors.WithStack(db.Delete(&model.Role{}, id).Error) +} + +func UpdateRolePermissionsPathPrefix(oldPath, newPath string) ([]uint, error) { + var roles []model.Role + var modifiedRoleIDs []uint + + if err := db.Find(&roles).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load roles") + } + + for _, role := range roles { + if role.Name == "admin" || role.Name == "guest" { + continue + } + updated := false + for i, entry := range role.PermissionScopes { + entryPath := path.Clean(entry.Path) + oldPathClean := path.Clean(oldPath) + + if entryPath == oldPathClean { + role.PermissionScopes[i].Path = newPath + updated = true + } else if strings.HasPrefix(entryPath, oldPathClean+"/") { + role.PermissionScopes[i].Path = newPath + entryPath[len(oldPathClean):] + updated = true + } + } + if updated { + if err := UpdateRole(&role); err != nil { + return nil, errors.WithMessagef(err, "failed to update role ID %d", role.ID) + } + modifiedRoleIDs = append(modifiedRoleIDs, role.ID) + } + } + return modifiedRoleIDs, nil +} diff --git a/internal/db/session.go b/internal/db/session.go new file mode 100644 index 00000000000..35c778c3ac8 --- /dev/null +++ b/internal/db/session.go @@ -0,0 +1,69 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "gorm.io/gorm/clause" +) + +func GetSession(userID uint, deviceKey string) (*model.Session, error) { + s := model.Session{UserID: userID, DeviceKey: deviceKey} + if err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where(&s).First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed find session") + } + return &s, nil +} + +func CreateSession(s *model.Session) error { + return errors.WithStack(db.Create(s).Error) +} + +func UpsertSession(s *model.Session) error { + return errors.WithStack(db.Clauses(clause.OnConflict{UpdateAll: true}).Create(s).Error) +} + +func DeleteSession(userID uint, deviceKey string) error { + return errors.WithStack(db.Where("user_id = ? AND device_key = ?", userID, deviceKey).Delete(&model.Session{}).Error) +} + +func CountActiveSessionsByUser(userID uint) (int64, error) { + var count int64 + err := db.Model(&model.Session{}). + Where("user_id = ? AND status = ?", userID, model.SessionActive). + Count(&count).Error + return count, errors.WithStack(err) +} + +func DeleteSessionsBefore(ts int64) error { + return errors.WithStack(db.Where("last_active < ?", ts).Delete(&model.Session{}).Error) +} + +// GetOldestActiveSession returns the oldest active session for the specified user. +func GetOldestActiveSession(userID uint) (*model.Session, error) { + var s model.Session + if err := db.Where("user_id = ? AND status = ?", userID, model.SessionActive). + Order("last_active ASC").First(&s).Error; err != nil { + return nil, errors.Wrap(err, "failed get oldest active session") + } + return &s, nil +} + +func UpdateSessionLastActive(userID uint, deviceKey string, lastActive int64) error { + return errors.WithStack(db.Model(&model.Session{}).Where("user_id = ? AND device_key = ?", userID, deviceKey).Update("last_active", lastActive).Error) +} + +func ListSessionsByUser(userID uint) ([]model.Session, error) { + var sessions []model.Session + err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where("user_id = ? AND status = ?", userID, model.SessionActive).Find(&sessions).Error + return sessions, errors.WithStack(err) +} + +func ListSessions() ([]model.Session, error) { + var sessions []model.Session + err := db.Select("user_id, device_key, last_active, status, user_agent, ip").Where("status = ?", model.SessionActive).Find(&sessions).Error + return sessions, errors.WithStack(err) +} + +func MarkInactive(sessionID string) error { + return errors.WithStack(db.Model(&model.Session{}).Where("device_key = ?", sessionID).Update("status", model.SessionInactive).Error) +} diff --git a/internal/db/settingitem.go b/internal/db/settingitem.go index 2ba0c665acd..9e61bfca00b 100644 --- a/internal/db/settingitem.go +++ b/internal/db/settingitem.go @@ -49,7 +49,8 @@ func GetSettingItemsByGroup(group int) ([]model.SettingItem, error) { func GetSettingItemsInGroups(groups []int) ([]model.SettingItem, error) { var settingItems []model.SettingItem - if err := db.Where(fmt.Sprintf("%s in ?", columnName("group")), groups).Find(&settingItems).Error; err != nil { + err := db.Order(columnName("index")).Where(fmt.Sprintf("%s in ?", columnName("group")), groups).Find(&settingItems).Error + if err != nil { return nil, errors.WithStack(err) } return settingItems, nil diff --git a/internal/db/share.go b/internal/db/share.go new file mode 100644 index 00000000000..711c3f278c3 --- /dev/null +++ b/internal/db/share.go @@ -0,0 +1,124 @@ +package db + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +func GetShareByShareID(shareID string) (*model.Share, error) { + var share model.Share + if err := db.Where("share_id = ?", shareID).Take(&share).Error; err != nil { + return nil, err + } + return &share, nil +} + +func GetShareByCreatorAndShareID(creatorID uint, shareID string) (*model.Share, error) { + var share model.Share + if err := db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Take(&share).Error; err != nil { + return nil, err + } + return &share, nil +} + +func ShareIDExists(shareID string) (bool, error) { + var count int64 + if err := db.Model(&model.Share{}).Where("share_id = ?", shareID).Count(&count).Error; err != nil { + return false, err + } + return count > 0, nil +} + +func ShareIDExistsExceptID(shareID string, id uint) (bool, error) { + var count int64 + if err := db.Model(&model.Share{}).Where("share_id = ? AND id <> ?", shareID, id).Count(&count).Error; err != nil { + return false, err + } + return count > 0, nil +} + +func CreateShare(share *model.Share) error { + return db.Create(share).Error +} + +func UpdateShare(share *model.Share) error { + return db.Save(share).Error +} + +func GetSharesByCreator(creatorID uint, pageIndex, pageSize int) (shares []model.Share, count int64, err error) { + tx := db.Model(&model.Share{}).Where("creator_id = ?", creatorID) + err = tx.Count(&count).Error + if err != nil { + return nil, 0, err + } + err = tx.Order("created_at desc").Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&shares).Error + return +} + +func DeleteShareByShareID(creatorID uint, shareID string) error { + return db.Where("creator_id = ? AND share_id = ?", creatorID, shareID).Delete(&model.Share{}).Error +} + +func DisableShareByShareID(creatorID uint, shareID string) error { + return db.Model(&model.Share{}). + Where("creator_id = ? AND share_id = ?", creatorID, shareID). + Update("enabled", false).Error +} + +func TouchShareView(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ?", shareID). + UpdateColumns(map[string]interface{}{ + "last_access_at": now, + "view_count": gorm.Expr("view_count + ?", 1), + }).Error +} + +func TouchShareDownload(shareID string) error { + now := time.Now() + return db.Model(&model.Share{}). + Where("share_id = ?", shareID). + UpdateColumns(map[string]interface{}{ + "last_access_at": now, + "download_count": gorm.Expr("download_count + ?", 1), + }).Error +} + +func RecordShareAccess(shareID string) (*model.Share, error) { + var updated model.Share + err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Clauses(clause.Locking{Strength: "UPDATE"}). + Where("share_id = ?", shareID). + Take(&updated).Error; err != nil { + return err + } + + now := time.Now() + updated.AccessCount++ + updated.LastAccessAt = &now + updates := map[string]interface{}{ + "access_count": updated.AccessCount, + "last_access_at": now, + } + + limit := updated.EffectiveAccessLimit() + if limit > 0 && updated.AccessCount >= limit { + updated.Enabled = false + updated.ConsumedAt = &now + updates["enabled"] = false + updates["consumed_at"] = now + } + + return tx.Model(&model.Share{}). + Where("id = ?", updated.ID). + Updates(updates).Error + }) + if err != nil { + return nil, err + } + return &updated, nil +} diff --git a/internal/db/sshkey.go b/internal/db/sshkey.go new file mode 100644 index 00000000000..f51dbfdc453 --- /dev/null +++ b/internal/db/sshkey.go @@ -0,0 +1,57 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" +) + +func GetSSHPublicKeyByUserId(userId uint, pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { + keyDB := db.Model(&model.SSHPublicKey{}) + query := model.SSHPublicKey{UserId: userId} + if err := keyDB.Where(query).Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get user's keys count") + } + if err := keyDB.Where(query).Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&keys).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get find user's keys") + } + return keys, count, nil +} + +func GetSSHPublicKeyById(id uint) (*model.SSHPublicKey, error) { + var k model.SSHPublicKey + if err := db.First(&k, id).Error; err != nil { + return nil, errors.Wrapf(err, "failed get old key") + } + return &k, nil +} + +func GetSSHPublicKeyByUserTitle(userId uint, title string) (*model.SSHPublicKey, error) { + key := model.SSHPublicKey{UserId: userId, Title: title} + if err := db.Where(key).First(&key).Error; err != nil { + return nil, errors.Wrapf(err, "failed find key with title of user") + } + return &key, nil +} + +func CreateSSHPublicKey(k *model.SSHPublicKey) error { + return errors.WithStack(db.Create(k).Error) +} + +func UpdateSSHPublicKey(k *model.SSHPublicKey) error { + return errors.WithStack(db.Save(k).Error) +} + +func GetSSHPublicKeys(pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { + keyDB := db.Model(&model.SSHPublicKey{}) + if err := keyDB.Count(&count).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get keys count") + } + if err := keyDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&keys).Error; err != nil { + return nil, 0, errors.Wrapf(err, "failed get find keys") + } + return keys, count, nil +} + +func DeleteSSHPublicKeyById(id uint) error { + return errors.WithStack(db.Delete(&model.SSHPublicKey{}, id).Error) +} diff --git a/internal/db/storage.go b/internal/db/storage.go index 105bc0aafda..376d42d7bf9 100644 --- a/internal/db/storage.go +++ b/internal/db/storage.go @@ -35,7 +35,7 @@ func GetStorages(pageIndex, pageSize int) ([]model.Storage, int64, error) { return nil, 0, errors.Wrapf(err, "failed get storages count") } var storages []model.Storage - if err := storageDB.Order(columnName("order")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { + if err := addStorageOrder(storageDB).Order(columnName("order")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&storages).Error; err != nil { return nil, 0, errors.WithStack(err) } return storages, count, nil @@ -62,7 +62,8 @@ func GetStorageByMountPath(mountPath string) (*model.Storage, error) { func GetEnabledStorages() ([]model.Storage, error) { var storages []model.Storage - if err := db.Where(fmt.Sprintf("%s = ?", columnName("disabled")), false).Find(&storages).Error; err != nil { + err := addStorageOrder(db).Where(fmt.Sprintf("%s = ?", columnName("disabled")), false).Find(&storages).Error + if err != nil { return nil, errors.WithStack(err) } return storages, nil diff --git a/internal/db/tasks.go b/internal/db/tasks.go new file mode 100644 index 00000000000..9d2de1cff64 --- /dev/null +++ b/internal/db/tasks.go @@ -0,0 +1,48 @@ +package db + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" +) + +func GetTaskDataByType(type_s string) (*model.TaskItem, error) { + task := model.TaskItem{Key: type_s} + if err := db.Where(task).First(&task).Error; err != nil { + return nil, errors.Wrapf(err, "failed find task") + } + return &task, nil +} + +func UpdateTaskData(t *model.TaskItem) error { + return errors.WithStack(db.Model(&model.TaskItem{}).Where("key = ?", t.Key).Update("persist_data", t.PersistData).Error) +} + +func CreateTaskData(t *model.TaskItem) error { + return errors.WithStack(db.Create(t).Error) +} + +func GetTaskDataFunc(type_s string, enabled bool) func() ([]byte, error) { + if !enabled { + return nil + } + task, err := GetTaskDataByType(type_s) + if err != nil { + return nil + } + return func() ([]byte, error) { + return []byte(task.PersistData), nil + } +} + +func UpdateTaskDataFunc(type_s string, enabled bool) func([]byte) error { + if !enabled { + return nil + } + return func(data []byte) error { + s := string(data) + if s == "null" || s == "" { + s = "[]" + } + return UpdateTaskData(&model.TaskItem{Key: type_s, PersistData: s}) + } +} diff --git a/internal/db/user.go b/internal/db/user.go index 822926664c9..4e5d67ad28e 100644 --- a/internal/db/user.go +++ b/internal/db/user.go @@ -2,19 +2,42 @@ package db import ( "encoding/base64" - + "fmt" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-webauthn/webauthn/webauthn" "github.com/pkg/errors" + "gorm.io/gorm" + "path" + "slices" + "strings" ) func GetUserByRole(role int) (*model.User, error) { - user := model.User{Role: role} - if err := db.Where(user).Take(&user).Error; err != nil { + var users []model.User + if err := db.Find(&users).Error; err != nil { return nil, err } - return &user, nil + for i := range users { + if users[i].Role.Contains(role) { + return &users[i], nil + } + } + return nil, gorm.ErrRecordNotFound +} + +func GetUsersByRole(roleID int) ([]model.User, error) { + var users []model.User + if err := db.Find(&users).Error; err != nil { + return nil, err + } + var result []model.User + for _, u := range users { + if slices.Contains(u.Role, roleID) { + result = append(result, u) + } + } + return result, nil } func GetUserByName(username string) (*model.User, error) { @@ -54,12 +77,20 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err if err := userDB.Count(&count).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get users count") } - if err := userDB.Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&users).Error; err != nil { + if err := userDB.Order(columnName("id")).Offset((pageIndex - 1) * pageSize).Limit(pageSize).Find(&users).Error; err != nil { return nil, 0, errors.Wrapf(err, "failed get find users") } return users, count, nil } +func GetAllUsers() ([]model.User, error) { + var users []model.User + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithStack(err) + } + return users, nil +} + func DeleteUserById(id uint) error { return errors.WithStack(db.Delete(&model.User{}, id).Error) } @@ -100,3 +131,50 @@ func RemoveAuthn(u *model.User, id string) error { } return UpdateAuthn(u.ID, string(res)) } + +func UpdateUserBasePathPrefix(oldPath, newPath string, usersOpt ...[]model.User) ([]string, error) { + var users []model.User + var modifiedUsernames []string + + oldPathClean := path.Clean(oldPath) + + if len(usersOpt) > 0 { + users = usersOpt[0] + } else { + if err := db.Find(&users).Error; err != nil { + return nil, errors.WithMessage(err, "failed to load users") + } + } + + for _, user := range users { + basePath := path.Clean(user.BasePath) + updated := false + + if basePath == oldPathClean { + user.BasePath = path.Clean(newPath) + updated = true + } else if strings.HasPrefix(basePath, oldPathClean+"/") { + user.BasePath = path.Clean(newPath + basePath[len(oldPathClean):]) + updated = true + } + + if updated { + if err := UpdateUser(&user); err != nil { + return nil, errors.WithMessagef(err, "failed to update user ID %d", user.ID) + } + modifiedUsernames = append(modifiedUsernames, user.Username) + } + } + + return modifiedUsernames, nil +} + +func CountUsersByRoleAndEnabledExclude(roleID uint, excludeUserID uint) (int64, error) { + var count int64 + jsonValue := fmt.Sprintf("[%d]", roleID) + err := db.Model(&model.User{}). + Where("disabled = ? AND id != ?", false, excludeUserID). + Where("JSON_CONTAINS(role, ?)", jsonValue). + Count(&count).Error + return count, err +} diff --git a/internal/db/util.go b/internal/db/util.go index 38a06bcdacc..57f615bdeb1 100644 --- a/internal/db/util.go +++ b/internal/db/util.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/alist-org/alist/v3/internal/conf" + "gorm.io/gorm" ) func columnName(name string) string { @@ -12,3 +13,7 @@ func columnName(name string) string { } return fmt.Sprintf("`%s`", name) } + +func addStorageOrder(db *gorm.DB) *gorm.DB { + return db.Order(fmt.Sprintf("%s, %s", columnName("order"), columnName("id"))) +} diff --git a/internal/device/session.go b/internal/device/session.go new file mode 100644 index 00000000000..1d9e7ea53cd --- /dev/null +++ b/internal/device/session.go @@ -0,0 +1,138 @@ +package device + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + "gorm.io/gorm" +) + +// Handle verifies device sessions for a user and upserts current session. +func Handle(userID uint, deviceKey, ua, ip string) error { + ttl := setting.GetInt(conf.DeviceSessionTTL, 86400) + if ttl > 0 { + _ = db.DeleteSessionsBefore(time.Now().Unix() - int64(ttl)) + } + + ip = utils.MaskIP(ip) + + now := time.Now().Unix() + sess, err := db.GetSession(userID, deviceKey) + if err == nil { + if sess.Status == model.SessionInactive { + return errors.WithStack(errs.SessionInactive) + } + sess.Status = model.SessionActive + sess.LastActive = now + sess.UserAgent = ua + sess.IP = ip + return db.UpsertSession(sess) + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, err := db.GetOldestActiveSession(userID); err == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + + s := &model.Session{UserID: userID, DeviceKey: deviceKey, UserAgent: ua, IP: ip, LastActive: now, Status: model.SessionActive} + return db.CreateSession(s) +} + +// EnsureActiveOnLogin is used only in login flow: +// - If session exists (even Inactive): reactivate and refresh fields. +// - If not exists: apply max-devices policy, then create Active session. +func EnsureActiveOnLogin(userID uint, deviceKey, ua, ip string) error { + ip = utils.MaskIP(ip) + now := time.Now().Unix() + + sess, err := db.GetSession(userID, deviceKey) + if err == nil { + if sess.Status == model.SessionInactive { + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, gerr := db.GetOldestActiveSession(userID); gerr == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + } + sess.Status = model.SessionActive + sess.LastActive = now + sess.UserAgent = ua + sess.IP = ip + return db.UpsertSession(sess) + } + if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) { + return err + } + + max := setting.GetInt(conf.MaxDevices, 0) + if max > 0 { + count, err := db.CountActiveSessionsByUser(userID) + if err != nil { + return err + } + if count >= int64(max) { + policy := setting.GetStr(conf.DeviceEvictPolicy, "deny") + if policy == "evict_oldest" { + if oldest, gerr := db.GetOldestActiveSession(userID); gerr == nil { + if err := db.MarkInactive(oldest.DeviceKey); err != nil { + return err + } + } + } else { + return errors.WithStack(errs.TooManyDevices) + } + } + } + + return db.CreateSession(&model.Session{ + UserID: userID, + DeviceKey: deviceKey, + UserAgent: ua, + IP: ip, + LastActive: now, + Status: model.SessionActive, + }) +} + +// Refresh updates last_active for the session. +func Refresh(userID uint, deviceKey string) { + _ = db.UpdateSessionLastActive(userID, deviceKey, time.Now().Unix()) +} diff --git a/internal/driver/config.go b/internal/driver/config.go index 35ff6e4f2ed..6068143cb71 100644 --- a/internal/driver/config.go +++ b/internal/driver/config.go @@ -11,7 +11,8 @@ type Config struct { DefaultRoot string `json:"default_root"` CheckStatus bool `json:"-"` Alert string `json:"alert"` //info,success,warning,danger - NoOverwriteUpload bool `json:"-"` + NoOverwriteUpload bool `json:"-"` // whether to support overwrite upload + ProxyRangeOption bool `json:"-"` } func (c Config) MustProxy() bool { diff --git a/internal/driver/driver.go b/internal/driver/driver.go index e0a7c93d908..9e9440b6700 100644 --- a/internal/driver/driver.go +++ b/internal/driver/driver.go @@ -77,7 +77,37 @@ type Remove interface { } type Put interface { - Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) error + // Put a file (provided as a FileStreamer) into the driver + // Besides the most basic upload functionality, the following features also need to be implemented: + // 1. Canceling (when `<-ctx.Done()` returns), which can be supported by the following methods: + // (1) Use request methods that carry context, such as the following: + // a. http.NewRequestWithContext + // b. resty.Request.SetContext + // c. s3manager.Uploader.UploadWithContext + // d. utils.CopyWithCtx + // (2) Use a `driver.ReaderWithCtx` or `driver.NewLimitedUploadStream` + // (3) Use `utils.IsCanceled` to check if the upload has been canceled during the upload process, + // this is typically applicable to chunked uploads. + // 2. Submit upload progress (via `up`) in real-time. There are three recommended ways as follows: + // (1) Use `utils.CopyWithCtx` + // (2) Use `driver.ReaderUpdatingProgress` + // (3) Use `driver.Progress` with `io.TeeReader` + // 3. Slow down upload speed (via `stream.ServerUploadLimit`). It requires you to wrap the read stream + // in a `driver.RateLimitReader` or a `driver.RateLimitFile` after calculating the file's hash and + // before uploading the file or file chunks. Or you can directly call `driver.ServerUploadLimitWaitN` + // if your file chunks are sufficiently small (less than about 50KB). + // NOTE that the network speed may be significantly slower than the stream's read speed. Therefore, if + // you use a `errgroup.Group` to upload each chunk in parallel, you should consider using a recursive + // mutex like `semaphore.Weighted` to limit the maximum number of upload threads, preventing excessive + // memory usage caused by buffering too many file chunks awaiting upload. + Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up UpdateProgress) error +} + +type PutURL interface { + // PutURL directly put a URL into the storage + // Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs + // Called when using SimpleHttp for offline downloading, skipping creating a download task + PutURL(ctx context.Context, dstDir model.Obj, name, url string) error } //type WriteResult interface { @@ -106,27 +136,75 @@ type CopyResult interface { } type PutResult interface { - Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up UpdateProgress) (model.Obj, error) -} - -type UpdateProgress func(percentage int) - -type Progress struct { - Total int64 - Done int64 - up UpdateProgress -} - -func (p *Progress) Write(b []byte) (n int, err error) { - n = len(b) - p.Done += int64(n) - p.up(int(float64(p.Done) / float64(p.Total) * 100)) - return -} - -func NewProgress(total int64, up UpdateProgress) *Progress { - return &Progress{ - Total: total, - up: up, - } + // Put a file (provided as a FileStreamer) into the driver and return the put obj + // Besides the most basic upload functionality, the following features also need to be implemented: + // 1. Canceling (when `<-ctx.Done()` returns), which can be supported by the following methods: + // (1) Use request methods that carry context, such as the following: + // a. http.NewRequestWithContext + // b. resty.Request.SetContext + // c. s3manager.Uploader.UploadWithContext + // d. utils.CopyWithCtx + // (2) Use a `driver.ReaderWithCtx` or `driver.NewLimitedUploadStream` + // (3) Use `utils.IsCanceled` to check if the upload has been canceled during the upload process, + // this is typically applicable to chunked uploads. + // 2. Submit upload progress (via `up`) in real-time. There are three recommended ways as follows: + // (1) Use `utils.CopyWithCtx` + // (2) Use `driver.ReaderUpdatingProgress` + // (3) Use `driver.Progress` with `io.TeeReader` + // 3. Slow down upload speed (via `stream.ServerUploadLimit`). It requires you to wrap the read stream + // in a `driver.RateLimitReader` or a `driver.RateLimitFile` after calculating the file's hash and + // before uploading the file or file chunks. Or you can directly call `driver.ServerUploadLimitWaitN` + // if your file chunks are sufficiently small (less than about 50KB). + // NOTE that the network speed may be significantly slower than the stream's read speed. Therefore, if + // you use a `errgroup.Group` to upload each chunk in parallel, you should consider using a recursive + // mutex like `semaphore.Weighted` to limit the maximum number of upload threads, preventing excessive + // memory usage caused by buffering too many file chunks awaiting upload. + Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up UpdateProgress) (model.Obj, error) +} + +type PutURLResult interface { + // PutURL directly put a URL into the storage + // Applicable to index-based drivers like URL-Tree or drivers that support uploading files as URLs + // Called when using SimpleHttp for offline downloading, skipping creating a download task + PutURL(ctx context.Context, dstDir model.Obj, name, url string) (model.Obj, error) +} + +type ArchiveReader interface { + // GetArchiveMeta get the meta-info of an archive + // return errs.WrongArchivePassword if the meta-info is also encrypted but provided password is wrong or empty + // return errs.NotImplement to use internal archive tools to get the meta-info, such as the following cases: + // 1. the driver do not support the format of the archive but there may be an internal tool do + // 2. handling archives is a VIP feature, but the driver does not have VIP access + GetArchiveMeta(ctx context.Context, obj model.Obj, args model.ArchiveArgs) (model.ArchiveMeta, error) + // ListArchive list the children of model.ArchiveArgs.InnerPath in the archive + // return errs.NotImplement to use internal archive tools to list the children + // return errs.NotSupport if the folder structure should be acquired from model.ArchiveMeta.GetTree + ListArchive(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) ([]model.Obj, error) + // Extract get url/filepath/reader of a file in the archive + // return errs.NotImplement to use internal archive tools to extract + Extract(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (*model.Link, error) +} + +type ArchiveGetter interface { + // ArchiveGet get file by inner path + // return errs.NotImplement to use internal archive tools to get the children + // return errs.NotSupport if the folder structure should be acquired from model.ArchiveMeta.GetTree + ArchiveGet(ctx context.Context, obj model.Obj, args model.ArchiveInnerArgs) (model.Obj, error) +} + +type ArchiveDecompress interface { + ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) error +} + +type ArchiveDecompressResult interface { + // ArchiveDecompress decompress an archive + // when args.PutIntoNewDir, the new sub-folder should be named the same to the archive but without the extension + // return each decompressed obj from the root path of the archive when args.PutIntoNewDir is false + // return only the newly created folder when args.PutIntoNewDir is true + // return errs.NotImplement to use internal archive tools to decompress + ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj, args model.ArchiveDecompressArgs) ([]model.Obj, error) +} + +type Reference interface { + InitReference(storage Driver) error } diff --git a/internal/driver/proxy.go b/internal/driver/proxy.go new file mode 100644 index 00000000000..f94273c9f92 --- /dev/null +++ b/internal/driver/proxy.go @@ -0,0 +1,7 @@ +package driver + +// ProxyDriver lets a driver override the default "must proxy" download behavior +// on a per-storage basis. +type ProxyDriver interface { + ShouldProxyDownloads() bool +} diff --git a/internal/driver/utils.go b/internal/driver/utils.go new file mode 100644 index 00000000000..2af850ecb8d --- /dev/null +++ b/internal/driver/utils.go @@ -0,0 +1,62 @@ +package driver + +import ( + "context" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "io" +) + +type UpdateProgress = model.UpdateProgress + +type Progress struct { + Total int64 + Done int64 + up UpdateProgress +} + +func (p *Progress) Write(b []byte) (n int, err error) { + n = len(b) + p.Done += int64(n) + p.up(float64(p.Done) / float64(p.Total) * 100) + return +} + +func NewProgress(total int64, up UpdateProgress) *Progress { + return &Progress{ + Total: total, + up: up, + } +} + +type RateLimitReader = stream.RateLimitReader + +type RateLimitWriter = stream.RateLimitWriter + +type RateLimitFile = stream.RateLimitFile + +func NewLimitedUploadStream(ctx context.Context, r io.Reader) *RateLimitReader { + return &RateLimitReader{ + Reader: r, + Limiter: stream.ServerUploadLimit, + Ctx: ctx, + } +} + +func NewLimitedUploadFile(ctx context.Context, f model.File) *RateLimitFile { + return &RateLimitFile{ + File: f, + Limiter: stream.ServerUploadLimit, + Ctx: ctx, + } +} + +func ServerUploadLimitWaitN(ctx context.Context, n int) error { + return stream.ServerUploadLimit.WaitN(ctx, n) +} + +type ReaderWithCtx = stream.ReaderWithCtx + +type ReaderUpdatingProgress = stream.ReaderUpdatingProgress + +type SimpleReaderWithSize = stream.SimpleReaderWithSize diff --git a/internal/errs/device.go b/internal/errs/device.go new file mode 100644 index 00000000000..3b79298a672 --- /dev/null +++ b/internal/errs/device.go @@ -0,0 +1,8 @@ +package errs + +import "errors" + +var ( + TooManyDevices = errors.New("too many active devices") + SessionInactive = errors.New("session inactive") +) diff --git a/internal/errs/driver.go b/internal/errs/driver.go index 4b6b5cac48e..7f67c0e2c2d 100644 --- a/internal/errs/driver.go +++ b/internal/errs/driver.go @@ -4,4 +4,5 @@ import "errors" var ( EmptyToken = errors.New("empty token") + LinkIsDir = errors.New("link is dir") ) diff --git a/internal/errs/errors.go b/internal/errs/errors.go index b48718778a6..2a22dca1e6f 100644 --- a/internal/errs/errors.go +++ b/internal/errs/errors.go @@ -3,6 +3,7 @@ package errs import ( "errors" "fmt" + pkgerr "github.com/pkg/errors" ) @@ -18,6 +19,10 @@ var ( StorageNotFound = errors.New("storage not found") StreamIncomplete = errors.New("upload/download stream incomplete, possible network issue") StreamPeekFail = errors.New("StreamPeekFail") + + UnknownArchiveFormat = errors.New("unknown archive format") + WrongArchivePassword = errors.New("wrong archive password") + DriverExtractNotSupported = errors.New("driver extraction not supported") ) // NewErr wrap constant error with an extra message @@ -29,3 +34,10 @@ func NewErr(err error, format string, a ...any) error { func IsNotFoundError(err error) bool { return errors.Is(pkgerr.Cause(err), ObjectNotFound) || errors.Is(pkgerr.Cause(err), StorageNotFound) } + +func IsNotSupportError(err error) bool { + return errors.Is(pkgerr.Cause(err), NotSupport) +} +func IsNotImplement(err error) bool { + return errors.Is(pkgerr.Cause(err), NotImplement) +} diff --git a/internal/errs/operate.go b/internal/errs/operate.go index 92fbd6a1a49..d2df47ddb9c 100644 --- a/internal/errs/operate.go +++ b/internal/errs/operate.go @@ -4,4 +4,5 @@ import "errors" var ( PermissionDenied = errors.New("permission denied") + InvalidName = errors.New("invalid file name") ) diff --git a/internal/errs/role.go b/internal/errs/role.go new file mode 100644 index 00000000000..a818ea21264 --- /dev/null +++ b/internal/errs/role.go @@ -0,0 +1,7 @@ +package errs + +import "errors" + +var ( + ErrChangeDefaultRole = errors.New("cannot modify admin role") +) diff --git a/internal/errs/search.go b/internal/errs/search.go index 9c864f4d241..3da92898e65 100644 --- a/internal/errs/search.go +++ b/internal/errs/search.go @@ -3,5 +3,6 @@ package errs import "fmt" var ( - SearchNotAvailable = fmt.Errorf("search not available") + SearchNotAvailable = fmt.Errorf("search not available") + BuildIndexIsRunning = fmt.Errorf("build index is running, please try later") ) diff --git a/internal/frp/frp.go b/internal/frp/frp.go new file mode 100644 index 00000000000..c244e18b870 --- /dev/null +++ b/internal/frp/frp.go @@ -0,0 +1,335 @@ +package frp + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/alist-org/alist/v3/cmd/flags" + frpclient "github.com/fatedier/frp/client" + "github.com/fatedier/frp/pkg/config/source" + v1 "github.com/fatedier/frp/pkg/config/v1" + frplog "github.com/fatedier/frp/pkg/util/log" + log "github.com/sirupsen/logrus" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" +) + +// Instance is the global FRP manager. +var Instance *Manager + +// Manager controls the lifecycle of the embedded FRP client. +type Manager struct { + mu sync.Mutex + cancel context.CancelFunc + wg sync.WaitGroup + status string + logs []string + logPath string +} + +const maxLogEntries = 300 +const maxTailBytes = 512 * 1024 + +// RuntimeInfo contains FRP runtime status and recent logs. +type RuntimeInfo struct { + Status string `json:"status"` + Logs []string `json:"logs"` +} + +// Init creates and returns a new Manager. +func Init() *Manager { + m := &Manager{ + status: "stopped", + logPath: filepath.Join(flags.DataDir, "log", "frp.log"), + } + m.logs = append(m.logs, fmt.Sprintf("[%s] initialized", time.Now().Format(time.RFC3339))) + return m +} + +// Start builds the FRP config from settings and starts the client. +func (m *Manager) Start() error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.cancel != nil { + m.appendLogLocked("start skipped: already running") + return nil // already running + } + + cfg, proxyCfgs, err := buildConfig() + if err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("start failed: %s", err.Error()) + return err + } + cfg.Log.To = m.logPath + cfg.Log.Level = "info" + cfg.Log.MaxDays = 7 + if err := os.MkdirAll(filepath.Dir(m.logPath), 0o755); err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("init log dir failed: %s", err.Error()) + return err + } + frplog.InitLogger(cfg.Log.To, cfg.Log.Level, int(cfg.Log.MaxDays), true) + + configSource := source.NewConfigSource() + if err := configSource.ReplaceAll(proxyCfgs, nil); err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("replace config failed: %s", err.Error()) + return err + } + aggregator := source.NewAggregator(configSource) + + svr, err := frpclient.NewService(frpclient.ServiceOptions{ + Common: cfg, + ConfigSourceAggregator: aggregator, + }) + if err != nil { + m.status = "error: " + err.Error() + m.appendLogLocked("create service failed: %s", err.Error()) + return err + } + + ctx, cancel := context.WithCancel(context.Background()) + m.cancel = cancel + m.status = "running" + m.appendLogLocked("service started") + m.wg.Add(1) + + go func() { + defer m.wg.Done() + if err := svr.Run(ctx); err != nil && ctx.Err() == nil { + // Context was not cancelled, so this is an unexpected error. + log.Warnf("frp client stopped unexpectedly: %v", err) + m.mu.Lock() + m.status = "error: " + err.Error() + m.appendLogLocked("service stopped unexpectedly: %s", err.Error()) + m.cancel = nil + m.mu.Unlock() + } + }() + + return nil +} + +// Stop gracefully shuts down the FRP client. +func (m *Manager) Stop() { + m.mu.Lock() + cancel := m.cancel + m.cancel = nil + m.mu.Unlock() + + if cancel != nil { + m.appendLog("stopping service") + cancel() + m.wg.Wait() + } + + m.mu.Lock() + m.status = "stopped" + m.appendLogLocked("service stopped") + m.mu.Unlock() +} + +// Restart stops any running client and starts a fresh one with current settings. +func (m *Manager) Restart() error { + m.appendLog("restarting service") + m.Stop() + return m.Start() +} + +// Status returns the current status string: "running", "stopped", or "error: ". +func (m *Manager) Status() string { + m.mu.Lock() + defer m.mu.Unlock() + return m.status +} + +// Runtime returns status and latest logs. +func (m *Manager) Runtime(limit int) RuntimeInfo { + m.mu.Lock() + status := m.status + logPath := m.logPath + m.mu.Unlock() + + logs, err := readLogTail(logPath, limit) + if err != nil { + m.mu.Lock() + logs = m.copyLogsLocked(limit) + m.mu.Unlock() + } + + return RuntimeInfo{ + Status: status, + Logs: logs, + } +} + +func (m *Manager) appendLog(format string, args ...interface{}) { + m.mu.Lock() + defer m.mu.Unlock() + m.appendLogLocked(format, args...) +} + +func (m *Manager) appendLogLocked(format string, args ...interface{}) { + line := fmt.Sprintf(format, args...) + entry := fmt.Sprintf("[%s] %s", time.Now().Format(time.RFC3339), line) + m.logs = append(m.logs, entry) + if len(m.logs) > maxLogEntries { + m.logs = m.logs[len(m.logs)-maxLogEntries:] + } +} + +func (m *Manager) copyLogsLocked(limit int) []string { + if limit <= 0 || limit > maxLogEntries { + limit = maxLogEntries + } + total := len(m.logs) + if total <= limit { + return append([]string(nil), m.logs...) + } + return append([]string(nil), m.logs[total-limit:]...) +} + +func readLogTail(path string, limit int) ([]string, error) { + if limit <= 0 || limit > maxLogEntries { + limit = maxLogEntries + } + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return nil, err + } + + size := info.Size() + start := int64(0) + if size > maxTailBytes { + start = size - maxTailBytes + } + if _, err = f.Seek(start, io.SeekStart); err != nil { + return nil, err + } + buf, err := io.ReadAll(f) + if err != nil { + return nil, err + } + + if start > 0 { + if idx := bytes.IndexByte(buf, '\n'); idx >= 0 && idx+1 < len(buf) { + buf = buf[idx+1:] + } + } + text := strings.TrimRight(string(buf), "\n") + if text == "" { + return []string{}, nil + } + lines := strings.Split(text, "\n") + if len(lines) <= limit { + return lines, nil + } + return lines[len(lines)-limit:], nil +} + +func buildConfig() (*v1.ClientCommonConfig, []v1.ProxyConfigurer, error) { + serverAddr := setting.GetStr(conf.FRPServerAddr) + if serverAddr == "" { + return nil, nil, fmt.Errorf("frp server address is required") + } + + serverPort := setting.GetInt(conf.FRPServerPort, 7000) + authToken := setting.GetStr(conf.FRPAuthToken) + proxyName := setting.GetStr(conf.FRPProxyName, "alist") + proxyType := setting.GetStr(conf.FRPProxyType, "http") + customDomain := setting.GetStr(conf.FRPCustomDomain) + subdomain := setting.GetStr(conf.FRPSubdomain) + remotePort := setting.GetInt(conf.FRPRemotePort, 0) + localPort := setting.GetInt(conf.FRPLocalPort, 5244) + tlsEnable := setting.GetBool(conf.FRPTLSEnable) + stcpSecretKey := setting.GetStr(conf.FRPSTCPSecretKey) + + cfg := &v1.ClientCommonConfig{ + ServerAddr: serverAddr, + ServerPort: serverPort, + Auth: v1.AuthClientConfig{ + Method: v1.AuthMethodToken, + Token: authToken, + }, + } + if tlsEnable { + enabled := true + cfg.Transport.TLS.Enable = &enabled + } + + backend := v1.ProxyBackend{ + LocalIP: "127.0.0.1", + LocalPort: localPort, + } + + var proxyCfgs []v1.ProxyConfigurer + + switch proxyType { + case "http": + p := &v1.HTTPProxyConfig{} + p.Name = proxyName + p.Type = "http" + p.ProxyBackend = backend + if customDomain != "" { + p.CustomDomains = []string{customDomain} + } + if subdomain != "" { + p.SubDomain = subdomain + } + proxyCfgs = append(proxyCfgs, p) + + case "https": + p := &v1.HTTPSProxyConfig{} + p.Name = proxyName + p.Type = "https" + p.ProxyBackend = backend + if customDomain != "" { + p.CustomDomains = []string{customDomain} + } + if subdomain != "" { + p.SubDomain = subdomain + } + proxyCfgs = append(proxyCfgs, p) + + case "tcp": + if remotePort <= 0 { + return nil, nil, fmt.Errorf("remote_port is required for tcp proxy type") + } + p := &v1.TCPProxyConfig{} + p.Name = proxyName + p.Type = "tcp" + p.ProxyBackend = backend + p.RemotePort = remotePort + proxyCfgs = append(proxyCfgs, p) + + case "stcp": + p := &v1.STCPProxyConfig{} + p.Name = proxyName + p.Type = "stcp" + p.ProxyBackend = backend + p.Secretkey = stcpSecretKey + p.AllowUsers = []string{"*"} + proxyCfgs = append(proxyCfgs, p) + + default: + return nil, nil, fmt.Errorf("unsupported proxy type: %s", proxyType) + } + + return cfg, proxyCfgs, nil +} diff --git a/internal/fs/archive.go b/internal/fs/archive.go new file mode 100644 index 00000000000..dbae9b338de --- /dev/null +++ b/internal/fs/archive.go @@ -0,0 +1,400 @@ +package fs + +import ( + "context" + stderrors "errors" + "fmt" + "io" + "math/rand" + "mime" + "net/http" + "os" + stdpath "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" +) + +type ArchiveDownloadTask struct { + task.TaskExtension + model.ArchiveDecompressArgs + status string + SrcObjPath string + DstDirPath string + srcStorage driver.Driver + dstStorage driver.Driver + SrcStorageMp string + DstStorageMp string +} + +func (t *ArchiveDownloadTask) GetName() string { + return fmt.Sprintf("decompress [%s](%s)[%s] to [%s](%s) with password <%s>", t.SrcStorageMp, t.SrcObjPath, + t.InnerPath, t.DstStorageMp, t.DstDirPath, t.Password) +} + +func (t *ArchiveDownloadTask) GetStatus() string { + return t.status +} + +func (t *ArchiveDownloadTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + uploadTask, err := t.RunWithoutPushUploadTask() + if err != nil { + return err + } + ArchiveContentUploadTaskManager.Add(uploadTask) + return nil +} + +func (t *ArchiveDownloadTask) RunWithoutPushUploadTask() (*ArchiveContentUploadTask, error) { + var err error + if t.srcStorage == nil { + t.srcStorage, err = op.GetStorageByMountPath(t.SrcStorageMp) + } + srcObj, tool, ss, err := op.GetArchiveToolAndStream(t.Ctx(), t.srcStorage, t.SrcObjPath, model.LinkArgs{ + Header: http.Header{}, + }) + if err != nil { + return nil, err + } + defer func() { + var e error + for _, s := range ss { + e = stderrors.Join(e, s.Close()) + } + if e != nil { + log.Errorf("failed to close file streamer, %v", e) + } + }() + var decompressUp model.UpdateProgress + if t.CacheFull { + var total, cur int64 = 0, 0 + for _, s := range ss { + total += s.GetSize() + } + t.SetTotalBytes(total) + t.status = "getting src object" + for _, s := range ss { + if s.GetFile() == nil { + _, err = stream.CacheFullInTempFileAndUpdateProgress(s, func(p float64) { + t.SetProgress((float64(cur) + float64(s.GetSize())*p/100.0) / float64(total)) + }) + } + cur += s.GetSize() + if err != nil { + return nil, err + } + } + t.SetProgress(100.0) + decompressUp = func(_ float64) {} + } else { + decompressUp = t.SetProgress + } + t.status = "walking and decompressing" + dir, err := os.MkdirTemp(conf.Conf.TempDir, "dir-*") + if err != nil { + return nil, err + } + err = tool.Decompress(ss, dir, t.ArchiveInnerArgs, decompressUp) + if err != nil { + return nil, err + } + baseName := strings.TrimSuffix(srcObj.GetName(), stdpath.Ext(srcObj.GetName())) + uploadTask := &ArchiveContentUploadTask{ + TaskExtension: task.TaskExtension{ + Creator: t.GetCreator(), + }, + ObjName: baseName, + InPlace: !t.PutIntoNewDir, + FilePath: dir, + DstDirPath: t.DstDirPath, + dstStorage: t.dstStorage, + DstStorageMp: t.DstStorageMp, + } + return uploadTask, nil +} + +var ArchiveDownloadTaskManager *tache.Manager[*ArchiveDownloadTask] + +type ArchiveContentUploadTask struct { + task.TaskExtension + status string + ObjName string + InPlace bool + FilePath string + DstDirPath string + dstStorage driver.Driver + DstStorageMp string + finalized bool +} + +func (t *ArchiveContentUploadTask) GetName() string { + return fmt.Sprintf("upload %s to [%s](%s)", t.ObjName, t.DstStorageMp, t.DstDirPath) +} + +func (t *ArchiveContentUploadTask) GetStatus() string { + return t.status +} + +func (t *ArchiveContentUploadTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + return t.RunWithNextTaskCallback(func(nextTsk *ArchiveContentUploadTask) error { + ArchiveContentUploadTaskManager.Add(nextTsk) + return nil + }) +} + +func (t *ArchiveContentUploadTask) RunWithNextTaskCallback(f func(nextTsk *ArchiveContentUploadTask) error) error { + var err error + if t.dstStorage == nil { + t.dstStorage, err = op.GetStorageByMountPath(t.DstStorageMp) + } + info, err := os.Stat(t.FilePath) + if err != nil { + return err + } + if info.IsDir() { + t.status = "src object is dir, listing objs" + nextDstPath := t.DstDirPath + if !t.InPlace { + nextDstPath = stdpath.Join(nextDstPath, t.ObjName) + err = op.MakeDir(t.Ctx(), t.dstStorage, nextDstPath) + if err != nil { + return err + } + } + entries, err := os.ReadDir(t.FilePath) + if err != nil { + return err + } + var es error + for _, entry := range entries { + var nextFilePath string + if entry.IsDir() { + nextFilePath, err = moveToTempPath(stdpath.Join(t.FilePath, entry.Name()), "dir-") + } else { + nextFilePath, err = moveToTempPath(stdpath.Join(t.FilePath, entry.Name()), "file-") + } + if err != nil { + es = stderrors.Join(es, err) + continue + } + err = f(&ArchiveContentUploadTask{ + TaskExtension: task.TaskExtension{ + Creator: t.GetCreator(), + }, + ObjName: entry.Name(), + InPlace: false, + FilePath: nextFilePath, + DstDirPath: nextDstPath, + dstStorage: t.dstStorage, + DstStorageMp: t.DstStorageMp, + }) + if err != nil { + es = stderrors.Join(es, err) + } + } + if es != nil { + return es + } + } else { + t.SetTotalBytes(info.Size()) + file, err := os.Open(t.FilePath) + if err != nil { + return err + } + fs := &stream.FileStream{ + Obj: &model.Object{ + Name: t.ObjName, + Size: info.Size(), + Modified: time.Now(), + }, + Mimetype: mime.TypeByExtension(filepath.Ext(t.ObjName)), + WebPutAsTask: true, + Reader: file, + } + fs.Closers.Add(file) + t.status = "uploading" + err = op.Put(t.Ctx(), t.dstStorage, t.DstDirPath, fs, t.SetProgress, true) + if err != nil { + return err + } + } + t.deleteSrcFile() + return nil +} + +func (t *ArchiveContentUploadTask) Cancel() { + t.TaskExtension.Cancel() + if !conf.Conf.Tasks.AllowRetryCanceled { + t.deleteSrcFile() + } +} + +func (t *ArchiveContentUploadTask) deleteSrcFile() { + if !t.finalized { + _ = os.RemoveAll(t.FilePath) + t.finalized = true + } +} + +func moveToTempPath(path, prefix string) (string, error) { + newPath, err := genTempFileName(prefix) + if err != nil { + return "", err + } + err = os.Rename(path, newPath) + if err != nil { + return "", err + } + return newPath, nil +} + +func genTempFileName(prefix string) (string, error) { + retry := 0 + for retry < 10000 { + newPath := stdpath.Join(conf.Conf.TempDir, prefix+strconv.FormatUint(uint64(rand.Uint32()), 10)) + if _, err := os.Stat(newPath); err != nil { + if os.IsNotExist(err) { + return newPath, nil + } else { + return "", err + } + } + retry++ + } + return "", errors.New("failed to generate temp-file name: too many retries") +} + +type archiveContentUploadTaskManagerType struct { + *tache.Manager[*ArchiveContentUploadTask] +} + +func (m *archiveContentUploadTaskManagerType) Remove(id string) { + if t, ok := m.GetByID(id); ok { + t.deleteSrcFile() + m.Manager.Remove(id) + } +} + +func (m *archiveContentUploadTaskManagerType) RemoveAll() { + tasks := m.GetAll() + for _, t := range tasks { + m.Remove(t.GetID()) + } +} + +func (m *archiveContentUploadTaskManagerType) RemoveByState(state ...tache.State) { + tasks := m.GetByState(state...) + for _, t := range tasks { + m.Remove(t.GetID()) + } +} + +func (m *archiveContentUploadTaskManagerType) RemoveByCondition(condition func(task *ArchiveContentUploadTask) bool) { + tasks := m.GetByCondition(condition) + for _, t := range tasks { + m.Remove(t.GetID()) + } +} + +var ArchiveContentUploadTaskManager = &archiveContentUploadTaskManagerType{ + Manager: nil, +} + +func archiveMeta(ctx context.Context, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { + storage, actualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return nil, errors.WithMessage(err, "failed get storage") + } + return op.GetArchiveMeta(ctx, storage, actualPath, args) +} + +func archiveList(ctx context.Context, path string, args model.ArchiveListArgs) ([]model.Obj, error) { + storage, actualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return nil, errors.WithMessage(err, "failed get storage") + } + return op.ListArchive(ctx, storage, actualPath, args) +} + +func archiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) (task.TaskExtensionInfo, error) { + srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) + if err != nil { + return nil, errors.WithMessage(err, "failed get src storage") + } + dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return nil, errors.WithMessage(err, "failed get dst storage") + } + if srcStorage.GetStorage() == dstStorage.GetStorage() { + err = op.ArchiveDecompress(ctx, srcStorage, srcObjActualPath, dstDirActualPath, args, lazyCache...) + if !errors.Is(err, errs.NotImplement) { + return nil, err + } + } + taskCreator, _ := ctx.Value("user").(*model.User) + tsk := &ArchiveDownloadTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + ArchiveDecompressArgs: args, + srcStorage: srcStorage, + dstStorage: dstStorage, + SrcObjPath: srcObjActualPath, + DstDirPath: dstDirActualPath, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, + } + if ctx.Value(conf.NoTaskKey) != nil { + uploadTask, err := tsk.RunWithoutPushUploadTask() + if err != nil { + return nil, errors.WithMessagef(err, "failed download [%s]", srcObjPath) + } + defer uploadTask.deleteSrcFile() + var callback func(t *ArchiveContentUploadTask) error + callback = func(t *ArchiveContentUploadTask) error { + e := t.RunWithNextTaskCallback(callback) + t.deleteSrcFile() + return e + } + return nil, uploadTask.RunWithNextTaskCallback(callback) + } else { + ArchiveDownloadTaskManager.Add(tsk) + return tsk, nil + } +} + +func archiveDriverExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { + storage, actualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed get storage") + } + return op.DriverExtract(ctx, storage, actualPath, args) +} + +func archiveInternalExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + storage, actualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return nil, 0, errors.WithMessage(err, "failed get storage") + } + return op.InternalExtract(ctx, storage, actualPath, args) +} diff --git a/internal/fs/copy.go b/internal/fs/copy.go index c3e387cb227..155e3cf7a87 100644 --- a/internal/fs/copy.go +++ b/internal/fs/copy.go @@ -3,44 +3,83 @@ package fs import ( "context" "fmt" + "github.com/alist-org/alist/v3/internal/errs" "net/http" stdpath "path" - "sync/atomic" + "time" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/pkg/task" + "github.com/alist-org/alist/v3/internal/task" "github.com/alist-org/alist/v3/pkg/utils" "github.com/pkg/errors" - log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" ) -var CopyTaskManager = task.NewTaskManager(3, func(tid *uint64) { - atomic.AddUint64(tid, 1) -}) +type CopyTask struct { + task.TaskExtension + Status string `json:"-"` //don't save status to save space + SrcObjPath string `json:"src_path"` + DstDirPath string `json:"dst_path"` + srcStorage driver.Driver `json:"-"` + dstStorage driver.Driver `json:"-"` + SrcStorageMp string `json:"src_storage_mp"` + DstStorageMp string `json:"dst_storage_mp"` +} + +func (t *CopyTask) GetName() string { + return fmt.Sprintf("copy [%s](%s) to [%s](%s)", t.SrcStorageMp, t.SrcObjPath, t.DstStorageMp, t.DstDirPath) +} + +func (t *CopyTask) GetStatus() string { + return t.Status +} + +func (t *CopyTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + var err error + if t.srcStorage == nil { + t.srcStorage, err = op.GetStorageByMountPath(t.SrcStorageMp) + } + if t.dstStorage == nil { + t.dstStorage, err = op.GetStorageByMountPath(t.DstStorageMp) + } + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + return copyBetween2Storages(t, t.srcStorage, t.dstStorage, t.SrcObjPath, t.DstDirPath) +} + +var CopyTaskManager *tache.Manager[*CopyTask] // Copy if in the same storage, call move method // if not, add copy task -func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (bool, error) { +func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(srcObjPath) if err != nil { - return false, errors.WithMessage(err, "failed get src storage") + return nil, errors.WithMessage(err, "failed get src storage") } dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { - return false, errors.WithMessage(err, "failed get dst storage") + return nil, errors.WithMessage(err, "failed get dst storage") } // copy if in the same storage, just call driver.Copy if srcStorage.GetStorage() == dstStorage.GetStorage() { - return false, op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + err = op.Copy(ctx, srcStorage, srcObjActualPath, dstDirActualPath, lazyCache...) + if !errors.Is(err, errs.NotImplement) && !errors.Is(err, errs.NotSupport) { + return nil, err + } } if ctx.Value(conf.NoTaskKey) != nil { srcObj, err := op.Get(ctx, srcStorage, srcObjActualPath) if err != nil { - return false, errors.WithMessagef(err, "failed get src [%s] file", srcObjPath) + return nil, errors.WithMessagef(err, "failed get src [%s] file", srcObjPath) } if !srcObj.IsDir() { // copy file directly @@ -48,7 +87,7 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool Header: http.Header{}, }) if err != nil { - return false, errors.WithMessagef(err, "failed get [%s] link", srcObjPath) + return nil, errors.WithMessagef(err, "failed get [%s] link", srcObjPath) } fs := stream.FileStream{ Obj: srcObj, @@ -57,65 +96,71 @@ func _copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool // any link provided is seekable ss, err := stream.NewSeekableStream(fs, link) if err != nil { - return false, errors.WithMessagef(err, "failed get [%s] stream", srcObjPath) + return nil, errors.WithMessagef(err, "failed get [%s] stream", srcObjPath) } - return false, op.Put(ctx, dstStorage, dstDirActualPath, ss, nil, false) + return nil, op.Put(ctx, dstStorage, dstDirActualPath, ss, nil, false) } } // not in the same storage - CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjActualPath, dstStorage.GetStorage().MountPath, dstDirActualPath), - Func: func(task *task.Task[uint64]) error { - return copyBetween2Storages(task, srcStorage, dstStorage, srcObjActualPath, dstDirActualPath) + taskCreator, _ := ctx.Value("user").(*model.User) + t := &CopyTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, }, - })) - return true, nil + srcStorage: srcStorage, + dstStorage: dstStorage, + SrcObjPath: srcObjActualPath, + DstDirPath: dstDirActualPath, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, + } + CopyTaskManager.Add(t) + return t, nil } -func copyBetween2Storages(t *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error { - t.SetStatus("getting src object") - srcObj, err := op.Get(t.Ctx, srcStorage, srcObjPath) +func copyBetween2Storages(t *CopyTask, srcStorage, dstStorage driver.Driver, srcObjPath, dstDirPath string) error { + t.Status = "getting src object" + srcObj, err := op.Get(t.Ctx(), srcStorage, srcObjPath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", srcObjPath) } if srcObj.IsDir() { - t.SetStatus("src object is dir, listing objs") - objs, err := op.List(t.Ctx, srcStorage, srcObjPath, model.ListArgs{}) + t.Status = "src object is dir, listing objs" + objs, err := op.List(t.Ctx(), srcStorage, srcObjPath, model.ListArgs{}) if err != nil { return errors.WithMessagef(err, "failed list src [%s] objs", srcObjPath) } for _, obj := range objs { - if utils.IsCanceled(t.Ctx) { + if utils.IsCanceled(t.Ctx()) { return nil } srcObjPath := stdpath.Join(srcObjPath, obj.GetName()) dstObjPath := stdpath.Join(dstDirPath, srcObj.GetName()) - CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjPath, dstStorage.GetStorage().MountPath, dstObjPath), - Func: func(t *task.Task[uint64]) error { - return copyBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstObjPath) + CopyTaskManager.Add(&CopyTask{ + TaskExtension: task.TaskExtension{ + Creator: t.GetCreator(), }, - })) + srcStorage: srcStorage, + dstStorage: dstStorage, + SrcObjPath: srcObjPath, + DstDirPath: dstObjPath, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, + }) } - } else { - CopyTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("copy [%s](%s) to [%s](%s)", srcStorage.GetStorage().MountPath, srcObjPath, dstStorage.GetStorage().MountPath, dstDirPath), - Func: func(t *task.Task[uint64]) error { - err := copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath) - log.Debugf("copy file between storages: %+v", err) - return err - }, - })) + t.Status = "src object is dir, added all copy tasks of objs" + return nil } - return nil + return copyFileBetween2Storages(t, srcStorage, dstStorage, srcObjPath, dstDirPath) } -func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage driver.Driver, srcFilePath, dstDirPath string) error { - srcFile, err := op.Get(tsk.Ctx, srcStorage, srcFilePath) +func copyFileBetween2Storages(tsk *CopyTask, srcStorage, dstStorage driver.Driver, srcFilePath, dstDirPath string) error { + srcFile, err := op.Get(tsk.Ctx(), srcStorage, srcFilePath) if err != nil { return errors.WithMessagef(err, "failed get src [%s] file", srcFilePath) } - link, _, err := op.Link(tsk.Ctx, srcStorage, srcFilePath, model.LinkArgs{ + tsk.SetTotalBytes(srcFile.GetSize()) + link, _, err := op.Link(tsk.Ctx(), srcStorage, srcFilePath, model.LinkArgs{ Header: http.Header{}, }) if err != nil { @@ -123,12 +168,12 @@ func copyFileBetween2Storages(tsk *task.Task[uint64], srcStorage, dstStorage dri } fs := stream.FileStream{ Obj: srcFile, - Ctx: tsk.Ctx, + Ctx: tsk.Ctx(), } // any link provided is seekable ss, err := stream.NewSeekableStream(fs, link) if err != nil { return errors.WithMessagef(err, "failed get [%s] stream", srcFilePath) } - return op.Put(tsk.Ctx, dstStorage, dstDirPath, ss, tsk.SetProgress, true) + return op.Put(tsk.Ctx(), dstStorage, dstDirPath, ss, tsk.SetProgress, true) } diff --git a/internal/fs/fs.go b/internal/fs/fs.go index 2b23142a662..01818e5fd71 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -2,10 +2,15 @@ package fs import ( "context" + log "github.com/sirupsen/logrus" + "io" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" - log "github.com/sirupsen/logrus" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" ) // the param named path of functions in this package is a mount path @@ -68,7 +73,7 @@ func Move(ctx context.Context, srcPath, dstDirPath string, lazyCache ...bool) er return err } -func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (bool, error) { +func Copy(ctx context.Context, srcObjPath, dstDirPath string, lazyCache ...bool) (task.TaskExtensionInfo, error) { res, err := _copy(ctx, srcObjPath, dstDirPath, lazyCache...) if err != nil { log.Errorf("failed copy %s to %s: %+v", srcObjPath, dstDirPath, err) @@ -100,12 +105,52 @@ func PutDirectly(ctx context.Context, dstDirPath string, file model.FileStreamer return err } -func PutAsTask(dstDirPath string, file model.FileStreamer) error { - err := putAsTask(dstDirPath, file) +func PutAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskExtensionInfo, error) { + t, err := putAsTask(ctx, dstDirPath, file) if err != nil { log.Errorf("failed put %s: %+v", dstDirPath, err) } - return err + return t, err +} + +func ArchiveMeta(ctx context.Context, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { + meta, err := archiveMeta(ctx, path, args) + if err != nil { + log.Errorf("failed get archive meta %s: %+v", path, err) + } + return meta, err +} + +func ArchiveList(ctx context.Context, path string, args model.ArchiveListArgs) ([]model.Obj, error) { + objs, err := archiveList(ctx, path, args) + if err != nil { + log.Errorf("failed list archive [%s]%s: %+v", path, args.InnerPath, err) + } + return objs, err +} + +func ArchiveDecompress(ctx context.Context, srcObjPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) (task.TaskExtensionInfo, error) { + t, err := archiveDecompress(ctx, srcObjPath, dstDirPath, args, lazyCache...) + if err != nil { + log.Errorf("failed decompress [%s]%s: %+v", srcObjPath, args.InnerPath, err) + } + return t, err +} + +func ArchiveDriverExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { + l, obj, err := archiveDriverExtract(ctx, path, args) + if err != nil { + log.Errorf("failed extract [%s]%s: %+v", path, args.InnerPath, err) + } + return l, obj, err +} + +func ArchiveInternalExtract(ctx context.Context, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + l, obj, err := archiveInternalExtract(ctx, path, args) + if err != nil { + log.Errorf("failed extract [%s]%s: %+v", path, args.InnerPath, err) + } + return l, obj, err } type GetStoragesArgs struct { @@ -126,3 +171,19 @@ func Other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { } return res, err } + +func PutURL(ctx context.Context, path, dstName, urlStr string) error { + storage, dstDirActualPath, err := op.GetStorageAndActualPath(path) + if err != nil { + return errors.WithMessage(err, "failed get storage") + } + if storage.Config().NoUpload { + return errors.WithStack(errs.UploadNotSupported) + } + _, ok := storage.(driver.PutURL) + _, okResult := storage.(driver.PutURLResult) + if !ok && !okResult { + return errs.NotImplement + } + return op.PutURL(ctx, storage, dstDirActualPath, dstName, urlStr) +} diff --git a/internal/fs/list.go b/internal/fs/list.go index 6e257cea6fa..927b6ead1a4 100644 --- a/internal/fs/list.go +++ b/internal/fs/list.go @@ -6,6 +6,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -24,7 +25,8 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) if storage != nil { _objs, err = op.List(ctx, storage, actualPath, model.ListArgs{ ReqPath: path, - }, args.Refresh) + Refresh: args.Refresh, + }) if err != nil { if !args.NoLog { log.Errorf("fs/list: %+v", err) @@ -44,8 +46,13 @@ func list(ctx context.Context, path string, args *ListArgs) ([]model.Obj, error) } func whetherHide(user *model.User, meta *model.Meta, path string) bool { - // if is admin, don't hide - if user == nil || user.CanSeeHides() { + // if user is nil, don't hide + if user == nil { + return false + } + perm := common.MergeRolePermissions(user, path) + // if user has see-hides permission, don't hide + if common.HasPermission(perm, common.PermSeeHides) { return false } // if meta is nil, don't hide diff --git a/internal/fs/other.go b/internal/fs/other.go index 85b7b1d17bf..14f8f63d3ad 100644 --- a/internal/fs/other.go +++ b/internal/fs/other.go @@ -2,10 +2,15 @@ package fs import ( "context" + "encoding/json" + stdpath "path" + "strings" + "github.com/alist-org/alist/v3/drivers/s3" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" "github.com/pkg/errors" ) @@ -53,6 +58,38 @@ func other(ctx context.Context, args model.FsOtherArgs) (interface{}, error) { if err != nil { return nil, errors.WithMessage(err, "failed get storage") } + originalPath := args.Path + + if _, ok := storage.(*s3.S3); ok { + method := strings.ToLower(strings.TrimSpace(args.Method)) + if method == s3.OtherMethodArchive || method == s3.OtherMethodThaw { + if S3TransitionTaskManager == nil { + return nil, errors.New("s3 transition task manager is not initialized") + } + var payload json.RawMessage + if args.Data != nil { + raw, err := json.Marshal(args.Data) + if err != nil { + return nil, errors.WithMessage(err, "failed to encode request payload") + } + payload = raw + } + taskCreator, _ := ctx.Value("user").(*model.User) + tsk := &S3TransitionTask{ + TaskExtension: task.TaskExtension{Creator: taskCreator}, + status: "queued", + StorageMountPath: storage.GetStorage().MountPath, + ObjectPath: actualPath, + DisplayPath: originalPath, + ObjectName: stdpath.Base(actualPath), + Transition: method, + Payload: payload, + } + S3TransitionTaskManager.Add(tsk) + return map[string]string{"task_id": tsk.GetID()}, nil + } + } + args.Path = actualPath return op.Other(ctx, storage, args) } diff --git a/internal/fs/put.go b/internal/fs/put.go index ab6d24bf571..bc33a3ac102 100644 --- a/internal/fs/put.go +++ b/internal/fs/put.go @@ -3,43 +3,69 @@ package fs import ( "context" "fmt" - "github.com/alist-org/alist/v3/internal/model" - "sync/atomic" - + "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" + "github.com/alist-org/alist/v3/internal/task" "github.com/pkg/errors" + "github.com/xhofe/tache" + "time" ) -var UploadTaskManager = task.NewTaskManager(3, func(tid *uint64) { - atomic.AddUint64(tid, 1) -}) +type UploadTask struct { + task.TaskExtension + storage driver.Driver + dstDirActualPath string + file model.FileStreamer +} + +func (t *UploadTask) GetName() string { + return fmt.Sprintf("upload %s to [%s](%s)", t.file.GetName(), t.storage.GetStorage().MountPath, t.dstDirActualPath) +} + +func (t *UploadTask) GetStatus() string { + return "uploading" +} + +func (t *UploadTask) Run() error { + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + return op.Put(t.Ctx(), t.storage, t.dstDirActualPath, t.file, t.SetProgress, true) +} + +var UploadTaskManager *tache.Manager[*UploadTask] // putAsTask add as a put task and return immediately -func putAsTask(dstDirPath string, file model.FileStreamer) error { +func putAsTask(ctx context.Context, dstDirPath string, file model.FileStreamer) (task.TaskExtensionInfo, error) { storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) if err != nil { - return errors.WithMessage(err, "failed get storage") + return nil, errors.WithMessage(err, "failed get storage") } if storage.Config().NoUpload { - return errors.WithStack(errs.UploadNotSupported) + return nil, errors.WithStack(errs.UploadNotSupported) } if file.NeedStore() { _, err := file.CacheFullInTempFile() if err != nil { - return errors.Wrapf(err, "failed to create temp file") + return nil, errors.Wrapf(err, "failed to create temp file") } //file.SetReader(tempFile) //file.SetTmpFile(tempFile) } - UploadTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("upload %s to [%s](%s)", file.GetName(), storage.GetStorage().MountPath, dstDirActualPath), - Func: func(task *task.Task[uint64]) error { - return op.Put(task.Ctx, storage, dstDirActualPath, file, task.SetProgress, true) + taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed + t := &UploadTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, }, - })) - return nil + storage: storage, + dstDirActualPath: dstDirActualPath, + file: file, + } + t.SetTotalBytes(file.GetSize()) + UploadTaskManager.Add(t) + return t, nil } // putDirect put the file and return after finish diff --git a/internal/fs/s3_transition.go b/internal/fs/s3_transition.go new file mode 100644 index 00000000000..395f3c5be8c --- /dev/null +++ b/internal/fs/s3_transition.go @@ -0,0 +1,310 @@ +package fs + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/alist-org/alist/v3/drivers/s3" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" + "github.com/xhofe/tache" +) + +const s3TransitionPollInterval = 15 * time.Second + +// S3TransitionTask represents an asynchronous S3 archive/thaw request that is +// tracked via the task manager so that clients can monitor the progress of the +// operation. +type S3TransitionTask struct { + task.TaskExtension + status string + + StorageMountPath string `json:"storage_mount_path"` + ObjectPath string `json:"object_path"` + DisplayPath string `json:"display_path"` + ObjectName string `json:"object_name"` + Transition string `json:"transition"` + Payload json.RawMessage `json:"payload,omitempty"` + + TargetStorageClass string `json:"target_storage_class,omitempty"` + RequestID string `json:"request_id,omitempty"` + VersionID string `json:"version_id,omitempty"` + + storage driver.Driver `json:"-"` +} + +// S3TransitionTaskManager holds asynchronous S3 archive/thaw tasks. +var S3TransitionTaskManager *tache.Manager[*S3TransitionTask] + +var _ task.TaskExtensionInfo = (*S3TransitionTask)(nil) + +func (t *S3TransitionTask) GetName() string { + action := strings.ToLower(t.Transition) + if action == "" { + action = "transition" + } + display := t.DisplayPath + if display == "" { + display = t.ObjectPath + } + if display == "" { + display = t.ObjectName + } + return fmt.Sprintf("s3 %s %s", action, display) +} + +func (t *S3TransitionTask) GetStatus() string { + return t.status +} + +func (t *S3TransitionTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + start := time.Now() + t.SetStartTime(start) + defer func() { t.SetEndTime(time.Now()) }() + + if err := t.ensureStorage(); err != nil { + t.status = fmt.Sprintf("locate storage failed: %v", err) + return err + } + + payload, err := t.decodePayload() + if err != nil { + t.status = fmt.Sprintf("decode payload failed: %v", err) + return err + } + + method := strings.ToLower(strings.TrimSpace(t.Transition)) + switch method { + case s3.OtherMethodArchive: + t.status = "submitting archive request" + t.SetProgress(0) + resp, err := op.Other(t.Ctx(), t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodArchive, + Data: payload, + }) + if err != nil { + t.status = fmt.Sprintf("archive request failed: %v", err) + return err + } + archiveResp, ok := toArchiveResponse(resp) + if ok { + if t.TargetStorageClass == "" { + t.TargetStorageClass = archiveResp.StorageClass + } + t.RequestID = archiveResp.RequestID + t.VersionID = archiveResp.VersionID + if archiveResp.StorageClass != "" { + t.status = fmt.Sprintf("archive requested, waiting for %s", archiveResp.StorageClass) + } else { + t.status = "archive requested" + } + } else if sc := t.extractTargetStorageClass(); sc != "" { + t.TargetStorageClass = sc + t.status = fmt.Sprintf("archive requested, waiting for %s", sc) + } else { + t.status = "archive requested" + } + if t.TargetStorageClass != "" { + t.TargetStorageClass = s3.NormalizeStorageClass(t.TargetStorageClass) + } + t.SetProgress(25) + return t.waitForArchive() + case s3.OtherMethodThaw: + t.status = "submitting thaw request" + t.SetProgress(0) + resp, err := op.Other(t.Ctx(), t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodThaw, + Data: payload, + }) + if err != nil { + t.status = fmt.Sprintf("thaw request failed: %v", err) + return err + } + thawResp, ok := toThawResponse(resp) + if ok { + t.RequestID = thawResp.RequestID + if thawResp.Status != nil && !thawResp.Status.Ongoing { + t.SetProgress(100) + t.status = thawCompletionMessage(thawResp.Status) + return nil + } + } + t.status = "thaw requested" + t.SetProgress(25) + return t.waitForThaw() + default: + return errors.Errorf("unsupported transition method: %s", t.Transition) + } +} + +func (t *S3TransitionTask) ensureStorage() error { + if t.storage != nil { + return nil + } + storage, err := op.GetStorageByMountPath(t.StorageMountPath) + if err != nil { + return err + } + t.storage = storage + return nil +} + +func (t *S3TransitionTask) decodePayload() (interface{}, error) { + if len(t.Payload) == 0 { + return nil, nil + } + var payload interface{} + if err := json.Unmarshal(t.Payload, &payload); err != nil { + return nil, err + } + return payload, nil +} + +func (t *S3TransitionTask) extractTargetStorageClass() string { + if len(t.Payload) == 0 { + return "" + } + var req s3.ArchiveRequest + if err := json.Unmarshal(t.Payload, &req); err != nil { + return "" + } + return s3.NormalizeStorageClass(req.StorageClass) +} + +func (t *S3TransitionTask) waitForArchive() error { + ticker := time.NewTicker(s3TransitionPollInterval) + defer ticker.Stop() + + ctx := t.Ctx() + for { + select { + case <-ctx.Done(): + t.status = "archive canceled" + return ctx.Err() + case <-ticker.C: + resp, err := op.Other(ctx, t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodArchiveStatus, + }) + if err != nil { + t.status = fmt.Sprintf("archive status error: %v", err) + return err + } + archiveResp, ok := toArchiveResponse(resp) + if !ok { + t.status = fmt.Sprintf("unexpected archive status response: %T", resp) + return errors.Errorf("unexpected archive status response: %T", resp) + } + currentClass := strings.TrimSpace(archiveResp.StorageClass) + target := strings.TrimSpace(t.TargetStorageClass) + if target == "" { + target = currentClass + t.TargetStorageClass = currentClass + } + if currentClass == "" { + t.status = "waiting for storage class update" + t.SetProgress(50) + continue + } + if strings.EqualFold(currentClass, target) { + t.SetProgress(100) + t.status = fmt.Sprintf("archive complete (%s)", currentClass) + return nil + } + t.status = fmt.Sprintf("storage class %s (target %s)", currentClass, target) + t.SetProgress(75) + } + } +} + +func (t *S3TransitionTask) waitForThaw() error { + ticker := time.NewTicker(s3TransitionPollInterval) + defer ticker.Stop() + + ctx := t.Ctx() + for { + select { + case <-ctx.Done(): + t.status = "thaw canceled" + return ctx.Err() + case <-ticker.C: + resp, err := op.Other(ctx, t.storage, model.FsOtherArgs{ + Path: t.ObjectPath, + Method: s3.OtherMethodThawStatus, + }) + if err != nil { + t.status = fmt.Sprintf("thaw status error: %v", err) + return err + } + thawResp, ok := toThawResponse(resp) + if !ok { + t.status = fmt.Sprintf("unexpected thaw status response: %T", resp) + return errors.Errorf("unexpected thaw status response: %T", resp) + } + status := thawResp.Status + if status == nil { + t.status = "waiting for thaw status" + t.SetProgress(50) + continue + } + if status.Ongoing { + t.status = fmt.Sprintf("thaw in progress (%s)", status.Raw) + t.SetProgress(75) + continue + } + t.SetProgress(100) + t.status = thawCompletionMessage(status) + return nil + } + } +} + +func thawCompletionMessage(status *s3.RestoreStatus) string { + if status == nil { + return "thaw complete" + } + if status.Expiry != "" { + return fmt.Sprintf("thaw complete, expires %s", status.Expiry) + } + return "thaw complete" +} + +func toArchiveResponse(v interface{}) (s3.ArchiveResponse, bool) { + switch resp := v.(type) { + case s3.ArchiveResponse: + return resp, true + case *s3.ArchiveResponse: + if resp != nil { + return *resp, true + } + } + return s3.ArchiveResponse{}, false +} + +func toThawResponse(v interface{}) (s3.ThawResponse, bool) { + switch resp := v.(type) { + case s3.ThawResponse: + return resp, true + case *s3.ThawResponse: + if resp != nil { + return *resp, true + } + } + return s3.ThawResponse{}, false +} + +// Ensure compatibility with persistence when tasks are restored. +func (t *S3TransitionTask) OnRestore() { + // The storage handle is not persisted intentionally; it will be lazily + // re-fetched on the next Run invocation. + t.storage = nil +} diff --git a/internal/model/archive.go b/internal/model/archive.go new file mode 100644 index 00000000000..01b83691e3f --- /dev/null +++ b/internal/model/archive.go @@ -0,0 +1,53 @@ +package model + +import "time" + +type ObjTree interface { + Obj + GetChildren() []ObjTree +} + +type ObjectTree struct { + Object + Children []ObjTree +} + +func (t *ObjectTree) GetChildren() []ObjTree { + return t.Children +} + +type ArchiveMeta interface { + GetComment() string + // IsEncrypted means if the content of the archive requires a password to access + // GetArchiveMeta should return errs.WrongArchivePassword if the meta-info is also encrypted, + // and the provided password is empty. + IsEncrypted() bool + // GetTree directly returns the full folder structure + // returns nil if the folder structure should be acquired by calling driver.ArchiveReader.ListArchive + GetTree() []ObjTree +} + +type ArchiveMetaInfo struct { + Comment string + Encrypted bool + Tree []ObjTree +} + +func (m *ArchiveMetaInfo) GetComment() string { + return m.Comment +} + +func (m *ArchiveMetaInfo) IsEncrypted() bool { + return m.Encrypted +} + +func (m *ArchiveMetaInfo) GetTree() []ObjTree { + return m.Tree +} + +type ArchiveMetaProvider struct { + ArchiveMeta + *Sort + DriverProviding bool + Expiration *time.Duration +} diff --git a/internal/model/args.go b/internal/model/args.go index ac3c1875bfa..f29c7e45ee9 100644 --- a/internal/model/args.go +++ b/internal/model/args.go @@ -13,13 +13,15 @@ import ( type ListArgs struct { ReqPath string S3ShowPlaceholder bool + Refresh bool } type LinkArgs struct { - IP string - Header http.Header - Type string - HttpReq *http.Request + IP string + Header http.Header + Type string + HttpReq *http.Request + Redirect bool } type Link struct { @@ -47,6 +49,33 @@ type FsOtherArgs struct { Method string `json:"method" form:"method"` Data interface{} `json:"data" form:"data"` } + +type ArchiveArgs struct { + Password string + LinkArgs +} + +type ArchiveInnerArgs struct { + ArchiveArgs + InnerPath string +} + +type ArchiveMetaArgs struct { + ArchiveArgs + Refresh bool +} + +type ArchiveListArgs struct { + ArchiveInnerArgs + Refresh bool +} + +type ArchiveDecompressArgs struct { + ArchiveInnerArgs + CacheFull bool + PutIntoNewDir bool +} + type RangeReadCloserIF interface { RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) utils.ClosersIF @@ -59,7 +88,7 @@ type RangeReadCloser struct { utils.Closers } -func (r RangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { +func (r *RangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { rc, err := r.RangeReader(ctx, httpRange) r.Closers.Add(rc) return rc, err diff --git a/internal/model/label.go b/internal/model/label.go new file mode 100644 index 00000000000..b397542f5c3 --- /dev/null +++ b/internal/model/label.go @@ -0,0 +1,12 @@ +package model + +import "time" + +type Label struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + Type int `json:"type"` // use to type + Name string `json:"name"` // use to name + Description string `json:"description"` // use to description + BgColor string `json:"bg_color"` // use to bg_color + CreateTime time.Time `json:"create_time"` +} diff --git a/internal/model/label_file_binding.go b/internal/model/label_file_binding.go new file mode 100644 index 00000000000..af57fed4d88 --- /dev/null +++ b/internal/model/label_file_binding.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type LabelFileBinding struct { + ID uint `json:"id" gorm:"primaryKey"` // unique key + UserId uint `json:"user_id"` // use to user_id + LabelId uint `json:"label_id"` // use to label_id + FileName string `json:"file_name"` // use to file_name + CreateTime time.Time `json:"create_time"` +} diff --git a/internal/model/obj.go b/internal/model/obj.go index 77c0700a35c..ed4e0451ec7 100644 --- a/internal/model/obj.go +++ b/internal/model/obj.go @@ -2,13 +2,14 @@ package model import ( "io" - "regexp" + "os" "sort" "strings" "time" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/dlclark/regexp2" mapset "github.com/deckarep/golang-set/v2" @@ -19,6 +20,10 @@ type ObjUnwrap interface { Unwrap() Obj } +type StorageClassProvider interface { + StorageClass() string +} + type Obj interface { GetSize() int64 GetName() string @@ -41,12 +46,32 @@ type FileStreamer interface { GetMimetype() string //SetReader(io.Reader) NeedStore() bool + IsForceStreamUpload() bool GetExist() Obj SetExist(Obj) //for a non-seekable Stream, RangeRead supports peeking some data, and CacheFullInTempFile still works RangeRead(http_range.Range) (io.Reader, error) //for a non-seekable Stream, if Read is called, this function won't work CacheFullInTempFile() (File, error) + SetTmpFile(r *os.File) + GetFile() File +} + +type UpdateProgress func(percentage float64) + +// Reference implementation from OpenListTeam: +// https://github.com/OpenListTeam/OpenList/blob/a703b736c9346c483bae56905a39bc07bf781cff/internal/model/obj.go#L58 +func UpdateProgressWithRange(inner UpdateProgress, start, end float64) UpdateProgress { + return func(p float64) { + if p < 0 { + p = 0 + } + if p > 100 { + p = 100 + } + scaled := start + (end-start)*(p/100.0) + inner(scaled) + } } type URL interface { @@ -111,15 +136,22 @@ func ExtractFolder(objs []Obj, extractFolder string) { } func WrapObjName(objs Obj) Obj { - return &ObjWrapName{Obj: objs} + return &ObjWrapName{Name: utils.MappingName(objs.GetName()), Obj: objs} } func WrapObjsName(objs []Obj) { for i := 0; i < len(objs); i++ { - objs[i] = &ObjWrapName{Obj: objs[i]} + objs[i] = &ObjWrapName{Name: utils.MappingName(objs[i].GetName()), Obj: objs[i]} } } +func WrapObjStorageClass(obj Obj, storageClass string) Obj { + if storageClass == "" { + return obj + } + return &ObjWrapStorageClass{Obj: obj, storageClass: storageClass} +} + func UnwrapObj(obj Obj) Obj { if unwrap, ok := obj.(ObjUnwrap); ok { obj = unwrap.Unwrap() @@ -147,6 +179,20 @@ func GetUrl(obj Obj) (url string, ok bool) { return url, false } +func GetStorageClass(obj Obj) (string, bool) { + if provider, ok := obj.(StorageClassProvider); ok { + value := provider.StorageClass() + if value == "" { + return "", false + } + return value, true + } + if unwrap, ok := obj.(ObjUnwrap); ok { + return GetStorageClass(unwrap.Unwrap()) + } + return "", false +} + func GetRawObject(obj Obj) *Object { switch v := obj.(type) { case *ObjThumbURL: @@ -169,7 +215,7 @@ func NewObjMerge() *ObjMerge { } type ObjMerge struct { - regs []*regexp.Regexp + regs []*regexp2.Regexp set mapset.Set[string] } @@ -190,7 +236,7 @@ func (om *ObjMerge) insertObjs(objs []Obj, objs_ ...Obj) []Obj { func (om *ObjMerge) clickObj(obj Obj) bool { for _, reg := range om.regs { - if reg.MatchString(obj.GetName()) { + if isMatch, _ := reg.MatchString(obj.GetName()); isMatch { return false } } @@ -199,9 +245,9 @@ func (om *ObjMerge) clickObj(obj Obj) bool { func (om *ObjMerge) InitHideReg(hides string) { rs := strings.Split(hides, "\n") - om.regs = make([]*regexp.Regexp, 0, len(rs)) + om.regs = make([]*regexp2.Regexp, 0, len(rs)) for _, r := range rs { - om.regs = append(om.regs, regexp.MustCompile(r)) + om.regs = append(om.regs, regexp2.MustCompile(r, regexp2.None)) } } diff --git a/internal/model/obj_file.go b/internal/model/obj_file.go new file mode 100644 index 00000000000..0fccd6b5cba --- /dev/null +++ b/internal/model/obj_file.go @@ -0,0 +1,18 @@ +package model + +import "time" + +type ObjFile struct { + Id string `json:"id"` + UserId uint `json:"user_id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` +} diff --git a/internal/model/object.go b/internal/model/object.go index 93f2c307a03..1617662cf9b 100644 --- a/internal/model/object.go +++ b/internal/model/object.go @@ -11,17 +11,33 @@ type ObjWrapName struct { Obj } +type ObjWrapStorageClass struct { + storageClass string + Obj +} + func (o *ObjWrapName) Unwrap() Obj { return o.Obj } func (o *ObjWrapName) GetName() string { - if o.Name == "" { - o.Name = utils.MappingName(o.Obj.GetName()) - } return o.Name } +func (o *ObjWrapStorageClass) Unwrap() Obj { + return o.Obj +} + +func (o *ObjWrapStorageClass) StorageClass() string { + return o.storageClass +} + +func (o *ObjWrapStorageClass) SetPath(path string) { + if setter, ok := o.Obj.(SetPath); ok { + setter.SetPath(path) + } +} + type Object struct { ID string Path string diff --git a/internal/model/paths.go b/internal/model/paths.go new file mode 100644 index 00000000000..8403de8e6a2 --- /dev/null +++ b/internal/model/paths.go @@ -0,0 +1,27 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Paths []string + +func (p Paths) Value() (driver.Value, error) { + return json.Marshal([]string(p)) +} + +func (p *Paths) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, (*[]string)(p)) + case string: + return json.Unmarshal([]byte(v), (*[]string)(p)) + case nil: + *p = nil + return nil + default: + return fmt.Errorf("cannot scan %T", value) + } +} diff --git a/internal/model/role.go b/internal/model/role.go new file mode 100644 index 00000000000..87855551ddb --- /dev/null +++ b/internal/model/role.go @@ -0,0 +1,53 @@ +package model + +import ( + "encoding/json" + + "gorm.io/gorm" +) + +// PermissionEntry defines permission bitmask for a specific path. +type PermissionEntry struct { + Path string `json:"path"` // path prefix, e.g. "/admin" + Permission int32 `json:"permission"` // bitmask permissions +} + +// Role represents a permission template which can be bound to users. +type Role struct { + ID uint `json:"id" gorm:"primaryKey"` + Name string `json:"name" gorm:"unique" binding:"required"` + Description string `json:"description"` + Default bool `json:"default" gorm:"default:false"` + // PermissionScopes stores structured permission list and is ignored by gorm. + PermissionScopes []PermissionEntry `json:"permission_scopes" gorm:"-"` + // RawPermission is the JSON representation of PermissionScopes stored in DB. + RawPermission string `json:"-" gorm:"type:text"` +} + +// BeforeSave GORM hook serializes PermissionScopes into RawPermission. +func (r *Role) BeforeSave(tx *gorm.DB) error { + if len(r.PermissionScopes) == 0 { + r.RawPermission = "" + return nil + } + bs, err := json.Marshal(r.PermissionScopes) + if err != nil { + return err + } + r.RawPermission = string(bs) + return nil +} + +// AfterFind GORM hook deserializes RawPermission into PermissionScopes. +func (r *Role) AfterFind(tx *gorm.DB) error { + if r.RawPermission == "" { + r.PermissionScopes = nil + return nil + } + var scopes []PermissionEntry + if err := json.Unmarshal([]byte(r.RawPermission), &scopes); err != nil { + return err + } + r.PermissionScopes = scopes + return nil +} diff --git a/internal/model/roles.go b/internal/model/roles.go new file mode 100644 index 00000000000..eb626cb93f7 --- /dev/null +++ b/internal/model/roles.go @@ -0,0 +1,36 @@ +package model + +import ( + "database/sql/driver" + "encoding/json" + "fmt" +) + +type Roles []int + +func (r Roles) Value() (driver.Value, error) { + return json.Marshal([]int(r)) +} + +func (r *Roles) Scan(value interface{}) error { + switch v := value.(type) { + case []byte: + return json.Unmarshal(v, (*[]int)(r)) + case string: + return json.Unmarshal([]byte(v), (*[]int)(r)) + case nil: + *r = nil + return nil + default: + return fmt.Errorf("cannot scan %T", value) + } +} + +func (r Roles) Contains(role int) bool { + for _, v := range r { + if v == role { + return true + } + } + return false +} diff --git a/internal/model/session.go b/internal/model/session.go new file mode 100644 index 00000000000..3cb6d0dab90 --- /dev/null +++ b/internal/model/session.go @@ -0,0 +1,16 @@ +package model + +// Session represents a device session of a user. +type Session struct { + UserID uint `json:"user_id" gorm:"index"` + DeviceKey string `json:"device_key" gorm:"primaryKey;size:64"` + UserAgent string `json:"user_agent" gorm:"size:255"` + IP string `json:"ip" gorm:"size:64"` + LastActive int64 `json:"last_active"` + Status int `json:"status"` +} + +const ( + SessionActive = iota + SessionInactive +) diff --git a/internal/model/setting.go b/internal/model/setting.go index f4202ee022c..9e23f9509e6 100644 --- a/internal/model/setting.go +++ b/internal/model/setting.go @@ -6,9 +6,14 @@ const ( STYLE PREVIEW GLOBAL - ARIA2 + OFFLINE_DOWNLOAD INDEX SSO + LDAP + S3 + FTP + TRAFFIC + FRP ) const ( @@ -19,13 +24,15 @@ const ( ) type SettingItem struct { - Key string `json:"key" gorm:"primaryKey" binding:"required"` // unique key - Value string `json:"value"` // value - Help string `json:"help"` // help message - Type string `json:"type"` // string, number, bool, select - Options string `json:"options"` // values for select - Group int `json:"group"` // use to group setting in frontend - Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. + Key string `json:"key" gorm:"primaryKey" binding:"required"` // unique key + Value string `json:"value"` // value + PreDefault string `json:"-" gorm:"-:all"` // deprecated value + Help string `json:"help"` // help message + Type string `json:"type"` // string, number, bool, select + Options string `json:"options"` // values for select + Group int `json:"group"` // use to group setting in frontend + Flag int `json:"flag"` // 0 = public, 1 = private, 2 = readonly, 3 = deprecated, etc. + Index uint `json:"index"` } func (s SettingItem) IsDeprecated() bool { diff --git a/internal/model/share.go b/internal/model/share.go new file mode 100644 index 00000000000..30fbb8cb7d4 --- /dev/null +++ b/internal/model/share.go @@ -0,0 +1,62 @@ +package model + +import "time" + +type Share struct { + ID uint `json:"id" gorm:"primaryKey"` + ShareID string `json:"share_id" gorm:"uniqueIndex;size:32;not null"` + CreatorID uint `json:"creator_id" gorm:"index;not null"` + Name string `json:"name" gorm:"size:255;not null"` + RootPath string `json:"root_path" gorm:"size:4096;not null"` + IsDir bool `json:"is_dir"` + PasswordHash string `json:"-" gorm:"size:64"` + PasswordSalt string `json:"-" gorm:"size:32"` + BurnAfterRead bool `json:"burn_after_read" gorm:"default:false"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + AllowPreview bool `json:"allow_preview" gorm:"default:true"` + AllowDownload bool `json:"allow_download" gorm:"default:true"` + Enabled bool `json:"enabled" gorm:"default:true;index"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +func (s Share) HasPassword() bool { + return s.PasswordHash != "" +} + +func (s Share) EffectiveAccessLimit() int64 { + if s.AccessLimit > 0 { + return s.AccessLimit + } + if s.BurnAfterRead { + return 1 + } + return 0 +} + +func (s Share) RemainingAccesses() int64 { + limit := s.EffectiveAccessLimit() + if limit <= 0 { + return 0 + } + remaining := limit - s.AccessCount + if remaining < 0 { + return 0 + } + return remaining +} + +func (s Share) IsConsumed() bool { + limit := s.EffectiveAccessLimit() + return s.ConsumedAt != nil || (limit > 0 && s.AccessCount >= limit) +} + +func (s Share) IsExpired(now time.Time) bool { + return s.ExpiresAt != nil && !s.ExpiresAt.After(now) +} diff --git a/internal/model/sshkey.go b/internal/model/sshkey.go new file mode 100644 index 00000000000..6e97c103017 --- /dev/null +++ b/internal/model/sshkey.go @@ -0,0 +1,28 @@ +package model + +import ( + "golang.org/x/crypto/ssh" + "time" +) + +type SSHPublicKey struct { + ID uint `json:"id" gorm:"primaryKey"` + UserId uint `json:"-"` + Title string `json:"title"` + Fingerprint string `json:"fingerprint"` + KeyStr string `gorm:"type:text" json:"-"` + AddedTime time.Time `json:"added_time"` + LastUsedTime time.Time `json:"last_used_time"` +} + +func (k *SSHPublicKey) GetKey() (ssh.PublicKey, error) { + pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k.KeyStr)) + if err != nil { + return nil, err + } + return pubKey, nil +} + +func (k *SSHPublicKey) UpdateLastUsedTime() { + k.LastUsedTime = time.Now() +} diff --git a/internal/model/storage.go b/internal/model/storage.go index 1045a00767b..4d9c062518d 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -1,6 +1,8 @@ package model -import "time" +import ( + "time" +) type Storage struct { ID uint `json:"id" gorm:"primaryKey"` // unique key @@ -13,6 +15,7 @@ type Storage struct { Remark string `json:"remark"` Modified time.Time `json:"modified"` Disabled bool `json:"disabled"` // if disabled + DisableIndex bool `json:"disable_index"` EnableSign bool `json:"enable_sign"` Sort Proxy @@ -25,9 +28,11 @@ type Sort struct { } type Proxy struct { - WebProxy bool `json:"web_proxy"` - WebdavPolicy string `json:"webdav_policy"` - DownProxyUrl string `json:"down_proxy_url"` + WebProxy bool `json:"web_proxy"` + WebdavPolicy string `json:"webdav_policy"` + ProxyRange bool `json:"proxy_range"` + DownProxyUrl string `json:"down_proxy_url"` + DownProxySign bool `json:"down_proxy_sign" gorm:"default:true"` } func (s *Storage) GetStorage() *Storage { diff --git a/internal/model/task.go b/internal/model/task.go new file mode 100644 index 00000000000..8a87c5a5062 --- /dev/null +++ b/internal/model/task.go @@ -0,0 +1,6 @@ +package model + +type TaskItem struct { + Key string `json:"key"` + PersistData string `gorm:"type:text" json:"persist_data"` +} diff --git a/internal/model/user.go b/internal/model/user.go index d7b2863cebe..f55b6a5a2a2 100644 --- a/internal/model/user.go +++ b/internal/model/user.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "encoding/json" "fmt" + "time" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/pkg/utils" @@ -16,31 +17,40 @@ const ( GENERAL = iota GUEST // only one exists ADMIN + NEWGENERAL ) const StaticHashSalt = "https://github.com/alist-org/alist" type User struct { - ID uint `json:"id" gorm:"primaryKey"` // unique key - Username string `json:"username" gorm:"unique" binding:"required"` // username - PwdHash string `json:"-"` // password hash - Salt string `json:"-"` // unique salt - Password string `json:"password"` // password - BasePath string `json:"base_path"` // base path - Role int `json:"role"` // user's role - Disabled bool `json:"disabled"` + ID uint `json:"id" gorm:"primaryKey"` // unique key + Username string `json:"username" gorm:"unique" binding:"required"` // username + PwdHash string `json:"-"` // password hash + PwdTS int64 `json:"-"` // password timestamp + Salt string `json:"-"` // unique salt + Password string `json:"password"` // password + BasePath string `json:"base_path"` // base path + Role Roles `json:"role" gorm:"type:text"` // user's roles + RolesDetail []Role `json:"-" gorm:"-"` + Disabled bool `json:"disabled"` // Determine permissions by bit - // 0: can see hidden files - // 1: can access without password - // 2: can add aria2 tasks - // 3: can mkdir and upload - // 4: can rename - // 5: can move - // 6: can copy - // 7: can remove - // 8: webdav read - // 9: webdav write - // 10: can add qbittorrent tasks + // 0: can see hidden files + // 1: can access without password + // 2: can add offline download tasks + // 3: can mkdir and upload + // 4: can rename + // 5: can move + // 6: can copy + // 7: can remove + // 8: webdav read + // 9: webdav write + // 10: ftp/sftp login and read + // 11: ftp/sftp write + // 12: can read archives + // 13: can decompress archives + // 14: check path limit + // 15: mcp read + // 16: mcp write Permission int32 `json:"permission"` OtpSecret string `json:"-"` SsoID string `json:"sso_id"` // unique by sso platform @@ -48,11 +58,11 @@ type User struct { } func (u *User) IsGuest() bool { - return u.Role == GUEST + return u.Role.Contains(GUEST) } func (u *User) IsAdmin() bool { - return u.Role == ADMIN + return u.Role.Contains(ADMIN) } func (u *User) ValidateRawPassword(password string) error { @@ -72,55 +82,102 @@ func (u *User) ValidatePwdStaticHash(pwdStaticHash string) error { func (u *User) SetPassword(pwd string) *User { u.Salt = random.String(16) u.PwdHash = TwoHashPwd(pwd, u.Salt) + u.PwdTS = time.Now().Unix() return u } func (u *User) CanSeeHides() bool { - return u.IsAdmin() || u.Permission&1 == 1 + return u.Permission&1 == 1 } func (u *User) CanAccessWithoutPassword() bool { - return u.IsAdmin() || (u.Permission>>1)&1 == 1 + return (u.Permission>>1)&1 == 1 } -func (u *User) CanAddAria2Tasks() bool { - return u.IsAdmin() || (u.Permission>>2)&1 == 1 +func (u *User) CanAddOfflineDownloadTasks() bool { + return (u.Permission>>2)&1 == 1 } func (u *User) CanWrite() bool { - return u.IsAdmin() || (u.Permission>>3)&1 == 1 + return (u.Permission>>3)&1 == 1 } func (u *User) CanRename() bool { - return u.IsAdmin() || (u.Permission>>4)&1 == 1 + return (u.Permission>>4)&1 == 1 } func (u *User) CanMove() bool { - return u.IsAdmin() || (u.Permission>>5)&1 == 1 + return (u.Permission>>5)&1 == 1 } func (u *User) CanCopy() bool { - return u.IsAdmin() || (u.Permission>>6)&1 == 1 + return (u.Permission>>6)&1 == 1 } func (u *User) CanRemove() bool { - return u.IsAdmin() || (u.Permission>>7)&1 == 1 + return (u.Permission>>7)&1 == 1 } func (u *User) CanWebdavRead() bool { - return u.IsAdmin() || (u.Permission>>8)&1 == 1 + return (u.Permission>>8)&1 == 1 } func (u *User) CanWebdavManage() bool { - return u.IsAdmin() || (u.Permission>>9)&1 == 1 + return (u.Permission>>9)&1 == 1 } -func (u *User) CanAddQbittorrentTasks() bool { - return u.IsAdmin() || (u.Permission>>10)&1 == 1 +func (u *User) CanFTPAccess() bool { + return (u.Permission>>10)&1 == 1 +} + +func (u *User) CanFTPManage() bool { + return (u.Permission>>11)&1 == 1 +} + +func (u *User) CanReadArchives() bool { + return (u.Permission>>12)&1 == 1 +} + +func (u *User) CanDecompress() bool { + return (u.Permission>>13)&1 == 1 +} + +func (u *User) CheckPathLimit() bool { + return (u.Permission>>14)&1 == 1 +} + +func (u *User) CanMCPAccess() bool { + return (u.Permission>>15)&1 == 1 +} + +func (u *User) CanMCPManage() bool { + return (u.Permission>>16)&1 == 1 } func (u *User) JoinPath(reqPath string) (string, error) { - return utils.JoinBasePath(u.BasePath, reqPath) + if reqPath == "/" { + return utils.FixAndCleanPath(u.BasePath), nil + } + path, err := utils.JoinBasePath(u.BasePath, reqPath) + if err != nil { + return "", err + } + + if path != "/" && u.CheckPathLimit() { + basePaths := GetAllBasePathsFromRoles(u) + match := false + for _, base := range basePaths { + if utils.IsSubPath(base, path) { + match = true + break + } + } + if !match { + return "", errs.PermissionDenied + } + } + + return path, nil } func StaticHash(password string) string { @@ -159,5 +216,35 @@ func (u *User) WebAuthnCredentials() []webauthn.Credential { } func (u *User) WebAuthnIcon() string { - return "https://alist.nn.ci/logo.svg" + return "https://alistgo.com/logo.svg" +} + +// FetchRole is used to load role details by id. It should be set by the op package +// to avoid an import cycle between model and op. +var FetchRole func(uint) (*Role, error) + +// GetAllBasePathsFromRoles returns all permission paths from user's roles +func GetAllBasePathsFromRoles(u *User) []string { + basePaths := make([]string, 0) + seen := make(map[string]struct{}) + + for _, rid := range u.Role { + if FetchRole == nil { + continue + } + role, err := FetchRole(uint(rid)) + if err != nil || role == nil { + continue + } + for _, entry := range role.PermissionScopes { + if entry.Path == "" { + continue + } + if _, ok := seen[entry.Path]; !ok { + basePaths = append(basePaths, entry.Path) + seen[entry.Path] = struct{}{} + } + } + } + return basePaths } diff --git a/internal/net/buf_race_test.go b/internal/net/buf_race_test.go new file mode 100644 index 00000000000..f8b614424c5 --- /dev/null +++ b/internal/net/buf_race_test.go @@ -0,0 +1,44 @@ +package net + +import ( + "context" + "sync" + "testing" +) + +// TestBufCloseWriteRace exercises the race fixed for issue #9537/#9190: +// Buf.Close() must synchronize with concurrent Buf.Write() calls so that +// nilling the underlying bytes.Buffer cannot panic an in-flight writer. +// Run with `go test -race ./internal/net/...` to be meaningful. +func TestBufCloseWriteRace(t *testing.T) { + const iters = 200 + const writers = 8 + + for i := 0; i < iters; i++ { + buf := NewBuf(context.Background(), 1024) + var wg sync.WaitGroup + for w := 0; w < writers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + t.Errorf("panic in Write: %v", r) + } + }() + for j := 0; j < 50; j++ { + _, _ = buf.Write([]byte("x")) + } + }() + } + go func() { + defer func() { + if r := recover(); r != nil { + t.Errorf("panic in Close: %v", r) + } + }() + buf.Close() + }() + wg.Wait() + } +} diff --git a/internal/net/request.go b/internal/net/request.go index b450ede5a02..f04747435db 100644 --- a/internal/net/request.go +++ b/internal/net/request.go @@ -5,13 +5,14 @@ import ( "context" "fmt" "io" - "math" "net/http" "strconv" "strings" "sync" "time" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/http_range" "github.com/aws/aws-sdk-go/aws/awsutil" log "github.com/sirupsen/logrus" @@ -19,7 +20,7 @@ import ( // DefaultDownloadPartSize is the default range of bytes to get at a time when // using Download(). -const DefaultDownloadPartSize = 1024 * 1024 * 10 +const DefaultDownloadPartSize = utils.MB * 10 // DefaultDownloadConcurrency is the default number of goroutines to spin up // when using Download(). @@ -28,6 +29,8 @@ const DefaultDownloadConcurrency = 2 // DefaultPartBodyMaxRetries is the default number of retries to make when a part fails to download. const DefaultPartBodyMaxRetries = 3 +var DefaultConcurrencyLimit *ConcurrencyLimit + type Downloader struct { PartSize int @@ -42,15 +45,15 @@ type Downloader struct { //RequestParam HttpRequestParams HttpClient HttpRequestFunc + + *ConcurrencyLimit } type HttpRequestFunc func(ctx context.Context, params *HttpRequestParams) (*http.Response, error) func NewDownloader(options ...func(*Downloader)) *Downloader { - d := &Downloader{ - HttpClient: DefaultHttpRequestFunc, - PartSize: DefaultDownloadPartSize, + d := &Downloader{ //允许不设置的选项 PartBodyMaxRetries: DefaultPartBodyMaxRetries, - Concurrency: DefaultDownloadConcurrency, + ConcurrencyLimit: DefaultConcurrencyLimit, } for _, option := range options { option(d) @@ -72,16 +75,16 @@ func (d Downloader) Download(ctx context.Context, p *HttpRequestParams) (readClo impl := downloader{params: &finalP, cfg: d, ctx: ctx} // Ensures we don't need nil checks later on - - impl.partBodyMaxRetries = d.PartBodyMaxRetries - + // 必需的选项 if impl.cfg.Concurrency == 0 { impl.cfg.Concurrency = DefaultDownloadConcurrency } - if impl.cfg.PartSize == 0 { impl.cfg.PartSize = DefaultDownloadPartSize } + if impl.cfg.HttpClient == nil { + impl.cfg.HttpClient = DefaultHttpRequestFunc + } return impl.download() } @@ -89,7 +92,7 @@ func (d Downloader) Download(ctx context.Context, p *HttpRequestParams) (readClo // downloader is the implementation structure used internally by Downloader. type downloader struct { ctx context.Context - cancel context.CancelFunc + cancel context.CancelCauseFunc cfg Downloader params *HttpRequestParams //http request params @@ -99,38 +102,78 @@ type downloader struct { m sync.Mutex nextChunk int //next chunk id - chunks []chunk bufs []*Buf - //totalBytes int64 - written int64 //total bytes of file downloaded from remote - err error + written int64 //total bytes of file downloaded from remote + err error + + concurrency int //剩余的并发数,递减。到0时停止并发 + maxPart int //有多少个分片 + pos int64 + maxPos int64 + m2 sync.Mutex + readingID int // 正在被读取的id +} + +type ConcurrencyLimit struct { + _m sync.Mutex + Limit int // 需要大于0 +} + +var ErrExceedMaxConcurrency = fmt.Errorf("ExceedMaxConcurrency") - partBodyMaxRetries int +func (l *ConcurrencyLimit) sub() error { + l._m.Lock() + defer l._m.Unlock() + if l.Limit-1 < 0 { + return ErrExceedMaxConcurrency + } + l.Limit-- + // log.Debugf("ConcurrencyLimit.sub: %d", l.Limit) + return nil +} +func (l *ConcurrencyLimit) add() { + l._m.Lock() + defer l._m.Unlock() + l.Limit++ + // log.Debugf("ConcurrencyLimit.add: %d", l.Limit) +} + +// 检测是否超过限制 +func (d *downloader) concurrencyCheck() error { + if d.cfg.ConcurrencyLimit != nil { + return d.cfg.ConcurrencyLimit.sub() + } + return nil +} +func (d *downloader) concurrencyFinish() { + if d.cfg.ConcurrencyLimit != nil { + d.cfg.ConcurrencyLimit.add() + } } // download performs the implementation of the object download across ranged GETs. func (d *downloader) download() (io.ReadCloser, error) { - d.ctx, d.cancel = context.WithCancel(d.ctx) + if err := d.concurrencyCheck(); err != nil { + return nil, err + } + d.ctx, d.cancel = context.WithCancelCause(d.ctx) - pos := d.params.Range.Start - maxPos := d.params.Range.Start + d.params.Range.Length - id := 0 - for pos < maxPos { - finalSize := int64(d.cfg.PartSize) - //check boundary - if pos+finalSize > maxPos { - finalSize = maxPos - pos - } - c := chunk{start: pos, size: finalSize, id: id} - d.chunks = append(d.chunks, c) - pos += finalSize - id++ + maxPart := int(d.params.Range.Length / int64(d.cfg.PartSize)) + if d.params.Range.Length%int64(d.cfg.PartSize) > 0 { + maxPart++ } - if len(d.chunks) < d.cfg.Concurrency { - d.cfg.Concurrency = len(d.chunks) + if maxPart < d.cfg.Concurrency { + d.cfg.Concurrency = maxPart } + log.Debugf("cfgConcurrency:%d", d.cfg.Concurrency) if d.cfg.Concurrency == 1 { + if d.cfg.ConcurrencyLimit != nil { + go func() { + <-d.ctx.Done() + d.concurrencyFinish() + }() + } resp, err := d.cfg.HttpClient(d.ctx, d.params) if err != nil { return nil, err @@ -141,60 +184,115 @@ func (d *downloader) download() (io.ReadCloser, error) { // workers d.chunkChannel = make(chan chunk, d.cfg.Concurrency) - for i := 0; i < d.cfg.Concurrency; i++ { - buf := NewBuf(d.ctx, d.cfg.PartSize, i) - d.bufs = append(d.bufs, buf) - go d.downloadPart() - } - // initial tasks - for i := 0; i < d.cfg.Concurrency; i++ { - d.sendChunkTask() - } + d.maxPart = maxPart + d.pos = d.params.Range.Start + d.maxPos = d.params.Range.Start + d.params.Range.Length + d.concurrency = d.cfg.Concurrency + d.sendChunkTask(true) - var rc io.ReadCloser = NewMultiReadCloser(d.chunks[0].buf, d.interrupt, d.finishBuf) + var rc io.ReadCloser = NewMultiReadCloser(d.bufs[0], d.interrupt, d.finishBuf) // Return error return rc, d.err } -func (d *downloader) sendChunkTask() *chunk { - ch := &d.chunks[d.nextChunk] - ch.buf = d.getBuf(d.nextChunk) - ch.buf.Reset(int(ch.size)) - d.chunkChannel <- *ch - d.nextChunk++ - return ch + +func (d *downloader) sendChunkTask(newConcurrency bool) error { + d.m.Lock() + defer d.m.Unlock() + isNewBuf := d.concurrency > 0 + if newConcurrency { + if d.concurrency <= 0 { + return nil + } + if d.nextChunk > 0 { // 第一个不检查,因为已经检查过了 + if err := d.concurrencyCheck(); err != nil { + return err + } + } + d.concurrency-- + go d.downloadPart() + } + + var buf *Buf + if isNewBuf { + buf = NewBuf(d.ctx, d.cfg.PartSize) + d.bufs = append(d.bufs, buf) + } else { + buf = d.getBuf(d.nextChunk) + } + + if d.pos < d.maxPos { + finalSize := int64(d.cfg.PartSize) + switch d.nextChunk { + case 0: + // 最小分片在前面有助视频播放? + firstSize := d.params.Range.Length % finalSize + if firstSize > 0 { + minSize := finalSize / 2 + if firstSize < minSize { // 最小分片太小就调整到一半 + finalSize = minSize + } else { + finalSize = firstSize + } + } + case 1: + firstSize := d.params.Range.Length % finalSize + minSize := finalSize / 2 + if firstSize > 0 && firstSize < minSize { + finalSize += firstSize - minSize + } + } + buf.Reset(int(finalSize)) + ch := chunk{ + start: d.pos, + size: finalSize, + id: d.nextChunk, + buf: buf, + + newConcurrency: newConcurrency, + } + d.pos += finalSize + d.nextChunk++ + d.chunkChannel <- ch + return nil + } + return nil } // when the final reader Close, we interrupt func (d *downloader) interrupt() error { - d.cancel() if d.written != d.params.Range.Length { log.Debugf("Downloader interrupt before finish") if d.getErr() == nil { d.setErr(fmt.Errorf("interrupted")) } } + d.cancel(d.err) defer func() { close(d.chunkChannel) for _, buf := range d.bufs { buf.Close() } + if d.concurrency > 0 { + d.concurrency = -d.concurrency + } + log.Debugf("maxConcurrency:%d", d.cfg.Concurrency+d.concurrency) }() return d.err } func (d *downloader) getBuf(id int) (b *Buf) { - - return d.bufs[id%d.cfg.Concurrency] + return d.bufs[id%len(d.bufs)] } -func (d *downloader) finishBuf(id int) (isLast bool, buf *Buf) { - if id >= len(d.chunks)-1 { +func (d *downloader) finishBuf(id int) (isLast bool, nextBuf *Buf) { + id++ + if id >= d.maxPart { return true, nil } - if d.nextChunk > id+1 { - return false, d.getBuf(id + 1) - } - ch := d.sendChunkTask() - return false, ch.buf + + d.sendChunkTask(false) + + d.readingID = id + return false, d.getBuf(id) } // downloadPart is an individual goroutine worker reading from the ch channel @@ -209,58 +307,122 @@ func (d *downloader) downloadPart() { if d.getErr() != nil { // Drain the channel if there is an error, to prevent deadlocking // of download producer. - continue + break } - log.Debugf("downloadPart tried to get chunk") if err := d.downloadChunk(&c); err != nil { + if err == errCancelConcurrency { + break + } + if err == context.Canceled { + if e := context.Cause(d.ctx); e != nil { + err = e + } + } d.setErr(err) + d.cancel(err) } } + d.concurrencyFinish() } // downloadChunk downloads the chunk func (d *downloader) downloadChunk(ch *chunk) error { - log.Debugf("start new chunk %+v buffer_id =%d", ch, ch.id) + log.Debugf("start chunk_%d, %+v", ch.id, ch) + params := d.getParamsFromChunk(ch) var n int64 var err error - params := d.getParamsFromChunk(ch) - for retry := 0; retry <= d.partBodyMaxRetries; retry++ { + for retry := 0; retry <= d.cfg.PartBodyMaxRetries; retry++ { if d.getErr() != nil { - return d.getErr() + return nil } n, err = d.tryDownloadChunk(params, ch) if err == nil { + d.incrWritten(n) + log.Debugf("chunk_%d downloaded", ch.id) break } - // Check if the returned error is an errReadingBody. - // If err is errReadingBody this indicates that an error - // occurred while copying the http response body. + if d.getErr() != nil { + return nil + } + if utils.IsCanceled(d.ctx) { + return d.ctx.Err() + } + // Check if the returned error is an errNeedRetry. // If this occurs we unwrap the err to set the underlying error // and attempt any remaining retries. - if bodyErr, ok := err.(*errReadingBody); ok { - err = bodyErr.Unwrap() + if e, ok := err.(*errNeedRetry); ok { + err = e.Unwrap() + if n > 0 { + // 测试:下载时 断开 alist向云盘发起的下载连接 + // 校验:下载完后校验文件哈希值 一致 + d.incrWritten(n) + ch.start += n + ch.size -= n + params.Range.Start = ch.start + params.Range.Length = ch.size + } + log.Warnf("err chunk_%d, object part download error %s, retrying attempt %d. %v", + ch.id, params.URL, retry, err) + } else if err == errInfiniteRetry { + retry-- + continue } else { - return err + break } - - //ch.cur = 0 - - log.Debugf("object part body download interrupted %s, err, %v, retrying attempt %d", - params.URL, err, retry) } - d.incrWritten(n) - log.Debugf("down_%d downloaded chunk", ch.id) - //ch.buf.buffer.wg1.Wait() - //log.Debugf("down_%d downloaded chunk,wg wait passed", ch.id) return err } -func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { +var errCancelConcurrency = fmt.Errorf("cancel concurrency") +var errInfiniteRetry = fmt.Errorf("infinite retry") +func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int64, error) { resp, err := d.cfg.HttpClient(d.ctx, params) if err != nil { - return 0, err + if resp == nil { + return 0, err + } + if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { + return 0, err + } + if ch.id == 0 { //第1个任务 有限的重试,超过重试就会结束请求 + switch resp.StatusCode { + default: + return 0, err + case http.StatusTooManyRequests: + case http.StatusBadGateway: + case http.StatusServiceUnavailable: + case http.StatusGatewayTimeout: + } + <-time.After(time.Millisecond * 200) + return 0, &errNeedRetry{err: fmt.Errorf("http request failure,status: %d", resp.StatusCode)} + } + + // 来到这 说明第1个分片下载 连接成功了 + // 后续分片下载出错都当超载处理 + log.Debugf("err chunk_%d, try downloading:%v", ch.id, err) + + d.m.Lock() + isCancelConcurrency := ch.newConcurrency + if d.concurrency > 0 { // 取消剩余的并发任务 + // 用于计算实际的并发数 + d.concurrency = -d.concurrency + isCancelConcurrency = true + } + if isCancelConcurrency { + d.concurrency-- + d.chunkChannel <- *ch + d.m.Unlock() + return 0, errCancelConcurrency + } + d.m.Unlock() + if ch.id != d.readingID { //正在被读取的优先重试 + d.m2.Lock() + defer d.m2.Unlock() + <-time.After(time.Millisecond * 200) + } + return 0, errInfiniteRetry } defer resp.Body.Close() //only check file size on the first task @@ -270,15 +432,15 @@ func (d *downloader) tryDownloadChunk(params *HttpRequestParams, ch *chunk) (int return 0, err } } - - n, err := io.Copy(ch.buf, resp.Body) + d.sendChunkTask(true) + n, err := utils.CopyWithBuffer(ch.buf, resp.Body) if err != nil { - return n, &errReadingBody{err: err} + return n, &errNeedRetry{err: err} } if n != ch.size { err = fmt.Errorf("chunk download size incorrect, expected=%d, got=%d", ch.size, n) - return n, &errReadingBody{err: err} + return n, &errNeedRetry{err: err} } return n, nil @@ -294,7 +456,7 @@ func (d *downloader) getParamsFromChunk(ch *chunk) *HttpRequestParams { func (d *downloader) checkTotalBytes(resp *http.Response) error { var err error - var totalBytes int64 = math.MinInt64 + totalBytes := int64(-1) contentRange := resp.Header.Get("Content-Range") if len(contentRange) == 0 { // ContentRange is nil when the full file contents is provided, and @@ -326,8 +488,9 @@ func (d *downloader) checkTotalBytes(resp *http.Response) error { err = fmt.Errorf("expect file size=%d unmatch remote report size=%d, need refresh cache", d.params.Size, totalBytes) } if err != nil { - _ = d.interrupt() + // _ = d.interrupt() d.setErr(err) + d.cancel(err) } return err @@ -366,9 +529,7 @@ type chunk struct { buf *Buf id int - // Downloader takes range (start,length), but this chunk is requesting equal/sub range of it. - // To convert the writer to reader eventually, we need to write within the boundary - //boundary http_range.Range + newConcurrency bool } func DefaultHttpRequestFunc(ctx context.Context, params *HttpRequestParams) (*http.Response, error) { @@ -376,7 +537,7 @@ func DefaultHttpRequestFunc(ctx context.Context, params *HttpRequestParams) (*ht res, err := RequestHttp(ctx, "GET", header, params.URL) if err != nil { - return nil, err + return res, err } return res, nil } @@ -389,15 +550,15 @@ type HttpRequestParams struct { //total file size Size int64 } -type errReadingBody struct { +type errNeedRetry struct { err error } -func (e *errReadingBody) Error() string { - return fmt.Sprintf("failed to read part body: %v", e.err) +func (e *errNeedRetry) Error() string { + return e.err.Error() } -func (e *errReadingBody) Unwrap() error { +func (e *errNeedRetry) Unwrap() error { return e.err } @@ -435,9 +596,13 @@ func (mr MultiReadCloser) Read(p []byte) (n int, err error) { } mr.cfg.curBuf = next mr.cfg.rPos++ - //current.Close() return n, nil } + if err == context.Canceled { + if e := context.Cause(mr.cfg.curBuf.ctx); e != nil { + err = e + } + } return n, err } func (mr MultiReadCloser) Close() error { @@ -449,16 +614,17 @@ type Buf struct { size int //expected size ctx context.Context off int - rw sync.RWMutex - notify chan struct{} + rw sync.Mutex } // NewBuf is a buffer that can have 1 read & 1 write at the same time. // when read is faster write, immediately feed data to read after written -func NewBuf(ctx context.Context, maxSize int, id int) *Buf { - d := make([]byte, 0, maxSize) - return &Buf{ctx: ctx, buffer: bytes.NewBuffer(d), size: maxSize, notify: make(chan struct{})} - +func NewBuf(ctx context.Context, maxSize int) *Buf { + return &Buf{ + ctx: ctx, + buffer: bytes.NewBuffer(make([]byte, 0, maxSize)), + size: maxSize, + } } func (br *Buf) Reset(size int) { br.buffer.Reset() @@ -476,9 +642,13 @@ func (br *Buf) Read(p []byte) (n int, err error) { if br.off >= br.size { return 0, io.EOF } - br.rw.RLock() + br.rw.Lock() + if br.buffer == nil { + br.rw.Unlock() + return 0, io.ErrClosedPipe + } n, err = br.buffer.Read(p) - br.rw.RUnlock() + br.rw.Unlock() if err == nil { br.off += n return n, err @@ -495,8 +665,6 @@ func (br *Buf) Read(p []byte) (n int, err error) { select { case <-br.ctx.Done(): return 0, br.ctx.Err() - case <-br.notify: - return 0, nil case <-time.After(time.Millisecond * 200): return 0, nil } @@ -508,14 +676,15 @@ func (br *Buf) Write(p []byte) (n int, err error) { } br.rw.Lock() defer br.rw.Unlock() - n, err = br.buffer.Write(p) - select { - case br.notify <- struct{}{}: - default: + if br.buffer == nil { + return 0, io.ErrClosedPipe } + n, err = br.buffer.Write(p) return } func (br *Buf) Close() { - close(br.notify) + br.rw.Lock() + defer br.rw.Unlock() + br.buffer = nil } diff --git a/internal/net/request_test.go b/internal/net/request_test.go index e41971fbf61..032b7376585 100644 --- a/internal/net/request_test.go +++ b/internal/net/request_test.go @@ -8,7 +8,6 @@ import ( "context" "fmt" "io" - "io/ioutil" "net/http" "sync" "testing" @@ -169,7 +168,7 @@ func newDownloadRangeClient(data []byte) (*downloadCaptureClient, *int, *[]strin header := &http.Header{} header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, fin-1, len(data))) return &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader(bodyBytes)), + Body: io.NopCloser(bytes.NewReader(bodyBytes)), Header: *header, ContentLength: int64(len(bodyBytes)), }, nil diff --git a/internal/net/serve.go b/internal/net/serve.go index a0566780759..bdeac0ac5f8 100644 --- a/internal/net/serve.go +++ b/internal/net/serve.go @@ -1,7 +1,9 @@ package net import ( + "compress/gzip" "context" + "crypto/tls" "fmt" "io" "mime" @@ -13,7 +15,6 @@ import ( "sync" "time" - "github.com/alist-org/alist/v3/drivers/base" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/http_range" @@ -51,18 +52,19 @@ import ( // // If the caller has set w's ETag header formatted per RFC 7232, section 2.3, // ServeHTTP uses it to handle requests using If-Match, If-None-Match, or If-Range. -func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReaderFunc model.RangeReaderFunc) { +func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, size int64, RangeReadCloser model.RangeReadCloserIF) error { + defer RangeReadCloser.Close() setLastModified(w, modTime) done, rangeReq := checkPreconditions(w, r, modTime) if done { - return + return nil } if size < 0 { // since too many functions need file size to work, // will not implement the support of unknown file size here http.Error(w, "negative content size not supported", http.StatusInternalServerError) - return + return nil } code := http.StatusOK @@ -86,9 +88,9 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time sendSize := size var sendContent io.ReadCloser ranges, err := http_range.ParseRange(rangeReq, size) - switch err { - case nil: - case http_range.ErrNoOverlap: + switch { + case err == nil: + case errors.Is(err, http_range.ErrNoOverlap): if size == 0 { // Some clients add a Range header to all requests to // limit the size of the response. If the file is empty, @@ -101,20 +103,28 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time fallthrough default: http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) - return + return nil } - if sumRangesSize(ranges) > size || size < 0 { + if sumRangesSize(ranges) > size { // The total number of bytes in all the ranges is larger than the size of the file // or unknown file size, ignore the range request. ranges = nil } + + // 使用请求的Context + // 不然从sendContent读不到数据,即使请求断开CopyBuffer也会一直堵塞 + ctx := context.WithValue(r.Context(), "request_header", r.Header) switch { case len(ranges) == 0: - reader, err := RangeReaderFunc(context.Background(), http_range.Range{Length: -1}) + reader, err := RangeReadCloser.RangeRead(ctx, http_range.Range{Length: -1}) if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return + code = http.StatusRequestedRangeNotSatisfiable + if err == ErrExceedMaxConcurrency { + code = http.StatusTooManyRequests + } + http.Error(w, err.Error(), code) + return nil } sendContent = reader case len(ranges) == 1: @@ -130,10 +140,14 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time // does not request multiple parts might not support // multipart responses." ra := ranges[0] - sendContent, err = RangeReaderFunc(context.Background(), ra) + sendContent, err = RangeReadCloser.RangeRead(ctx, ra) if err != nil { - http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) - return + code = http.StatusRequestedRangeNotSatisfiable + if err == ErrExceedMaxConcurrency { + code = http.StatusTooManyRequests + } + http.Error(w, err.Error(), code) + return nil } sendSize = ra.Length code = http.StatusPartialContent @@ -157,16 +171,15 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time pw.CloseWithError(err) return } - reader, err := RangeReaderFunc(context.Background(), ra) + reader, err := RangeReadCloser.RangeRead(ctx, ra) if err != nil { pw.CloseWithError(err) return } - if _, err := io.CopyN(part, reader, ra.Length); err != nil { + if _, err := utils.CopyWithBufferN(part, reader, ra.Length); err != nil { pw.CloseWithError(err) return } - //defer reader.Close() } mw.Close() @@ -182,16 +195,21 @@ func ServeHTTP(w http.ResponseWriter, r *http.Request, name string, modTime time w.WriteHeader(code) if r.Method != "HEAD" { - written, err := io.CopyN(w, sendContent, sendSize) + written, err := utils.CopyWithBufferN(w, sendContent, sendSize) if err != nil { log.Warnf("ServeHttp error. err: %s ", err) if written != sendSize { log.Warnf("Maybe size incorrect or reader not giving correct/full data, or connection closed before finish. written bytes: %d ,sendSize:%d, ", written, sendSize) } - http.Error(w, err.Error(), http.StatusInternalServerError) + code = http.StatusInternalServerError + if err == ErrExceedMaxConcurrency { + code = http.StatusTooManyRequests + } + w.WriteHeader(code) + return err } } - //defer sendContent.Close() + return nil } func ProcessHeader(origin, override http.Header) http.Header { result := http.Header{} @@ -222,12 +240,23 @@ func RequestHttp(ctx context.Context, httpMethod string, headerOverride http.Hea } // TODO clean header with blocklist or passlist res.Header.Del("set-cookie") + var reader io.Reader if res.StatusCode >= 400 { - all, _ := io.ReadAll(res.Body) + // 根据 Content-Encoding 判断 Body 是否压缩 + switch res.Header.Get("Content-Encoding") { + case "gzip": + // 使用gzip.NewReader解压缩 + reader, _ = gzip.NewReader(res.Body) + defer reader.(*gzip.Reader).Close() + default: + // 没有Content-Encoding,直接读取 + reader = res.Body + } + all, _ := io.ReadAll(reader) _ = res.Body.Close() msg := string(all) log.Debugln(msg) - return nil, fmt.Errorf("http request [%s] failure,status: %d response:%s", URL, res.StatusCode, msg) + return res, fmt.Errorf("http request [%s] failure,status: %d response:%s", URL, res.StatusCode, msg) } return res, nil } @@ -237,7 +266,7 @@ var httpClient *http.Client func HttpClient() *http.Client { once.Do(func() { - httpClient = base.NewHttpClient() + httpClient = NewHttpClient() httpClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { if len(via) >= 10 { return errors.New("stopped after 10 redirects") @@ -248,3 +277,13 @@ func HttpClient() *http.Client { }) return httpClient } + +func NewHttpClient() *http.Client { + return &http.Client{ + Timeout: time.Hour * 48, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}, + }, + } +} diff --git a/internal/net/util.go b/internal/net/util.go index 99f95c9a420..5b335a7f718 100644 --- a/internal/net/util.go +++ b/internal/net/util.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/http_range" log "github.com/sirupsen/logrus" ) @@ -69,6 +71,7 @@ func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { if im == "" { return condNone } + r.Header.Del("If-Match") for { im = textproto.TrimString(im) if len(im) == 0 { @@ -96,7 +99,11 @@ func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { ius := r.Header.Get("If-Unmodified-Since") - if ius == "" || isZeroTime(modtime) { + if ius == "" { + return condNone + } + r.Header.Del("If-Unmodified-Since") + if isZeroTime(modtime) { return condNone } t, err := http.ParseTime(ius) @@ -118,6 +125,7 @@ func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult { if inm == "" { return condNone } + r.Header.Del("If-None-Match") buf := inm for { buf = textproto.TrimString(buf) @@ -148,7 +156,11 @@ func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { return condNone } ims := r.Header.Get("If-Modified-Since") - if ims == "" || isZeroTime(modtime) { + if ims == "" { + return condNone + } + r.Header.Del("If-Modified-Since") + if isZeroTime(modtime) { return condNone } t, err := http.ParseTime(ims) @@ -172,6 +184,7 @@ func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) con if ir == "" { return condNone } + r.Header.Del("If-Range") etag, _ := scanETag(ir) if etag != "" { if etagStrongMatch(etag, w.Header().Get("Etag")) { @@ -327,10 +340,10 @@ func GetRangedHttpReader(readCloser io.ReadCloser, offset, length int64) (io.Rea length_int = int(length) if offset > 100*1024*1024 { - log.Warnf("offset is more than 100MB, if loading data from internet, high-latency and wasting of bandwith is expected") + log.Warnf("offset is more than 100MB, if loading data from internet, high-latency and wasting of bandwidth is expected") } - if _, err := io.Copy(io.Discard, io.LimitReader(readCloser, offset)); err != nil { + if _, err := utils.CopyWithBuffer(io.Discard, io.LimitReader(readCloser, offset)); err != nil { return nil, err } diff --git a/internal/offline_download/115/client.go b/internal/offline_download/115/client.go new file mode 100644 index 00000000000..3f9d804dabf --- /dev/null +++ b/internal/offline_download/115/client.go @@ -0,0 +1,142 @@ +package _115 + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" + + "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" +) + +type Cloud115 struct { + refreshTaskCache bool +} + +func (p *Cloud115) Name() string { + return "115 Cloud" +} + +func (p *Cloud115) Items() []model.SettingItem { + return nil +} + +func (p *Cloud115) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (p *Cloud115) Init() (string, error) { + p.refreshTaskCache = false + return "ok", nil +} + +func (p *Cloud115) IsReady() bool { + tempDir := setting.GetStr(conf.Pan115TempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*_115.Pan115); !ok { + return false + } + return true +} + +func (p *Cloud115) AddURL(args *tool.AddUrlArgs) (string, error) { + // 添加新任务刷新缓存 + p.refreshTaskCache = true + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver115, ok := storage.(*_115.Pan115) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") + } + + ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + + hashs, err := driver115.OfflineDownload(ctx, []string{args.Url}, parentDir) + if err != nil || len(hashs) < 1 { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + + return hashs[0], nil +} + +func (p *Cloud115) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + driver115, ok := storage.(*_115.Pan115) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") + } + + ctx := context.Background() + if err := driver115.DeleteOfflineTasks(ctx, []string{task.GID}, false); err != nil { + return err + } + return nil +} + +func (p *Cloud115) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver115, ok := storage.(*_115.Pan115) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only 115 Cloud is supported") + } + + tasks, err := driver115.OfflineList(context.Background()) + if err != nil { + return nil, err + } + + s := &tool.Status{ + Progress: 0, + NewGID: "", + Completed: false, + Status: "the task has been deleted", + Err: nil, + } + for _, t := range tasks { + if t.InfoHash == task.GID { + s.Progress = t.Percent + s.Status = t.GetStatus() + s.Completed = t.IsDone() + s.TotalBytes = t.Size + if t.IsFailed() { + s.Err = fmt.Errorf(t.GetStatus()) + } + return s, nil + } + } + s.Err = fmt.Errorf("the task has been deleted") + return nil, nil +} + +var _ tool.Tool = (*Cloud115)(nil) + +func init() { + tool.Tools.Add(&Cloud115{}) +} diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go new file mode 100644 index 00000000000..1ba191e47e8 --- /dev/null +++ b/internal/offline_download/all.go @@ -0,0 +1,12 @@ +package offline_download + +import ( + _ "github.com/alist-org/alist/v3/internal/offline_download/115" + _ "github.com/alist-org/alist/v3/internal/offline_download/aria2" + _ "github.com/alist-org/alist/v3/internal/offline_download/guangyapan" + _ "github.com/alist-org/alist/v3/internal/offline_download/http" + _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" + _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" + _ "github.com/alist-org/alist/v3/internal/offline_download/thunder" + _ "github.com/alist-org/alist/v3/internal/offline_download/transmission" +) diff --git a/internal/offline_download/aria2/aria2.go b/internal/offline_download/aria2/aria2.go new file mode 100644 index 00000000000..fb212b35990 --- /dev/null +++ b/internal/offline_download/aria2/aria2.go @@ -0,0 +1,128 @@ +package aria2 + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/alist-org/alist/v3/internal/errs" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/aria2/rpc" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +var notify = NewNotify() + +type Aria2 struct { + client rpc.Client +} + +func (a *Aria2) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (a *Aria2) Name() string { + return "aria2" +} + +func (a *Aria2) Items() []model.SettingItem { + // aria2 settings + return []model.SettingItem{ + {Key: conf.Aria2Uri, Value: "http://localhost:6800/jsonrpc", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.Aria2Secret, Value: "", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (a *Aria2) Init() (string, error) { + a.client = nil + uri := setting.GetStr(conf.Aria2Uri) + secret := setting.GetStr(conf.Aria2Secret) + c, err := rpc.New(context.Background(), uri, secret, 4*time.Second, notify) + if err != nil { + return "", errors.Wrap(err, "failed to init aria2 client") + } + version, err := c.GetVersion() + if err != nil { + return "", errors.Wrapf(err, "failed get aria2 version") + } + a.client = c + log.Infof("using aria2 version: %s", version.Version) + return fmt.Sprintf("aria2 version: %s", version.Version), nil +} + +func (a *Aria2) IsReady() bool { + return a.client != nil +} + +func (a *Aria2) AddURL(args *tool.AddUrlArgs) (string, error) { + options := map[string]interface{}{ + "dir": args.TempDir, + } + gid, err := a.client.AddURI([]string{args.Url}, options) + if err != nil { + return "", err + } + notify.Signals.Store(gid, args.Signal) + return gid, nil +} + +func (a *Aria2) Remove(task *tool.DownloadTask) error { + _, err := a.client.Remove(task.GID) + return err +} + +func (a *Aria2) Status(task *tool.DownloadTask) (*tool.Status, error) { + info, err := a.client.TellStatus(task.GID) + if err != nil { + return nil, err + } + total, err := strconv.ParseInt(info.TotalLength, 10, 64) + if err != nil { + total = 0 + } + downloaded, err := strconv.ParseUint(info.CompletedLength, 10, 64) + if err != nil { + downloaded = 0 + } + s := &tool.Status{ + Completed: info.Status == "complete", + Err: err, + TotalBytes: total, + } + s.Progress = float64(downloaded) / float64(total) * 100 + if len(info.FollowedBy) != 0 { + s.NewGID = info.FollowedBy[0] + notify.Signals.Delete(task.GID) + notify.Signals.Store(s.NewGID, task.Signal) + } + switch info.Status { + case "complete": + s.Completed = true + case "error": + s.Err = errors.Errorf("failed to download %s, error: %s", task.GID, info.ErrorMessage) + case "active": + s.Status = "aria2: " + info.Status + if info.Seeder == "true" { + s.Completed = true + } + case "waiting", "paused": + s.Status = "aria2: " + info.Status + case "removed": + s.Err = errors.Errorf("failed to download %s, removed", task.GID) + default: + return nil, errors.Errorf("[aria2] unknown status %s", info.Status) + } + return s, nil +} + +var _ tool.Tool = (*Aria2)(nil) + +func init() { + tool.Tools.Add(&Aria2{}) +} diff --git a/internal/aria2/notify.go b/internal/offline_download/aria2/notify.go similarity index 100% rename from internal/aria2/notify.go rename to internal/offline_download/aria2/notify.go diff --git a/internal/offline_download/guangyapan/guangyapan.go b/internal/offline_download/guangyapan/guangyapan.go new file mode 100644 index 00000000000..eb58af0bd4a --- /dev/null +++ b/internal/offline_download/guangyapan/guangyapan.go @@ -0,0 +1,131 @@ +package guangyapan + +import ( + "context" + "errors" + "fmt" + + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" +) + +type GuangYaPan struct { + refreshTaskCache bool +} + +func (g *GuangYaPan) Name() string { + return "GuangYaPan" +} + +func (g *GuangYaPan) Items() []model.SettingItem { + return nil +} + +func (g *GuangYaPan) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (g *GuangYaPan) Init() (string, error) { + g.refreshTaskCache = false + return "ok", nil +} + +func (g *GuangYaPan) IsReady() bool { + tempDir := setting.GetStr(conf.GuangYaPanTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + return false + } + return true +} + +func (g *GuangYaPan) AddURL(args *tool.AddUrlArgs) (string, error) { + g.refreshTaskCache = true + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return "", errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + + ctx := context.Background() + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + task, err := driver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + return task.TaskID, nil +} + +func (g *GuangYaPan) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + ctx := context.Background() + if err := driver.DeleteOfflineTasks(ctx, []string{task.GID}, false); err != nil { + return err + } + g.DelTaskCache(driver, task.GID) + return nil +} + +func (g *GuangYaPan) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + driver, ok := storage.(*guangyapandriver.GuangYaPan) + if !ok { + return nil, errors.New("GuangYaPan offline download only supports GuangYaPan destination storage") + } + tasks, err := g.GetTasks(driver, task.GID) + if err != nil { + return nil, err + } + status := &tool.Status{ + Status: "the task has been deleted", + } + for _, t := range tasks { + if t.TaskID != task.GID { + continue + } + status.Progress = float64(t.Progress) + status.TotalBytes = t.TotalSize + status.Completed = t.Status == offlineStatusCompleted || t.Status == offlineStatusPartiallyCompleted + status.Status = taskStatusText(t) + if t.Status == offlineStatusFailed || t.Status == offlineStatusCanceled { + status.Err = errors.New(status.Status) + } + return status, nil + } + status.Err = errors.New("the task has been deleted") + return status, nil +} + +func init() { + tool.Tools.Add(&GuangYaPan{}) +} diff --git a/internal/offline_download/guangyapan/util.go b/internal/offline_download/guangyapan/util.go new file mode 100644 index 00000000000..cb7bb926ead --- /dev/null +++ b/internal/offline_download/guangyapan/util.go @@ -0,0 +1,77 @@ +package guangyapan + +import ( + "context" + "fmt" + "time" + + "github.com/Xhofe/go-cache" + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +const ( + offlineStatusQueued = 0 + offlineStatusRunning = 1 + offlineStatusCompleted = 2 + offlineStatusFailed = 3 + offlineStatusCanceled = 4 + offlineStatusPartiallyCompleted = 5 +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]guangyapandriver.OfflineTask](16)) +var taskG singleflight.Group[[]guangyapandriver.OfflineTask] + +func (g *GuangYaPan) GetTasks(driver *guangyapandriver.GuangYaPan, taskID string) ([]guangyapandriver.OfflineTask, error) { + key := op.Key(driver, "/cloudcollection/v1/list_task/"+taskID) + if !g.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + g.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]guangyapandriver.OfflineTask, error) { + ctx := context.Background() + tasks, err := driver.OfflineList(ctx, []string{taskID}, nil, "", 0) + if err != nil { + return nil, err + } + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]guangyapandriver.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} + +func (g *GuangYaPan) DelTaskCache(driver *guangyapandriver.GuangYaPan, taskID string) { + taskCache.Del(op.Key(driver, "/cloudcollection/v1/list_task/"+taskID)) +} + +func taskStatusText(task guangyapandriver.OfflineTask) string { + switch task.Status { + case offlineStatusQueued: + return "queued" + case offlineStatusRunning: + if task.Progress > 0 { + return fmt.Sprintf("running (%d%%)", task.Progress) + } + return "running" + case offlineStatusCompleted: + return "completed" + case offlineStatusFailed: + return "failed" + case offlineStatusCanceled: + return "canceled" + case offlineStatusPartiallyCompleted: + return "partially completed" + default: + return fmt.Sprintf("unknown status %d", task.Status) + } +} diff --git a/internal/offline_download/http/client.go b/internal/offline_download/http/client.go new file mode 100644 index 00000000000..9b83400ea34 --- /dev/null +++ b/internal/offline_download/http/client.go @@ -0,0 +1,93 @@ +package http + +import ( + "fmt" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/pkg/utils" +) + +type SimpleHttp struct { + client http.Client +} + +func (s SimpleHttp) Name() string { + return "SimpleHttp" +} + +func (s SimpleHttp) Items() []model.SettingItem { + return nil +} + +func (s SimpleHttp) Init() (string, error) { + return "ok", nil +} + +func (s SimpleHttp) IsReady() bool { + return true +} + +func (s SimpleHttp) AddURL(args *tool.AddUrlArgs) (string, error) { + panic("should not be called") +} + +func (s SimpleHttp) Remove(task *tool.DownloadTask) error { + panic("should not be called") +} + +func (s SimpleHttp) Status(task *tool.DownloadTask) (*tool.Status, error) { + panic("should not be called") +} + +func (s SimpleHttp) Run(task *tool.DownloadTask) error { + u := task.Url + // parse url + _u, err := url.Parse(u) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(task.Ctx(), http.MethodGet, u, nil) + if err != nil { + return err + } + resp, err := s.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + return fmt.Errorf("http status code %d", resp.StatusCode) + } + // If Path is empty, use Hostname; otherwise, filePath euqals TempDir which causes os.Create to fail + urlPath := _u.Path + if urlPath == "" { + urlPath = strings.ReplaceAll(_u.Host, ".", "_") + } + filename := path.Base(urlPath) + if n, err := parseFilenameFromContentDisposition(resp.Header.Get("Content-Disposition")); err == nil { + filename = n + } + // save to temp dir + _ = os.MkdirAll(task.TempDir, os.ModePerm) + filePath := filepath.Join(task.TempDir, filename) + file, err := os.Create(filePath) + if err != nil { + return err + } + defer file.Close() + fileSize := resp.ContentLength + task.SetTotalBytes(fileSize) + err = utils.CopyWithCtx(task.Ctx(), file, resp.Body, fileSize, task.SetProgress) + return err +} + +func init() { + tool.Tools.Add(&SimpleHttp{}) +} diff --git a/internal/offline_download/http/util.go b/internal/offline_download/http/util.go new file mode 100644 index 00000000000..eefefec24ac --- /dev/null +++ b/internal/offline_download/http/util.go @@ -0,0 +1,21 @@ +package http + +import ( + "fmt" + "mime" +) + +func parseFilenameFromContentDisposition(contentDisposition string) (string, error) { + if contentDisposition == "" { + return "", fmt.Errorf("Content-Disposition is empty") + } + _, params, err := mime.ParseMediaType(contentDisposition) + if err != nil { + return "", err + } + filename := params["filename"] + if filename == "" { + return "", fmt.Errorf("filename not found in Content-Disposition: [%s]", contentDisposition) + } + return filename, nil +} diff --git a/internal/offline_download/pikpak/pikpak.go b/internal/offline_download/pikpak/pikpak.go new file mode 100644 index 00000000000..8fdfb3405cf --- /dev/null +++ b/internal/offline_download/pikpak/pikpak.go @@ -0,0 +1,142 @@ +package pikpak + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" + "strconv" + + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" +) + +type PikPak struct { + refreshTaskCache bool +} + +func (p *PikPak) Name() string { + return "PikPak" +} + +func (p *PikPak) Items() []model.SettingItem { + return nil +} + +func (p *PikPak) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (p *PikPak) Init() (string, error) { + p.refreshTaskCache = false + return "ok", nil +} + +func (p *PikPak) IsReady() bool { + tempDir := setting.GetStr(conf.PikPakTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*pikpak.PikPak); !ok { + return false + } + return true +} + +func (p *PikPak) AddURL(args *tool.AddUrlArgs) (string, error) { + // 添加新任务刷新缓存 + p.refreshTaskCache = true + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + pikpakDriver, ok := storage.(*pikpak.PikPak) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") + } + + ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + + t, err := pikpakDriver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + + return t.ID, nil +} + +func (p *PikPak) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + pikpakDriver, ok := storage.(*pikpak.PikPak) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") + } + ctx := context.Background() + err = pikpakDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false) + if err != nil { + return err + } + return nil +} + +func (p *PikPak) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + pikpakDriver, ok := storage.(*pikpak.PikPak) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only Pikpak is supported") + } + tasks, err := p.GetTasks(pikpakDriver) + if err != nil { + return nil, err + } + s := &tool.Status{ + Progress: 0, + NewGID: "", + Completed: false, + Status: "the task has been deleted", + Err: nil, + } + for _, t := range tasks { + if t.ID == task.GID { + s.Progress = float64(t.Progress) + s.Status = t.Message + s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE") + s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) + if err != nil { + s.TotalBytes = 0 + } + if t.Phase == "PHASE_TYPE_ERROR" { + s.Err = fmt.Errorf(t.Message) + } + return s, nil + } + } + s.Err = fmt.Errorf("the task has been deleted") + return s, nil +} + +func init() { + tool.Tools.Add(&PikPak{}) +} diff --git a/internal/offline_download/pikpak/util.go b/internal/offline_download/pikpak/util.go new file mode 100644 index 00000000000..f7bf9282621 --- /dev/null +++ b/internal/offline_download/pikpak/util.go @@ -0,0 +1,43 @@ +package pikpak + +import ( + "context" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]pikpak.OfflineTask](16)) +var taskG singleflight.Group[[]pikpak.OfflineTask] + +func (p *PikPak) GetTasks(pikpakDriver *pikpak.PikPak) ([]pikpak.OfflineTask, error) { + key := op.Key(pikpakDriver, "/drive/v1/task") + if !p.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + p.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]pikpak.OfflineTask, error) { + ctx := context.Background() + phase := []string{"PHASE_TYPE_RUNNING", "PHASE_TYPE_ERROR", "PHASE_TYPE_PENDING", "PHASE_TYPE_COMPLETE"} + tasks, err := pikpakDriver.OfflineList(ctx, "", phase) + if err != nil { + return nil, err + } + // 添加缓存 10s + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]pikpak.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} diff --git a/internal/offline_download/qbit/qbit.go b/internal/offline_download/qbit/qbit.go new file mode 100644 index 00000000000..458de03f02f --- /dev/null +++ b/internal/offline_download/qbit/qbit.go @@ -0,0 +1,86 @@ +package qbit + +import ( + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/qbittorrent" + "github.com/pkg/errors" +) + +type QBittorrent struct { + client qbittorrent.Client +} + +func (a *QBittorrent) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (a *QBittorrent) Name() string { + return "qBittorrent" +} + +func (a *QBittorrent) Items() []model.SettingItem { + // qBittorrent settings + return []model.SettingItem{ + {Key: conf.QbittorrentUrl, Value: "http://admin:adminadmin@localhost:8080/", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.QbittorrentSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (a *QBittorrent) Init() (string, error) { + a.client = nil + url := setting.GetStr(conf.QbittorrentUrl) + qbClient, err := qbittorrent.New(url) + if err != nil { + return "", err + } + a.client = qbClient + return "ok", nil +} + +func (a *QBittorrent) IsReady() bool { + return a.client != nil +} + +func (a *QBittorrent) AddURL(args *tool.AddUrlArgs) (string, error) { + err := a.client.AddFromLink(args.Url, args.TempDir, args.UID) + if err != nil { + return "", err + } + return args.UID, nil +} + +func (a *QBittorrent) Remove(task *tool.DownloadTask) error { + err := a.client.Delete(task.GID, false) + return err +} + +func (a *QBittorrent) Status(task *tool.DownloadTask) (*tool.Status, error) { + info, err := a.client.GetInfo(task.GID) + if err != nil { + return nil, err + } + s := &tool.Status{} + s.TotalBytes = info.Size + s.Progress = float64(info.Completed) / float64(info.Size) * 100 + switch info.State { + case qbittorrent.UPLOADING, qbittorrent.PAUSEDUP, qbittorrent.QUEUEDUP, qbittorrent.STALLEDUP, qbittorrent.FORCEDUP, qbittorrent.CHECKINGUP: + s.Completed = true + case qbittorrent.ALLOCATING, qbittorrent.DOWNLOADING, qbittorrent.METADL, qbittorrent.PAUSEDDL, qbittorrent.QUEUEDDL, qbittorrent.STALLEDDL, qbittorrent.CHECKINGDL, qbittorrent.FORCEDDL, qbittorrent.CHECKINGRESUMEDATA, qbittorrent.MOVING: + s.Status = "[qBittorrent] downloading" + case qbittorrent.ERROR, qbittorrent.MISSINGFILES, qbittorrent.UNKNOWN: + s.Err = errors.Errorf("[qBittorrent] failed to download %s, error: %s", task.GID, info.State) + default: + s.Err = errors.Errorf("[qBittorrent] unknown error occurred downloading %s", task.GID) + } + return s, nil +} + +var _ tool.Tool = (*QBittorrent)(nil) + +func init() { + tool.Tools.Add(&QBittorrent{}) +} diff --git a/internal/offline_download/thunder/thunder.go b/internal/offline_download/thunder/thunder.go new file mode 100644 index 00000000000..81b9486184f --- /dev/null +++ b/internal/offline_download/thunder/thunder.go @@ -0,0 +1,143 @@ +package thunder + +import ( + "context" + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" + "strconv" + + "github.com/alist-org/alist/v3/drivers/thunder" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" +) + +type Thunder struct { + refreshTaskCache bool +} + +func (t *Thunder) Name() string { + return "Thunder" +} + +func (t *Thunder) Items() []model.SettingItem { + return nil +} + +func (t *Thunder) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (t *Thunder) Init() (string, error) { + t.refreshTaskCache = false + return "ok", nil +} + +func (t *Thunder) IsReady() bool { + tempDir := setting.GetStr(conf.ThunderTempDir) + if tempDir == "" { + return false + } + storage, _, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return false + } + if _, ok := storage.(*thunder.Thunder); !ok { + return false + } + return true +} + +func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { + // 添加新任务刷新缓存 + t.refreshTaskCache = true + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + + ctx := context.Background() + + if err := op.MakeDir(ctx, storage, actualPath); err != nil { + return "", err + } + + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + + task, err := thunderDriver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + + return task.ID, nil +} + +func (t *Thunder) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + ctx := context.Background() + err = thunderDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false) + if err != nil { + return err + } + return nil +} + +func (t *Thunder) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.TempDir) + if err != nil { + return nil, err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + tasks, err := t.GetTasks(thunderDriver) + if err != nil { + return nil, err + } + s := &tool.Status{ + Progress: 0, + NewGID: "", + Completed: false, + Status: "the task has been deleted", + Err: nil, + } + for _, t := range tasks { + if t.ID == task.GID { + s.Progress = float64(t.Progress) + s.Status = t.Message + s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE") + s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) + if err != nil { + s.TotalBytes = 0 + } + if t.Phase == "PHASE_TYPE_ERROR" { + s.Err = errors.New(t.Message) + } + return s, nil + } + } + s.Err = fmt.Errorf("the task has been deleted") + return s, nil +} + +func init() { + tool.Tools.Add(&Thunder{}) +} diff --git a/internal/offline_download/thunder/util.go b/internal/offline_download/thunder/util.go new file mode 100644 index 00000000000..ea400f321d0 --- /dev/null +++ b/internal/offline_download/thunder/util.go @@ -0,0 +1,42 @@ +package thunder + +import ( + "context" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/drivers/thunder" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]thunder.OfflineTask](16)) +var taskG singleflight.Group[[]thunder.OfflineTask] + +func (t *Thunder) GetTasks(thunderDriver *thunder.Thunder) ([]thunder.OfflineTask, error) { + key := op.Key(thunderDriver, "/drive/v1/task") + if !t.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + t.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]thunder.OfflineTask, error) { + ctx := context.Background() + tasks, err := thunderDriver.OfflineList(ctx, "") + if err != nil { + return nil, err + } + // 添加缓存 10s + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]thunder.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go new file mode 100644 index 00000000000..92e7a3e1dfb --- /dev/null +++ b/internal/offline_download/tool/add.go @@ -0,0 +1,144 @@ +package tool + +import ( + "context" + "net/url" + stdpath "path" + "path/filepath" + + _115 "github.com/alist-org/alist/v3/drivers/115" + "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/drivers/thunder" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/task" + "github.com/google/uuid" + "github.com/pkg/errors" +) + +type DeletePolicy string + +const ( + DeleteOnUploadSucceed DeletePolicy = "delete_on_upload_succeed" + DeleteOnUploadFailed DeletePolicy = "delete_on_upload_failed" + DeleteNever DeletePolicy = "delete_never" + DeleteAlways DeletePolicy = "delete_always" +) + +type AddURLArgs struct { + URL string + DstDirPath string + Tool string + DeletePolicy DeletePolicy +} + +func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, error) { + // check storage + storage, dstDirActualPath, err := op.GetStorageAndActualPath(args.DstDirPath) + if err != nil { + return nil, errors.WithMessage(err, "failed get storage") + } + // check is it could upload + if storage.Config().NoUpload { + return nil, errors.WithStack(errs.UploadNotSupported) + } + // check path is valid + obj, err := op.Get(ctx, storage, dstDirActualPath) + if err != nil { + if !errs.IsObjectNotFound(err) { + return nil, errors.WithMessage(err, "failed get object") + } + } else { + if !obj.IsDir() { + // can't add to a file + return nil, errors.WithStack(errs.NotFolder) + } + } + // try putting url + if args.Tool == "SimpleHttp" { + err = tryPutUrl(ctx, args.DstDirPath, args.URL) + if err == nil || !errors.Is(err, errs.NotImplement) { + return nil, err + } + } + + // get tool + tool, err := Tools.Get(args.Tool) + if err != nil { + return nil, errors.Wrapf(err, "failed get tool") + } + // check tool is ready + if !tool.IsReady() { + // try to init tool + if _, err := tool.Init(); err != nil { + return nil, errors.Wrapf(err, "failed init tool %s", args.Tool) + } + } + + uid := uuid.NewString() + tempDir := filepath.Join(conf.Conf.TempDir, args.Tool, uid) + deletePolicy := args.DeletePolicy + + // 如果当前 storage 是对应网盘,则直接下载到目标路径,无需转存 + switch args.Tool { + case "115 Cloud": + if _, ok := storage.(*_115.Pan115); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.Pan115TempDir), uid) + } + case "PikPak": + if _, ok := storage.(*pikpak.PikPak); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.PikPakTempDir), uid) + } + case "Thunder": + if _, ok := storage.(*thunder.Thunder); ok { + tempDir = args.DstDirPath + } else { + tempDir = filepath.Join(setting.GetStr(conf.ThunderTempDir), uid) + } + case "GuangYaPan": + if _, ok := storage.(*guangyapan.GuangYaPan); ok { + tempDir = args.DstDirPath + } else { + tempBase := setting.GetStr(conf.GuangYaPanTempDir) + if tempBase == "" { + return nil, errors.New("GuangYaPan temp dir is not set") + } + tempDir = filepath.Join(tempBase, uid) + } + } + + taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed + t := &DownloadTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + Url: args.URL, + DstDirPath: args.DstDirPath, + TempDir: tempDir, + DeletePolicy: deletePolicy, + Toolname: args.Tool, + tool: tool, + } + DownloadTaskManager.Add(t) + return t, nil +} + +func tryPutUrl(ctx context.Context, path, urlStr string) error { + var dstName string + u, err := url.Parse(urlStr) + if err == nil { + dstName = stdpath.Base(u.Path) + } else { + dstName = "UnnamedURL" + } + return fs.PutURL(ctx, path, dstName, urlStr) +} diff --git a/internal/offline_download/tool/base.go b/internal/offline_download/tool/base.go new file mode 100644 index 00000000000..b14169f8a83 --- /dev/null +++ b/internal/offline_download/tool/base.go @@ -0,0 +1,38 @@ +package tool + +import ( + "github.com/alist-org/alist/v3/internal/model" +) + +type AddUrlArgs struct { + Url string + UID string + TempDir string + Signal chan int +} + +type Status struct { + TotalBytes int64 + Progress float64 + NewGID string + Completed bool + Status string + Err error +} + +type Tool interface { + Name() string + // Items return the setting items the tool need + Items() []model.SettingItem + Init() (string, error) + IsReady() bool + // AddURL add an uri to download, return the task id + AddURL(args *AddUrlArgs) (string, error) + // Remove the download if task been canceled + Remove(task *DownloadTask) error + // Status return the status of the download task, if an error occurred, return the error in Status.Err + Status(task *DownloadTask) (*Status, error) + + // Run for simple http download + Run(task *DownloadTask) error +} diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go new file mode 100644 index 00000000000..c6ad09947e1 --- /dev/null +++ b/internal/offline_download/tool/download.go @@ -0,0 +1,183 @@ +package tool + +import ( + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/task" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" +) + +type DownloadTask struct { + task.TaskExtension + Url string `json:"url"` + DstDirPath string `json:"dst_dir_path"` + TempDir string `json:"temp_dir"` + DeletePolicy DeletePolicy `json:"delete_policy"` + Toolname string `json:"toolname"` + Status string `json:"-"` + Signal chan int `json:"-"` + GID string `json:"-"` + tool Tool + callStatusRetried int +} + +func (t *DownloadTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + if t.tool == nil { + tool, err := Tools.Get(t.Toolname) + if err != nil { + return errors.WithMessage(err, "failed get tool") + } + t.tool = tool + } + if err := t.tool.Run(t); !errs.IsNotSupportError(err) { + if err == nil { + return t.Transfer() + } + return err + } + t.Signal = make(chan int) + defer func() { + t.Signal = nil + }() + gid, err := t.tool.AddURL(&AddUrlArgs{ + Url: t.Url, + UID: t.ID, + TempDir: t.TempDir, + Signal: t.Signal, + }) + if err != nil { + return err + } + t.GID = gid + var ok bool +outer: + for { + select { + case <-t.CtxDone(): + err := t.tool.Remove(t) + return err + case <-t.Signal: + ok, err = t.Update() + if ok { + break outer + } + case <-time.After(time.Second * 3): + ok, err = t.Update() + if ok { + break outer + } + } + } + if err != nil { + return err + } + if t.tool.Name() == "Pikpak" { + return nil + } + if t.tool.Name() == "Thunder" { + return nil + } + if t.tool.Name() == "GuangYaPan" { + return nil + } + if t.tool.Name() == "115 Cloud" { + // hack for 115 + <-time.After(time.Second * 1) + err := t.tool.Remove(t) + if err != nil { + log.Errorln(err.Error()) + } + return nil + } + t.Status = "offline download completed, maybe transferring" + // hack for qBittorrent + if t.tool.Name() == "qBittorrent" { + seedTime := setting.GetInt(conf.QbittorrentSeedtime, 0) + if seedTime >= 0 { + t.Status = "offline download completed, waiting for seeding" + <-time.After(time.Minute * time.Duration(seedTime)) + err := t.tool.Remove(t) + if err != nil { + log.Errorln(err.Error()) + } + } + } + + if t.tool.Name() == "Transmission" { + // hack for transmission + seedTime := setting.GetInt(conf.TransmissionSeedtime, 0) + if seedTime >= 0 { + t.Status = "offline download completed, waiting for seeding" + <-time.After(time.Minute * time.Duration(seedTime)) + err := t.tool.Remove(t) + if err != nil { + log.Errorln(err.Error()) + } + } + } + return nil +} + +// Update download status, return true if download completed +func (t *DownloadTask) Update() (bool, error) { + info, err := t.tool.Status(t) + if err != nil { + t.callStatusRetried++ + log.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) + return false, nil + } + if t.callStatusRetried > 5 { + return true, errors.Errorf("failed to get status of %s, retried %d times", t.ID, t.callStatusRetried) + } + t.callStatusRetried = 0 + t.SetProgress(info.Progress) + t.SetTotalBytes(info.TotalBytes) + t.Status = fmt.Sprintf("[%s]: %s", t.tool.Name(), info.Status) + if info.NewGID != "" { + log.Debugf("followen by: %+v", info.NewGID) + t.GID = info.NewGID + return false, nil + } + // if download completed + if info.Completed { + err := t.Transfer() + return true, errors.WithMessage(err, "failed to transfer file") + } + // if download failed + if info.Err != nil { + return true, errors.Errorf("failed to download %s, error: %s", t.ID, info.Err.Error()) + } + return false, nil +} + +func (t *DownloadTask) Transfer() error { + toolName := t.tool.Name() + if toolName == "115 Cloud" || toolName == "PikPak" || toolName == "Thunder" || toolName == "GuangYaPan" { + // 如果不是直接下载到目标路径,则进行转存 + if t.TempDir != t.DstDirPath { + return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) + } + return nil + } + return transferStd(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy) +} + +func (t *DownloadTask) GetName() string { + return fmt.Sprintf("download %s to (%s)", t.Url, t.DstDirPath) +} + +func (t *DownloadTask) GetStatus() string { + return t.Status +} + +var DownloadTaskManager *tache.Manager[*DownloadTask] diff --git a/internal/offline_download/tool/tools.go b/internal/offline_download/tool/tools.go new file mode 100644 index 00000000000..4a31ac7f6b9 --- /dev/null +++ b/internal/offline_download/tool/tools.go @@ -0,0 +1,43 @@ +package tool + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "sort" +) + +var ( + Tools = make(ToolsManager) +) + +type ToolsManager map[string]Tool + +func (t ToolsManager) Get(name string) (Tool, error) { + if tool, ok := t[name]; ok { + return tool, nil + } + return nil, fmt.Errorf("tool %s not found", name) +} + +func (t ToolsManager) Add(tool Tool) { + t[tool.Name()] = tool +} + +func (t ToolsManager) Names() []string { + names := make([]string, 0, len(t)) + for name := range t { + if tool, err := t.Get(name); err == nil && tool.IsReady() { + names = append(names, name) + } + } + sort.Strings(names) + return names +} + +func (t ToolsManager) Items() []model.SettingItem { + var items []model.SettingItem + for _, tool := range t { + items = append(items, tool.Items()...) + } + return items +} diff --git a/internal/offline_download/tool/transfer.go b/internal/offline_download/tool/transfer.go new file mode 100644 index 00000000000..1d5ece612ce --- /dev/null +++ b/internal/offline_download/tool/transfer.go @@ -0,0 +1,275 @@ +package tool + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/xhofe/tache" + "net/http" + "os" + stdpath "path" + "path/filepath" + "time" +) + +type TransferTask struct { + task.TaskExtension + Status string `json:"-"` //don't save status to save space + SrcObjPath string `json:"src_obj_path"` + DstDirPath string `json:"dst_dir_path"` + SrcStorage driver.Driver `json:"-"` + DstStorage driver.Driver `json:"-"` + SrcStorageMp string `json:"src_storage_mp"` + DstStorageMp string `json:"dst_storage_mp"` + DeletePolicy DeletePolicy `json:"delete_policy"` +} + +func (t *TransferTask) Run() error { + t.ReinitCtx() + t.ClearEndTime() + t.SetStartTime(time.Now()) + defer func() { t.SetEndTime(time.Now()) }() + if t.SrcStorage == nil { + return transferStdPath(t) + } else { + return transferObjPath(t) + } +} + +func (t *TransferTask) GetName() string { + return fmt.Sprintf("transfer [%s](%s) to [%s](%s)", t.SrcStorageMp, t.SrcObjPath, t.DstStorageMp, t.DstDirPath) +} + +func (t *TransferTask) GetStatus() string { + return t.Status +} + +func (t *TransferTask) OnSucceeded() { + if t.DeletePolicy == DeleteOnUploadSucceed || t.DeletePolicy == DeleteAlways { + if t.SrcStorage == nil { + removeStdTemp(t) + } else { + removeObjTemp(t) + } + } +} + +func (t *TransferTask) OnFailed() { + if t.DeletePolicy == DeleteOnUploadFailed || t.DeletePolicy == DeleteAlways { + if t.SrcStorage == nil { + removeStdTemp(t) + } else { + removeObjTemp(t) + } + } +} + +var ( + TransferTaskManager *tache.Manager[*TransferTask] +) + +func transferStd(ctx context.Context, tempDir, dstDirPath string, deletePolicy DeletePolicy) error { + dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get dst storage") + } + entries, err := os.ReadDir(tempDir) + if err != nil { + return err + } + taskCreator, _ := ctx.Value("user").(*model.User) + for _, entry := range entries { + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + SrcObjPath: stdpath.Join(tempDir, entry.Name()), + DstDirPath: dstDirActualPath, + DstStorage: dstStorage, + DstStorageMp: dstStorage.GetStorage().MountPath, + DeletePolicy: deletePolicy, + } + TransferTaskManager.Add(t) + } + return nil +} + +func transferStdPath(t *TransferTask) error { + t.Status = "getting src object" + info, err := os.Stat(t.SrcObjPath) + if err != nil { + return err + } + if info.IsDir() { + t.Status = "src object is dir, listing objs" + entries, err := os.ReadDir(t.SrcObjPath) + if err != nil { + return err + } + for _, entry := range entries { + srcRawPath := stdpath.Join(t.SrcObjPath, entry.Name()) + dstObjPath := stdpath.Join(t.DstDirPath, info.Name()) + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: t.Creator, + }, + SrcObjPath: srcRawPath, + DstDirPath: dstObjPath, + DstStorage: t.DstStorage, + SrcStorageMp: t.SrcStorageMp, + DstStorageMp: t.DstStorageMp, + DeletePolicy: t.DeletePolicy, + } + TransferTaskManager.Add(t) + } + t.Status = "src object is dir, added all transfer tasks of files" + return nil + } + return transferStdFile(t) +} + +func transferStdFile(t *TransferTask) error { + rc, err := os.Open(t.SrcObjPath) + if err != nil { + return errors.Wrapf(err, "failed to open file %s", t.SrcObjPath) + } + info, err := rc.Stat() + if err != nil { + return errors.Wrapf(err, "failed to get file %s", t.SrcObjPath) + } + mimetype := utils.GetMimeType(t.SrcObjPath) + s := &stream.FileStream{ + Ctx: nil, + Obj: &model.Object{ + Name: filepath.Base(t.SrcObjPath), + Size: info.Size(), + Modified: info.ModTime(), + IsFolder: false, + }, + Reader: rc, + Mimetype: mimetype, + Closers: utils.NewClosers(rc), + } + t.SetTotalBytes(info.Size()) + return op.Put(t.Ctx(), t.DstStorage, t.DstDirPath, s, t.SetProgress) +} + +func removeStdTemp(t *TransferTask) { + info, err := os.Stat(t.SrcObjPath) + if err != nil || info.IsDir() { + return + } + if err := os.Remove(t.SrcObjPath); err != nil { + log.Errorf("failed to delete temp file %s, error: %s", t.SrcObjPath, err.Error()) + } +} + +func transferObj(ctx context.Context, tempDir, dstDirPath string, deletePolicy DeletePolicy) error { + srcStorage, srcObjActualPath, err := op.GetStorageAndActualPath(tempDir) + if err != nil { + return errors.WithMessage(err, "failed get src storage") + } + dstStorage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed get dst storage") + } + objs, err := op.List(ctx, srcStorage, srcObjActualPath, model.ListArgs{}) + if err != nil { + return errors.WithMessagef(err, "failed list src [%s] objs", tempDir) + } + taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed + for _, obj := range objs { + t := &TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: taskCreator, + }, + SrcObjPath: stdpath.Join(srcObjActualPath, obj.GetName()), + DstDirPath: dstDirActualPath, + SrcStorage: srcStorage, + DstStorage: dstStorage, + SrcStorageMp: srcStorage.GetStorage().MountPath, + DstStorageMp: dstStorage.GetStorage().MountPath, + DeletePolicy: deletePolicy, + } + TransferTaskManager.Add(t) + } + return nil +} + +func transferObjPath(t *TransferTask) error { + t.Status = "getting src object" + srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil { + return errors.WithMessagef(err, "failed get src [%s] file", t.SrcObjPath) + } + if srcObj.IsDir() { + t.Status = "src object is dir, listing objs" + objs, err := op.List(t.Ctx(), t.SrcStorage, t.SrcObjPath, model.ListArgs{}) + if err != nil { + return errors.WithMessagef(err, "failed list src [%s] objs", t.SrcObjPath) + } + for _, obj := range objs { + if utils.IsCanceled(t.Ctx()) { + return nil + } + srcObjPath := stdpath.Join(t.SrcObjPath, obj.GetName()) + dstObjPath := stdpath.Join(t.DstDirPath, srcObj.GetName()) + TransferTaskManager.Add(&TransferTask{ + TaskExtension: task.TaskExtension{ + Creator: t.Creator, + }, + SrcObjPath: srcObjPath, + DstDirPath: dstObjPath, + SrcStorage: t.SrcStorage, + DstStorage: t.DstStorage, + SrcStorageMp: t.SrcStorageMp, + DstStorageMp: t.DstStorageMp, + DeletePolicy: t.DeletePolicy, + }) + } + t.Status = "src object is dir, added all transfer tasks of objs" + return nil + } + return transferObjFile(t) +} + +func transferObjFile(t *TransferTask) error { + srcFile, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil { + return errors.WithMessagef(err, "failed get src [%s] file", t.SrcObjPath) + } + link, _, err := op.Link(t.Ctx(), t.SrcStorage, t.SrcObjPath, model.LinkArgs{ + Header: http.Header{}, + }) + if err != nil { + return errors.WithMessagef(err, "failed get [%s] link", t.SrcObjPath) + } + fs := stream.FileStream{ + Obj: srcFile, + Ctx: t.Ctx(), + } + // any link provided is seekable + ss, err := stream.NewSeekableStream(fs, link) + if err != nil { + return errors.WithMessagef(err, "failed get [%s] stream", t.SrcObjPath) + } + t.SetTotalBytes(srcFile.GetSize()) + return op.Put(t.Ctx(), t.DstStorage, t.DstDirPath, ss, t.SetProgress) +} + +func removeObjTemp(t *TransferTask) { + srcObj, err := op.Get(t.Ctx(), t.SrcStorage, t.SrcObjPath) + if err != nil || srcObj.IsDir() { + return + } + if err := op.Remove(t.Ctx(), t.SrcStorage, t.SrcObjPath); err != nil { + log.Errorf("failed to delete temp obj %s, error: %s", t.SrcObjPath, err.Error()) + } +} diff --git a/internal/offline_download/transmission/client.go b/internal/offline_download/transmission/client.go new file mode 100644 index 00000000000..ae136009875 --- /dev/null +++ b/internal/offline_download/transmission/client.go @@ -0,0 +1,177 @@ +package transmission + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/hekmon/transmissionrpc/v3" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type Transmission struct { + client *transmissionrpc.Client +} + +func (t *Transmission) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (t *Transmission) Name() string { + return "Transmission" +} + +func (t *Transmission) Items() []model.SettingItem { + // transmission settings + return []model.SettingItem{ + {Key: conf.TransmissionUri, Value: "http://localhost:9091/transmission/rpc", Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.TransmissionSeedtime, Value: "0", Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } +} + +func (t *Transmission) Init() (string, error) { + t.client = nil + uri := setting.GetStr(conf.TransmissionUri) + endpoint, err := url.Parse(uri) + if err != nil { + return "", errors.Wrap(err, "failed to init transmission client") + } + c, err := transmissionrpc.New(endpoint, nil) + if err != nil { + return "", errors.Wrap(err, "failed to init transmission client") + } + + ok, serverVersion, serverMinimumVersion, err := c.RPCVersion(context.Background()) + if err != nil { + return "", errors.Wrapf(err, "failed get transmission version") + } + + if !ok { + return "", fmt.Errorf("remote transmission RPC version (v%d) is incompatible with the transmission library (v%d): remote needs at least v%d", + serverVersion, transmissionrpc.RPCVersion, serverMinimumVersion) + } + + t.client = c + log.Infof("remote transmission RPC version (v%d) is compatible with our transmissionrpc library (v%d)\n", + serverVersion, transmissionrpc.RPCVersion) + log.Infof("using transmission version: %d", serverVersion) + return fmt.Sprintf("transmission version: %d", serverVersion), nil +} + +func (t *Transmission) IsReady() bool { + return t.client != nil +} + +func (t *Transmission) AddURL(args *tool.AddUrlArgs) (string, error) { + endpoint, err := url.Parse(args.Url) + if err != nil { + return "", errors.Wrap(err, "failed to parse transmission uri") + } + + rpcPayload := transmissionrpc.TorrentAddPayload{ + DownloadDir: &args.TempDir, + } + // http url for .torrent file + if endpoint.Scheme == "http" || endpoint.Scheme == "https" { + resp, err := http.Get(args.Url) + if err != nil { + return "", errors.Wrap(err, "failed to get .torrent file") + } + defer resp.Body.Close() + buffer := new(bytes.Buffer) + encoder := base64.NewEncoder(base64.StdEncoding, buffer) + // Stream file to the encoder + if _, err = utils.CopyWithBuffer(encoder, resp.Body); err != nil { + return "", errors.Wrap(err, "can't copy file content into the base64 encoder") + } + // Flush last bytes + if err = encoder.Close(); err != nil { + return "", errors.Wrap(err, "can't flush last bytes of the base64 encoder") + } + // Get the string form + b64 := buffer.String() + rpcPayload.MetaInfo = &b64 + } else { // magnet uri + rpcPayload.Filename = &args.Url + } + + torrent, err := t.client.TorrentAdd(context.TODO(), rpcPayload) + if err != nil { + return "", err + } + + if torrent.ID == nil { + return "", fmt.Errorf("failed get torrent ID") + } + gid := strconv.FormatInt(*torrent.ID, 10) + return gid, nil +} + +func (t *Transmission) Remove(task *tool.DownloadTask) error { + gid, err := strconv.ParseInt(task.GID, 10, 64) + if err != nil { + return err + } + err = t.client.TorrentRemove(context.TODO(), transmissionrpc.TorrentRemovePayload{ + IDs: []int64{gid}, + DeleteLocalData: false, + }) + return err +} + +func (t *Transmission) Status(task *tool.DownloadTask) (*tool.Status, error) { + gid, err := strconv.ParseInt(task.GID, 10, 64) + if err != nil { + return nil, err + } + infos, err := t.client.TorrentGetAllFor(context.TODO(), []int64{gid}) + if err != nil { + return nil, err + } + + if len(infos) < 1 { + return nil, fmt.Errorf("failed get status, wrong gid: %s", task.GID) + } + info := infos[0] + + s := &tool.Status{ + Completed: *info.IsFinished, + Err: err, + } + s.Progress = *info.PercentDone * 100 + s.TotalBytes = int64(*info.SizeWhenDone / 8) + + switch *info.Status { + case transmissionrpc.TorrentStatusCheckWait, + transmissionrpc.TorrentStatusDownloadWait, + transmissionrpc.TorrentStatusCheck, + transmissionrpc.TorrentStatusDownload, + transmissionrpc.TorrentStatusIsolated: + s.Status = "[transmission] " + info.Status.String() + case transmissionrpc.TorrentStatusSeedWait, + transmissionrpc.TorrentStatusSeed: + s.Completed = true + case transmissionrpc.TorrentStatusStopped: + s.Err = errors.Errorf("[transmission] failed to download %s, status: %s, error: %s", task.GID, info.Status.String(), *info.ErrorString) + default: + s.Err = errors.Errorf("[transmission] unknown status occurred downloading %s, err: %s", task.GID, *info.ErrorString) + } + return s, nil +} + +var _ tool.Tool = (*Transmission)(nil) + +func init() { + tool.Tools.Add(&Transmission{}) +} diff --git a/internal/op/archive.go b/internal/op/archive.go new file mode 100644 index 00000000000..38b870c70e3 --- /dev/null +++ b/internal/op/archive.go @@ -0,0 +1,518 @@ +package op + +import ( + "context" + stderrors "errors" + "fmt" + "io" + stdpath "path" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/stream" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/singleflight" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +var archiveMetaCache = cache.NewMemCache(cache.WithShards[*model.ArchiveMetaProvider](64)) +var archiveMetaG singleflight.Group[*model.ArchiveMetaProvider] + +func GetArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (*model.ArchiveMetaProvider, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + key := Key(storage, path) + if !args.Refresh { + if meta, ok := archiveMetaCache.Get(key); ok { + log.Debugf("use cache when get %s archive meta", path) + return meta, nil + } + } + fn := func() (*model.ArchiveMetaProvider, error) { + _, m, err := getArchiveMeta(ctx, storage, path, args) + if err != nil { + return nil, errors.Wrapf(err, "failed to get %s archive met: %+v", path, err) + } + if m.Expiration != nil { + archiveMetaCache.Set(key, m, cache.WithEx[*model.ArchiveMetaProvider](*m.Expiration)) + } + return m, nil + } + if storage.Config().OnlyLocal { + meta, err := fn() + return meta, err + } + meta, err, _ := archiveMetaG.Do(key, fn) + return meta, err +} + +func GetArchiveToolAndStream(ctx context.Context, storage driver.Driver, path string, args model.LinkArgs) (model.Obj, tool.Tool, []*stream.SeekableStream, error) { + l, obj, err := Link(ctx, storage, path, args) + if err != nil { + return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] link", path) + } + baseName, ext, found := strings.Cut(obj.GetName(), ".") + if !found { + if l.MFile != nil { + _ = l.MFile.Close() + } + if l.RangeReadCloser != nil { + _ = l.RangeReadCloser.Close() + } + return nil, nil, nil, errors.Errorf("failed get archive tool: the obj does not have an extension.") + } + partExt, t, err := tool.GetArchiveTool("." + ext) + if err != nil { + var e error + partExt, t, e = tool.GetArchiveTool(stdpath.Ext(obj.GetName())) + if e != nil { + if l.MFile != nil { + _ = l.MFile.Close() + } + if l.RangeReadCloser != nil { + _ = l.RangeReadCloser.Close() + } + return nil, nil, nil, errors.WithMessagef(stderrors.Join(err, e), "failed get archive tool: %s", ext) + } + } + ss, err := stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: obj}, l) + if err != nil { + if l.MFile != nil { + _ = l.MFile.Close() + } + if l.RangeReadCloser != nil { + _ = l.RangeReadCloser.Close() + } + return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) + } + ret := []*stream.SeekableStream{ss} + if partExt == nil { + return obj, t, ret, nil + } else { + index := partExt.SecondPartIndex + dir := stdpath.Dir(path) + for { + p := stdpath.Join(dir, baseName+fmt.Sprintf(partExt.PartFileFormat, index)) + var o model.Obj + l, o, err = Link(ctx, storage, p, args) + if err != nil { + break + } + ss, err = stream.NewSeekableStream(stream.FileStream{Ctx: ctx, Obj: o}, l) + if err != nil { + if l.MFile != nil { + _ = l.MFile.Close() + } + if l.RangeReadCloser != nil { + _ = l.RangeReadCloser.Close() + } + for _, s := range ret { + _ = s.Close() + } + return nil, nil, nil, errors.WithMessagef(err, "failed get [%s] stream", path) + } + ret = append(ret, ss) + index++ + } + return obj, t, ret, nil + } +} + +func getArchiveMeta(ctx context.Context, storage driver.Driver, path string, args model.ArchiveMetaArgs) (model.Obj, *model.ArchiveMetaProvider, error) { + storageAr, ok := storage.(driver.ArchiveReader) + if ok { + obj, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file") + } + if obj.IsDir() { + return nil, nil, errors.WithStack(errs.NotFile) + } + meta, err := storageAr.GetArchiveMeta(ctx, obj, args.ArchiveArgs) + if !errors.Is(err, errs.NotImplement) { + archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: true} + if meta != nil && meta.GetTree() != nil { + archiveMetaProvider.Sort = &storage.GetStorage().Sort + } + if !storage.Config().NoCache { + Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) + archiveMetaProvider.Expiration = &Expiration + } + return obj, archiveMetaProvider, err + } + } + obj, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + if err != nil { + return nil, nil, err + } + defer func() { + var e error + for _, s := range ss { + e = stderrors.Join(e, s.Close()) + } + if e != nil { + log.Errorf("failed to close file streamer, %v", e) + } + }() + meta, err := t.GetMeta(ss, args.ArchiveArgs) + if err != nil { + return nil, nil, err + } + archiveMetaProvider := &model.ArchiveMetaProvider{ArchiveMeta: meta, DriverProviding: false} + if meta.GetTree() != nil { + archiveMetaProvider.Sort = &storage.GetStorage().Sort + } + if !storage.Config().NoCache { + Expiration := time.Minute * time.Duration(storage.GetStorage().CacheExpiration) + archiveMetaProvider.Expiration = &Expiration + } else if ss[0].Link.MFile == nil { + // alias、crypt 驱动 + archiveMetaProvider.Expiration = ss[0].Link.Expiration + } + return obj, archiveMetaProvider, err +} + +var archiveListCache = cache.NewMemCache(cache.WithShards[[]model.Obj](64)) +var archiveListG singleflight.Group[[]model.Obj] + +func ListArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) ([]model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + metaKey := Key(storage, path) + key := stdpath.Join(metaKey, args.InnerPath) + if !args.Refresh { + if files, ok := archiveListCache.Get(key); ok { + log.Debugf("use cache when list archive [%s]%s", path, args.InnerPath) + return files, nil + } + // if meta, ok := archiveMetaCache.Get(metaKey); ok { + // log.Debugf("use meta cache when list archive [%s]%s", path, args.InnerPath) + // return getChildrenFromArchiveMeta(meta, args.InnerPath) + // } + } + objs, err, _ := archiveListG.Do(key, func() ([]model.Obj, error) { + obj, files, err := listArchive(ctx, storage, path, args) + if err != nil { + return nil, errors.Wrapf(err, "failed to list archive [%s]%s: %+v", path, args.InnerPath, err) + } + // set path + for _, f := range files { + if s, ok := f.(model.SetPath); ok && f.GetPath() == "" && obj.GetPath() != "" { + s.SetPath(stdpath.Join(obj.GetPath(), args.InnerPath, f.GetName())) + } + } + // warp obj name + model.WrapObjsName(files) + // sort objs + if storage.Config().LocalSort { + model.SortFiles(files, storage.GetStorage().OrderBy, storage.GetStorage().OrderDirection) + } + model.ExtractFolder(files, storage.GetStorage().ExtractFolder) + if !storage.Config().NoCache { + if len(files) > 0 { + log.Debugf("set cache: %s => %+v", key, files) + archiveListCache.Set(key, files, cache.WithEx[[]model.Obj](time.Minute*time.Duration(storage.GetStorage().CacheExpiration))) + } else { + log.Debugf("del cache: %s", key) + archiveListCache.Del(key) + } + } + return files, nil + }) + return objs, err +} + +func _listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { + storageAr, ok := storage.(driver.ArchiveReader) + if ok { + obj, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file") + } + if obj.IsDir() { + return nil, nil, errors.WithStack(errs.NotFile) + } + files, err := storageAr.ListArchive(ctx, obj, args.ArchiveInnerArgs) + if !errors.Is(err, errs.NotImplement) { + return obj, files, err + } + } + obj, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + if err != nil { + return nil, nil, err + } + defer func() { + var e error + for _, s := range ss { + e = stderrors.Join(e, s.Close()) + } + if e != nil { + log.Errorf("failed to close file streamer, %v", e) + } + }() + files, err := t.List(ss, args.ArchiveInnerArgs) + return obj, files, err +} + +func listArchive(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, []model.Obj, error) { + obj, files, err := _listArchive(ctx, storage, path, args) + if errors.Is(err, errs.NotSupport) { + var meta model.ArchiveMeta + meta, err = GetArchiveMeta(ctx, storage, path, model.ArchiveMetaArgs{ + ArchiveArgs: args.ArchiveArgs, + Refresh: args.Refresh, + }) + if err != nil { + return nil, nil, err + } + files, err = getChildrenFromArchiveMeta(meta, args.InnerPath) + if err != nil { + return nil, nil, err + } + } + if err == nil && obj == nil { + obj, err = GetUnwrap(ctx, storage, path) + } + if err != nil { + return nil, nil, err + } + return obj, files, err +} + +func getChildrenFromArchiveMeta(meta model.ArchiveMeta, innerPath string) ([]model.Obj, error) { + obj := meta.GetTree() + if obj == nil { + return nil, errors.WithStack(errs.NotImplement) + } + dirs := splitPath(innerPath) + for _, dir := range dirs { + var next model.ObjTree + for _, c := range obj { + if c.GetName() == dir { + next = c + break + } + } + if next == nil { + return nil, errors.WithStack(errs.ObjectNotFound) + } + if !next.IsDir() || next.GetChildren() == nil { + return nil, errors.WithStack(errs.NotFolder) + } + obj = next.GetChildren() + } + return utils.SliceConvert(obj, func(src model.ObjTree) (model.Obj, error) { + return src, nil + }) +} + +func splitPath(path string) []string { + var parts []string + for { + dir, file := stdpath.Split(path) + if file == "" { + break + } + parts = append([]string{file}, parts...) + path = strings.TrimSuffix(dir, "/") + } + return parts +} + +func ArchiveGet(ctx context.Context, storage driver.Driver, path string, args model.ArchiveListArgs) (model.Obj, model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + path = utils.FixAndCleanPath(path) + af, err := GetUnwrap(ctx, storage, path) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed to get file") + } + if af.IsDir() { + return nil, nil, errors.WithStack(errs.NotFile) + } + if g, ok := storage.(driver.ArchiveGetter); ok { + obj, err := g.ArchiveGet(ctx, af, args.ArchiveInnerArgs) + if err == nil { + return af, model.WrapObjName(obj), nil + } + } + + if utils.PathEqual(args.InnerPath, "/") { + return af, &model.ObjWrapName{ + Name: RootName, + Obj: &model.Object{ + Name: af.GetName(), + Path: af.GetPath(), + ID: af.GetID(), + Size: af.GetSize(), + Modified: af.ModTime(), + IsFolder: true, + }, + }, nil + } + + innerDir, name := stdpath.Split(args.InnerPath) + args.InnerPath = strings.TrimSuffix(innerDir, "/") + files, err := ListArchive(ctx, storage, path, args) + if err != nil { + return nil, nil, errors.WithMessage(err, "failed get parent list") + } + for _, f := range files { + if f.GetName() == name { + return af, f, nil + } + } + return nil, nil, errors.WithStack(errs.ObjectNotFound) +} + +type extractLink struct { + Link *model.Link + Obj model.Obj +} + +var extractCache = cache.NewMemCache(cache.WithShards[*extractLink](16)) +var extractG singleflight.Group[*extractLink] + +func DriverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*model.Link, model.Obj, error) { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return nil, nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + key := stdpath.Join(Key(storage, path), args.InnerPath) + if link, ok := extractCache.Get(key); ok { + return link.Link, link.Obj, nil + } else if link, ok := extractCache.Get(key + ":" + args.IP); ok { + return link.Link, link.Obj, nil + } + fn := func() (*extractLink, error) { + link, err := driverExtract(ctx, storage, path, args) + if err != nil { + return nil, errors.Wrapf(err, "failed extract archive") + } + if link.Link.Expiration != nil { + if link.Link.IPCacheKey { + key = key + ":" + args.IP + } + extractCache.Set(key, link, cache.WithEx[*extractLink](*link.Link.Expiration)) + } + return link, nil + } + if storage.Config().OnlyLocal { + link, err := fn() + if err != nil { + return nil, nil, err + } + return link.Link, link.Obj, nil + } + link, err, _ := extractG.Do(key, fn) + if err != nil { + return nil, nil, err + } + return link.Link, link.Obj, err +} + +func driverExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (*extractLink, error) { + storageAr, ok := storage.(driver.ArchiveReader) + if !ok { + return nil, errs.DriverExtractNotSupported + } + archiveFile, extracted, err := ArchiveGet(ctx, storage, path, model.ArchiveListArgs{ + ArchiveInnerArgs: args, + Refresh: false, + }) + if err != nil { + return nil, errors.WithMessage(err, "failed to get file") + } + if extracted.IsDir() { + return nil, errors.WithStack(errs.NotFile) + } + link, err := storageAr.Extract(ctx, archiveFile, args) + return &extractLink{Link: link, Obj: extracted}, err +} + +type streamWithParent struct { + rc io.ReadCloser + parents []*stream.SeekableStream +} + +func (s *streamWithParent) Read(p []byte) (int, error) { + return s.rc.Read(p) +} + +func (s *streamWithParent) Close() error { + err := s.rc.Close() + for _, ss := range s.parents { + err = stderrors.Join(err, ss.Close()) + } + return err +} + +func InternalExtract(ctx context.Context, storage driver.Driver, path string, args model.ArchiveInnerArgs) (io.ReadCloser, int64, error) { + _, t, ss, err := GetArchiveToolAndStream(ctx, storage, path, args.LinkArgs) + if err != nil { + return nil, 0, err + } + rc, size, err := t.Extract(ss, args) + if err != nil { + var e error + for _, s := range ss { + e = stderrors.Join(e, s.Close()) + } + if e != nil { + log.Errorf("failed to close file streamer, %v", e) + err = stderrors.Join(err, e) + } + return nil, 0, err + } + return &streamWithParent{rc: rc, parents: ss}, size, nil +} + +func ArchiveDecompress(ctx context.Context, storage driver.Driver, srcPath, dstDirPath string, args model.ArchiveDecompressArgs, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + srcPath = utils.FixAndCleanPath(srcPath) + dstDirPath = utils.FixAndCleanPath(dstDirPath) + srcObj, err := GetUnwrap(ctx, storage, srcPath) + if err != nil { + return errors.WithMessage(err, "failed to get src object") + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessage(err, "failed to get dst dir") + } + + switch s := storage.(type) { + case driver.ArchiveDecompressResult: + var newObjs []model.Obj + newObjs, err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) + if err == nil { + if newObjs != nil && len(newObjs) > 0 { + for _, newObj := range newObjs { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.ArchiveDecompress: + err = s.ArchiveDecompress(ctx, srcObj, dstDir, args) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errs.NotImplement + } + return errors.WithStack(err) +} diff --git a/internal/op/driver.go b/internal/op/driver.go index 66d1ae3ce9c..4099fbbf5dd 100644 --- a/internal/op/driver.go +++ b/internal/op/driver.go @@ -93,6 +93,17 @@ func getMainItems(config driver.Config) []driver.Item { Required: true, }, }...) + if config.ProxyRangeOption { + item := driver.Item{ + Name: "proxy_range", + Type: conf.TypeBool, + Help: "Need to enable proxy", + } + if config.Name == "139Yun" { + item.Default = "true" + } + items = append(items, item) + } } else { items = append(items, driver.Item{ Name: "webdav_policy", @@ -106,6 +117,11 @@ func getMainItems(config driver.Config) []driver.Item { Name: "down_proxy_url", Type: conf.TypeText, }) + items = append(items, driver.Item{ + Name: "down_proxy_sign", + Type: conf.TypeBool, + Default: "true", + }) if config.LocalSort { items = append(items, []driver.Item{{ Name: "order_by", @@ -122,6 +138,12 @@ func getMainItems(config driver.Config) []driver.Item { Type: conf.TypeSelect, Options: "front,back", }) + items = append(items, driver.Item{ + Name: "disable_index", + Type: conf.TypeBool, + Default: "false", + Required: true, + }) items = append(items, driver.Item{ Name: "enable_sign", Type: conf.TypeBool, diff --git a/internal/op/fs.go b/internal/op/fs.go index 8ee6993e091..64e993356bc 100644 --- a/internal/op/fs.go +++ b/internal/op/fs.go @@ -3,12 +3,14 @@ package op import ( "context" stdpath "path" + "slices" "time" "github.com/Xhofe/go-cache" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/generic_sync" "github.com/alist-org/alist/v3/pkg/singleflight" "github.com/alist-org/alist/v3/pkg/utils" @@ -25,6 +27,12 @@ func updateCacheObj(storage driver.Driver, path string, oldObj model.Obj, newObj key := Key(storage, path) objs, ok := listCache.Get(key) if ok { + for i, obj := range objs { + if obj.GetName() == newObj.GetName() { + objs = slices.Delete(objs, i, i+1) + break + } + } for i, obj := range objs { if obj.GetName() == oldObj.GetName() { objs[i] = newObj @@ -100,14 +108,14 @@ func Key(storage driver.Driver, path string) string { } // List files in storage, not contains virtual file -func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs, refresh ...bool) ([]model.Obj, error) { +func List(ctx context.Context, storage driver.Driver, path string, args model.ListArgs) ([]model.Obj, error) { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return nil, errors.Errorf("storage not init: %s", storage.GetStorage().Status) } path = utils.FixAndCleanPath(path) log.Debugf("op.List %s", path) key := Key(storage, path) - if !utils.IsBool(refresh...) { + if !args.Refresh { if files, ok := listCache.Get(key); ok { log.Debugf("use cache when list %s", path) return files, nil @@ -136,9 +144,7 @@ func List(ctx context.Context, storage driver.Driver, path string, args model.Li model.WrapObjsName(files) // call hooks go func(reqPath string, files []model.Obj) { - for _, hook := range objsUpdateHooks { - hook(reqPath, files) - } + HandleObjsUpdateHook(reqPath, files) }(utils.GetFullPath(storage.GetStorage().MountPath, path), files) // sort objs @@ -177,30 +183,32 @@ func Get(ctx context.Context, storage driver.Driver, path string) (model.Obj, er // is root folder if utils.PathEqual(path, "/") { var rootObj model.Obj - switch r := storage.GetAddition().(type) { - case driver.IRootId: - rootObj = &model.Object{ - ID: r.GetRootId(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, - } - case driver.IRootPath: - rootObj = &model.Object{ - Path: r.GetRootPath(), - Name: RootName, - Size: 0, - Modified: storage.GetStorage().Modified, - IsFolder: true, + if getRooter, ok := storage.(driver.GetRooter); ok { + obj, err := getRooter.GetRoot(ctx) + if err != nil { + return nil, errors.WithMessage(err, "failed get root obj") } - default: - if storage, ok := storage.(driver.GetRooter); ok { - obj, err := storage.GetRoot(ctx) - if err != nil { - return nil, errors.WithMessage(err, "failed get root obj") + rootObj = obj + } else { + switch r := storage.GetAddition().(type) { + case driver.IRootId: + rootObj = &model.Object{ + ID: r.GetRootId(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, + } + case driver.IRootPath: + rootObj = &model.Object{ + Path: r.GetRootPath(), + Name: RootName, + Size: 0, + Modified: storage.GetStorage().Modified, + IsFolder: true, } - rootObj = obj + default: + return nil, errors.Errorf("please implement IRootPath or IRootId or GetRooter method") } } if rootObj == nil { @@ -267,6 +275,12 @@ func Link(ctx context.Context, storage driver.Driver, path string, args model.Li } return link, nil } + + if storage.Config().OnlyLocal { + link, err := fn() + return link, file, err + } + link, err, _ := linkG.Do(key, fn) return link, file, err } @@ -464,6 +478,9 @@ func Remove(ctx context.Context, storage driver.Driver, path string) error { if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { return errors.Errorf("storage not init: %s", storage.GetStorage().Status) } + if utils.PathEqual(path, "/") { + return errors.New("delete root folder is not allowed, please goto the manage page to delete the storage instead") + } path = utils.FixAndCleanPath(path) rawObj, err := Get(ctx, storage, path) if err != nil { @@ -501,6 +518,12 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod log.Errorf("failed to close file streamer, %v", err) } }() + // UrlTree PUT + if storage.GetStorage().Driver == "UrlTree" { + var link string + dstDirPath, link = urlTreeSplitLineFormPath(stdpath.Join(dstDirPath, file.GetName())) + file = &stream.FileStream{Obj: &model.Object{Name: link}} + } // if file exist and size = 0, delete it dstDirPath = utils.FixAndCleanPath(dstDirPath) dstPath := stdpath.Join(dstDirPath, file.GetName()) @@ -534,7 +557,7 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } // if up is nil, set a default to prevent panic if up == nil { - up = func(p int) {} + up = func(p float64) {} } switch s := storage.(type) { @@ -577,3 +600,43 @@ func Put(ctx context.Context, storage driver.Driver, dstDirPath string, file mod } return errors.WithStack(err) } + +func PutURL(ctx context.Context, storage driver.Driver, dstDirPath, dstName, url string, lazyCache ...bool) error { + if storage.Config().CheckStatus && storage.GetStorage().Status != WORK { + return errors.Errorf("storage not init: %s", storage.GetStorage().Status) + } + dstDirPath = utils.FixAndCleanPath(dstDirPath) + _, err := GetUnwrap(ctx, storage, stdpath.Join(dstDirPath, dstName)) + if err == nil { + return errors.New("obj already exists") + } + err = MakeDir(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessagef(err, "failed to put url") + } + dstDir, err := GetUnwrap(ctx, storage, dstDirPath) + if err != nil { + return errors.WithMessagef(err, "failed to put url") + } + switch s := storage.(type) { + case driver.PutURLResult: + var newObj model.Obj + newObj, err = s.PutURL(ctx, dstDir, dstName, url) + if err == nil { + if newObj != nil { + addCacheObj(storage, dstDirPath, model.WrapObjName(newObj)) + } else if !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + } + case driver.PutURL: + err = s.PutURL(ctx, dstDir, dstName, url) + if err == nil && !utils.IsBool(lazyCache...) { + ClearCache(storage, dstDirPath) + } + default: + return errs.NotImplement + } + log.Debugf("put url [%s](%s) done", dstName, url) + return errors.WithStack(err) +} diff --git a/internal/op/hook.go b/internal/op/hook.go index e37e52df269..f08966c4ade 100644 --- a/internal/op/hook.go +++ b/internal/op/hook.go @@ -2,6 +2,7 @@ package op import ( "regexp" + "strconv" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -78,6 +79,22 @@ var settingItemHooks = map[string]SettingItemHook{ log.Debugf("filename char mapping: %+v", conf.FilenameCharMap) return nil }, + conf.IgnoreDirectLinkParams: func(item *model.SettingItem) error { + conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",") + return nil + }, + conf.DefaultRole: func(item *model.SettingItem) error { + v := strings.TrimSpace(item.Value) + if v == "" { + return nil + } + id, err := strconv.Atoi(v) + if err != nil { + return errors.WithStack(err) + } + _, err = GetRole(uint(id)) + return err + }, } func RegisterSettingItemHook(key string, hook SettingItemHook) { diff --git a/internal/op/label.go b/internal/op/label.go new file mode 100644 index 00000000000..7e913edfa8d --- /dev/null +++ b/internal/op/label.go @@ -0,0 +1,24 @@ +package op + +import ( + "context" + "github.com/alist-org/alist/v3/internal/db" + "github.com/pkg/errors" +) + +func DeleteLabelById(ctx context.Context, id, userId uint) error { + _, err := db.GetLabelById(id) + if err != nil { + return errors.WithMessage(err, "failed get label") + } + + if db.GetLabelFileBinDingByLabelIdExists(id, userId) { + return errors.New("label have binding relationships") + } + + // delete the label in the database + if err := db.DeleteLabelById(id); err != nil { + return errors.WithMessage(err, "failed delete label in database") + } + return nil +} diff --git a/internal/op/label_file_binding.go b/internal/op/label_file_binding.go new file mode 100644 index 00000000000..2802f0c0b38 --- /dev/null +++ b/internal/op/label_file_binding.go @@ -0,0 +1,195 @@ +package op + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "strconv" + "strings" + "time" +) + +type CreateLabelFileBinDingReq struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + LabelIds string `json:"label_ids"` + LabelIDs []uint64 `json:"labelIdList"` +} + +type ObjLabelResp struct { + Id string `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + LabelList []model.Label `json:"label_list"` +} + +func GetLabelByFileName(userId uint, fileName string) ([]model.Label, error) { + labelIds, err := db.GetLabelIds(userId, fileName) + if err != nil { + return nil, errors.WithMessage(err, "failed get label_file_binding") + } + var labels []model.Label + if len(labelIds) > 0 { + if labels, err = db.GetLabelByIds(labelIds); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + } + return labels, nil +} + +func GetLabelsByFileNamesPublic(fileNames []string) (map[string][]model.Label, error) { + return db.GetLabelsByFileNamesPublic(fileNames) +} + +func CreateLabelFileBinDing(req CreateLabelFileBinDingReq, userId uint) error { + if err := db.DelLabelFileBinDingByFileName(userId, req.Name); err != nil { + return errors.WithMessage(err, "failed del label_file_bin_ding in database") + } + + ids, err := collectLabelIDs(req) + if err != nil { + return err + } + if len(ids) == 0 { + return nil + } + + for _, id := range ids { + if err = db.CreateLabelFileBinDing(req.Name, uint(id), userId); err != nil { + return errors.WithMessage(err, "failed labels in database") + } + } + + if !db.GetFileByNameExists(req.Name) { + objFile := model.ObjFile{ + Id: req.Id, + UserId: userId, + Path: req.Path, + Name: req.Name, + Size: req.Size, + IsDir: req.IsDir, + Modified: req.Modified, + Created: req.Created, + Sign: req.Sign, + Thumb: req.Thumb, + Type: req.Type, + HashInfoStr: req.HashInfoStr, + } + if err := db.CreateObjFile(objFile); err != nil { + return errors.WithMessage(err, "failed file in database") + } + } + return nil +} + +func GetFileByLabel(userId uint, labelId string) (result []ObjLabelResp, err error) { + labelMap := strings.Split(labelId, ",") + var labelIds []uint + var labelsFile []model.LabelFileBinding + var labels []model.Label + var labelsFileMap = make(map[string][]model.Label) + var labelsMap = make(map[uint]model.Label) + if labelIds, err = StringSliceToUintSlice(labelMap); err != nil { + return nil, errors.WithMessage(err, "failed string to uint err") + } + //查询标签信息 + if labels, err = db.GetLabelByIds(labelIds); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + for _, val := range labels { + labelsMap[val.ID] = val + } + //查询标签对应文件名列表 + if labelsFile, err = db.GetLabelFileBinDingByLabelId(labelIds, userId); err != nil { + return nil, errors.WithMessage(err, "failed labels in database") + } + for _, value := range labelsFile { + var labelTemp model.Label + labelTemp = labelsMap[value.LabelId] + labelsFileMap[value.FileName] = append(labelsFileMap[value.FileName], labelTemp) + } + for index, v := range labelsFileMap { + objFile, err := db.GetFileByName(index, userId) + if err != nil { + return nil, errors.WithMessage(err, "failed GetFileByName in database") + } + objLabel := ObjLabelResp{ + Id: objFile.Id, + Path: objFile.Path, + Name: objFile.Name, + Size: objFile.Size, + IsDir: objFile.IsDir, + Modified: objFile.Modified, + Created: objFile.Created, + Sign: objFile.Sign, + Thumb: objFile.Thumb, + Type: objFile.Type, + HashInfoStr: objFile.HashInfoStr, + LabelList: v, + } + result = append(result, objLabel) + } + return result, nil +} + +func StringSliceToUintSlice(strSlice []string) ([]uint, error) { + uintSlice := make([]uint, len(strSlice)) + for i, str := range strSlice { + // 使用strconv.ParseUint将字符串转换为uint64 + uint64Value, err := strconv.ParseUint(str, 10, 64) + if err != nil { + return nil, err // 如果转换失败,返回错误 + } + // 将uint64值转换为uint(注意:这里可能存在精度损失,如果uint64值超出了uint的范围) + uintSlice[i] = uint(uint64Value) + } + return uintSlice, nil +} + +func RestoreLabelFileBindings(bindings []model.LabelFileBinding, keepIDs bool, override bool) error { + return db.RestoreLabelFileBindings(bindings, keepIDs, override) +} + +func collectLabelIDs(req CreateLabelFileBinDingReq) ([]uint64, error) { + if len(req.LabelIDs) > 0 { + return req.LabelIDs, nil + } + s := strings.TrimSpace(req.LabelIds) + if s == "" { + return nil, nil + } + replacer := strings.NewReplacer(",", ",", "、", ",", ";", ",", ";", ",") + s = replacer.Replace(s) + parts := strings.Split(s, ",") + ids := make([]uint64, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + id, err := strconv.ParseUint(p, 10, 64) + if err != nil { + return nil, fmt.Errorf("invalid label ID '%s': %v", p, err) + } + ids = append(ids, id) + } + return ids, nil +} diff --git a/internal/op/meta.go b/internal/op/meta.go index 930f49634c3..29146fcc72c 100644 --- a/internal/op/meta.go +++ b/internal/op/meta.go @@ -2,9 +2,11 @@ package op import ( stdpath "path" + "strconv" "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" @@ -19,6 +21,17 @@ var metaCache = cache.NewMemCache(cache.WithShards[*model.Meta](2)) // metaG maybe not needed var metaG singleflight.Group[*model.Meta] +const ( + metaCacheExpiration = time.Hour + defaultMetaNotFoundCacheSec = 60 +) + +func init() { + RegisterSettingChangingCallback(func() { + metaCache.Clear() + }) +} + func GetNearestMeta(path string) (*model.Meta, error) { return getNearestMeta(utils.FixAndCleanPath(path)) } @@ -51,17 +64,34 @@ func getMetaByPath(path string) (*model.Meta, error) { _meta, err := db.GetMetaByPath(path) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { - metaCache.Set(path, nil) + if ex := metaNotFoundCacheExpiration(); ex > 0 { + metaCache.Set(path, nil, cache.WithEx[*model.Meta](ex)) + } return nil, errs.MetaNotFound } return nil, err } - metaCache.Set(path, _meta, cache.WithEx[*model.Meta](time.Hour)) + metaCache.Set(path, _meta, cache.WithEx[*model.Meta](metaCacheExpiration)) return _meta, nil }) return meta, err } +func metaNotFoundCacheExpiration() time.Duration { + item, err := GetSettingItemByKey(conf.MetaNotFoundCacheExpire) + if err != nil || item == nil { + return time.Second * defaultMetaNotFoundCacheSec + } + seconds, err := strconv.Atoi(item.Value) + if err != nil { + return time.Second * defaultMetaNotFoundCacheSec + } + if seconds <= 0 { + return 0 + } + return time.Second * time.Duration(seconds) +} + func DeleteMetaById(id uint) error { old, err := db.GetMetaById(id) if err != nil { @@ -78,6 +108,7 @@ func UpdateMeta(u *model.Meta) error { return err } metaCache.Del(old.Path) + metaCache.Del(u.Path) return db.UpdateMeta(u) } diff --git a/internal/op/path.go b/internal/op/path.go index 27f7e183228..912a00008e1 100644 --- a/internal/op/path.go +++ b/internal/op/path.go @@ -2,6 +2,7 @@ package op import ( "github.com/alist-org/alist/v3/internal/errs" + stdpath "path" "strings" "github.com/alist-org/alist/v3/internal/driver" @@ -27,3 +28,30 @@ func GetStorageAndActualPath(rawPath string) (storage driver.Driver, actualPath actualPath = utils.FixAndCleanPath(strings.TrimPrefix(rawPath, mountPath)) return } + +// urlTreeSplitLineFormPath 分割path中分割真实路径和UrlTree定义字符串 +func urlTreeSplitLineFormPath(path string) (pp string, file string) { + // url.PathUnescape 会移除 // ,手动加回去 + path = strings.Replace(path, "https:/", "https://", 1) + path = strings.Replace(path, "http:/", "http://", 1) + if strings.Contains(path, ":https:/") || strings.Contains(path, ":http:/") { + // URL-Tree模式 /url_tree_drivr/file_name[:size[:time]]:https://example.com/file + fPath := strings.SplitN(path, ":", 2)[0] + pp, _ = stdpath.Split(fPath) + file = path[len(pp):] + } else if strings.Contains(path, "/https:/") || strings.Contains(path, "/http:/") { + // URL-Tree模式 /url_tree_drivr/https://example.com/file + index := strings.Index(path, "/http://") + if index == -1 { + index = strings.Index(path, "/https://") + } + pp = path[:index] + file = path[index+1:] + } else { + pp, file = stdpath.Split(path) + } + if pp == "" { + pp = "/" + } + return +} diff --git a/internal/op/role.go b/internal/op/role.go new file mode 100644 index 00000000000..bd874eeed19 --- /dev/null +++ b/internal/op/role.go @@ -0,0 +1,225 @@ +package op + +import ( + "fmt" + "strconv" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/singleflight" + "github.com/alist-org/alist/v3/pkg/utils" +) + +var roleCache = cache.NewMemCache[*model.Role](cache.WithShards[*model.Role](2)) +var roleG singleflight.Group[*model.Role] + +func init() { + model.FetchRole = GetRole +} + +func enforceAdminRoleDefaults(r *model.Role) error { + if r == nil || r.Name != "admin" { + return nil + } + if len(r.PermissionScopes) == 1 { + scopePath := utils.FixAndCleanPath(r.PermissionScopes[0].Path) + if scopePath == "/" && r.PermissionScopes[0].Permission == 0xFFFF { + r.PermissionScopes[0].Path = "/" + return nil + } + } + + r.PermissionScopes = []model.PermissionEntry{ + {Path: "/", Permission: 0xFFFF}, + } + return db.UpdateRole(r) +} + +func GetRole(id uint) (*model.Role, error) { + key := fmt.Sprint(id) + if r, ok := roleCache.Get(key); ok { + return r, nil + } + r, err, _ := roleG.Do(key, func() (*model.Role, error) { + _r, err := db.GetRole(id) + if err != nil { + return nil, err + } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } + roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(_r.Name, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + return r, err +} + +func GetRoleByName(name string) (*model.Role, error) { + if r, ok := roleCache.Get(name); ok { + return r, nil + } + r, err, _ := roleG.Do(name, func() (*model.Role, error) { + _r, err := db.GetRoleByName(name) + if err != nil { + return nil, err + } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } + roleCache.Set(name, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(fmt.Sprint(_r.ID), _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + return r, err +} + +func GetDefaultRoleID() int { + item, err := GetSettingItemByKey(conf.DefaultRole) + if err == nil && item != nil && item.Value != "" { + if id, err := strconv.Atoi(item.Value); err == nil && id != 0 { + return id + } + if r, err := db.GetRoleByName(item.Value); err == nil { + return int(r.ID) + } + } + var r model.Role + if err := db.GetDb().Where("`default` = ?", true).First(&r).Error; err == nil { + return int(r.ID) + } + return int(model.GUEST) +} + +func GetRolesByUserID(userID uint) ([]model.Role, error) { + user, err := GetUserById(userID) + if err != nil { + return nil, err + } + + var roles []model.Role + for _, roleID := range user.Role { + key := fmt.Sprint(roleID) + + if r, ok := roleCache.Get(key); ok { + roles = append(roles, *r) + continue + } + + r, err, _ := roleG.Do(key, func() (*model.Role, error) { + _r, err := db.GetRole(uint(roleID)) + if err != nil { + return nil, err + } + if err := enforceAdminRoleDefaults(_r); err != nil { + return nil, err + } + roleCache.Set(key, _r, cache.WithEx[*model.Role](time.Hour)) + roleCache.Set(_r.Name, _r, cache.WithEx[*model.Role](time.Hour)) + return _r, nil + }) + if err != nil { + return nil, err + } + roles = append(roles, *r) + } + + return roles, nil +} + +func GetRoles(pageIndex, pageSize int) ([]model.Role, int64, error) { + return db.GetRoles(pageIndex, pageSize) +} + +func CreateRole(r *model.Role) error { + for i := range r.PermissionScopes { + r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) + } + roleCache.Del(fmt.Sprint(r.ID)) + roleCache.Del(r.Name) + if err := db.CreateRole(r); err != nil { + return err + } + if r.Default { + roleCache.Clear() + item, err := GetSettingItemByKey(conf.DefaultRole) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + if err := SaveSettingItem(item); err != nil { + return err + } + } + return nil +} + +func UpdateRole(r *model.Role) error { + old, err := db.GetRole(r.ID) + if err != nil { + return err + } + switch old.Name { + case "admin": + return errs.ErrChangeDefaultRole + case "guest": + r.Name = "guest" + } + for i := range r.PermissionScopes { + r.PermissionScopes[i].Path = utils.FixAndCleanPath(r.PermissionScopes[i].Path) + } + //if len(old.PermissionScopes) > 0 && len(r.PermissionScopes) > 0 && + // old.PermissionScopes[0].Path != r.PermissionScopes[0].Path { + // + // oldPath := old.PermissionScopes[0].Path + // newPath := r.PermissionScopes[0].Path + // + // users, err := db.GetUsersByRole(int(r.ID)) + // if err != nil { + // return errors.WithMessage(err, "failed to get users by role") + // } + // + // modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldPath, newPath, users) + // if err != nil { + // return errors.WithMessage(err, "failed to update user base path when role updated") + // } + // + // for _, name := range modifiedUsernames { + // userCache.Del(name) + // } + //} + roleCache.Del(fmt.Sprint(r.ID)) + roleCache.Del(r.Name) + if err := db.UpdateRole(r); err != nil { + return err + } + if r.Default { + roleCache.Clear() + item, err := GetSettingItemByKey(conf.DefaultRole) + if err != nil { + return err + } + item.Value = strconv.Itoa(int(r.ID)) + if err := SaveSettingItem(item); err != nil { + return err + } + } + return nil +} + +func DeleteRole(id uint) error { + old, err := db.GetRole(id) + if err != nil { + return err + } + if old.Name == "admin" || old.Name == "guest" { + return errs.ErrChangeDefaultRole + } + roleCache.Del(fmt.Sprint(id)) + roleCache.Del(old.Name) + return db.DeleteRole(id) +} diff --git a/internal/op/setting.go b/internal/op/setting.go index 83d19c12fbe..36a792b0a6a 100644 --- a/internal/op/setting.go +++ b/internal/op/setting.go @@ -26,9 +26,18 @@ var settingGroupCacheF = func(key string, item []model.SettingItem) { settingGroupCache.Set(key, item, cache.WithEx[[]model.SettingItem](time.Hour)) } -func settingCacheUpdate() { +var settingChangingCallbacks = make([]func(), 0) + +func RegisterSettingChangingCallback(f func()) { + settingChangingCallbacks = append(settingChangingCallbacks, f) +} + +func SettingCacheUpdate() { settingCache.Clear() settingGroupCache.Clear() + for _, cb := range settingChangingCallbacks { + cb() + } } func GetPublicSettingsMap() map[string]string { @@ -167,7 +176,7 @@ func SaveSettingItems(items []model.SettingItem) error { } } if len(errs) < len(items)-len(noHookItems)+1 { - settingCacheUpdate() + SettingCacheUpdate() } return utils.MergeErrors(errs...) } @@ -181,7 +190,7 @@ func SaveSettingItem(item *model.SettingItem) (err error) { if err = db.SaveSettingItem(item); err != nil { return err } - settingCacheUpdate() + SettingCacheUpdate() return nil } @@ -193,6 +202,6 @@ func DeleteSettingItemByKey(key string) error { if !old.IsDeprecated() { return errors.Errorf("setting [%s] is not deprecated", key) } - settingCacheUpdate() + SettingCacheUpdate() return db.DeleteSettingItemByKey(key) } diff --git a/internal/op/sshkey.go b/internal/op/sshkey.go new file mode 100644 index 00000000000..139698e6ca5 --- /dev/null +++ b/internal/op/sshkey.go @@ -0,0 +1,47 @@ +package op + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/pkg/errors" + "golang.org/x/crypto/ssh" + "time" +) + +func CreateSSHPublicKey(k *model.SSHPublicKey) (error, bool) { + _, err := db.GetSSHPublicKeyByUserTitle(k.UserId, k.Title) + if err == nil { + return errors.New("key with the same title already exists"), true + } + pubKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(k.KeyStr)) + if err != nil { + return err, false + } + k.Fingerprint = ssh.FingerprintSHA256(pubKey) + k.AddedTime = time.Now() + k.LastUsedTime = k.AddedTime + return db.CreateSSHPublicKey(k), true +} + +func GetSSHPublicKeyByUserId(userId uint, pageIndex, pageSize int) (keys []model.SSHPublicKey, count int64, err error) { + return db.GetSSHPublicKeyByUserId(userId, pageIndex, pageSize) +} + +func GetSSHPublicKeyByIdAndUserId(id uint, userId uint) (*model.SSHPublicKey, error) { + key, err := db.GetSSHPublicKeyById(id) + if err != nil { + return nil, err + } + if key.UserId != userId { + return nil, errors.Wrapf(err, "failed get old key") + } + return key, nil +} + +func UpdateSSHPublicKey(k *model.SSHPublicKey) error { + return db.UpdateSSHPublicKey(k) +} + +func DeleteSSHPublicKeyById(keyId uint) error { + return db.DeleteSSHPublicKeyById(keyId) +} diff --git a/internal/op/storage.go b/internal/op/storage.go index 2f0831c452e..3961e32ed9a 100644 --- a/internal/op/storage.go +++ b/internal/op/storage.go @@ -2,12 +2,15 @@ package op import ( "context" + "fmt" + "runtime" "sort" "strings" "time" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/generic_sync" "github.com/alist-org/alist/v3/pkg/utils" @@ -38,11 +41,28 @@ func GetStorageByMountPath(mountPath string) (driver.Driver, error) { return storageDriver, nil } +func firstPathSegment(p string) string { + p = utils.FixAndCleanPath(p) + p = strings.TrimPrefix(p, "/") + if p == "" { + return "" + } + if i := strings.Index(p, "/"); i >= 0 { + return p[:i] + } + return p +} + // CreateStorage Save the storage to database so storage can get an id // then instantiate corresponding driver and save it in memory func CreateStorage(ctx context.Context, storage model.Storage) (uint, error) { storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + + //if storage.MountPath == "/" { + // return 0, errors.New("Mount path cannot be '/'") + //} + var err error // check driver first driverName := storage.Driver @@ -83,13 +103,50 @@ func LoadStorage(ctx context.Context, storage model.Storage) error { return err } +func getCurrentGoroutineStack() string { + buf := make([]byte, 1<<16) + n := runtime.Stack(buf, false) + return string(buf[:n]) +} + // initStorage initialize the driver and store to storagesMap func initStorage(ctx context.Context, storage model.Storage, storageDriver driver.Driver) (err error) { storageDriver.SetStorage(storage) driverStorage := storageDriver.GetStorage() - + defer func() { + if err := recover(); err != nil { + errInfo := fmt.Sprintf("[panic] err: %v\nstack: %s\n", err, getCurrentGoroutineStack()) + log.Errorf("panic init storage: %s", errInfo) + driverStorage.SetStatus(errInfo) + MustSaveDriverStorage(storageDriver) + storagesMap.Store(driverStorage.MountPath, storageDriver) + } + }() // Unmarshal Addition err = utils.Json.UnmarshalFromString(driverStorage.Addition, storageDriver.GetAddition()) + if err == nil { + if ref, ok := storageDriver.(driver.Reference); ok { + if strings.HasPrefix(driverStorage.Remark, "ref:/") { + refMountPath := driverStorage.Remark + i := strings.Index(refMountPath, "\n") + if i > 0 { + refMountPath = refMountPath[4:i] + } else { + refMountPath = refMountPath[4:] + } + var refStorage driver.Driver + refStorage, err = GetStorageByMountPath(refMountPath) + if err != nil { + err = fmt.Errorf("ref: %w", err) + } else { + err = ref.InitReference(refStorage) + if err != nil && errs.IsNotSupportError(err) { + err = fmt.Errorf("ref: storage is not %s", storageDriver.Config().Name) + } + } + } + } + } if err == nil { err = storageDriver.Init(ctx) } @@ -165,17 +222,46 @@ func UpdateStorage(ctx context.Context, storage model.Storage) error { } storage.Modified = time.Now() storage.MountPath = utils.FixAndCleanPath(storage.MountPath) + //if storage.MountPath == "/" { + // return errors.New("Mount path cannot be '/'") + //} err = db.UpdateStorage(&storage) if err != nil { return errors.WithMessage(err, "failed update storage in database") } + storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) + if err == nil { + ClearCache(storageDriver, "/") + } if storage.Disabled { return nil } - storageDriver, err := GetStorageByMountPath(oldStorage.MountPath) if oldStorage.MountPath != storage.MountPath { // mount path renamed, need to drop the storage storagesMap.Delete(oldStorage.MountPath) + modifiedRoleIDs, err := db.UpdateRolePermissionsPathPrefix(oldStorage.MountPath, storage.MountPath) + if err != nil { + return errors.WithMessage(err, "failed to update role permissions") + } + for _, id := range modifiedRoleIDs { + roleCache.Del(fmt.Sprint(id)) + } + + //modifiedUsernames, err := db.UpdateUserBasePathPrefix(oldStorage.MountPath, storage.MountPath) + //if err != nil { + // return errors.WithMessage(err, "failed to update user base path") + //} + for _, id := range modifiedRoleIDs { + roleCache.Del(fmt.Sprint(id)) + + users, err := db.GetUsersByRole(int(id)) + if err != nil { + return errors.WithMessage(err, "failed to get users by role") + } + for _, user := range users { + userCache.Del(user.Username) + } + } } if err != nil { return errors.WithMessage(err, "failed get storage driver") @@ -196,6 +282,34 @@ func DeleteStorageById(ctx context.Context, id uint) error { if err != nil { return errors.WithMessage(err, "failed get storage") } + firstMount := firstPathSegment(storage.MountPath) + if firstMount != "" { + roles, err := db.GetAllRoles() + if err != nil { + return errors.WithMessage(err, "failed to load roles") + } + users, err := db.GetAllUsers() + if err != nil { + return errors.WithMessage(err, "failed to load users") + } + var usedBy []string + for _, r := range roles { + for _, entry := range r.PermissionScopes { + if firstPathSegment(entry.Path) == firstMount { + usedBy = append(usedBy, "role:"+r.Name) + break + } + } + } + for _, u := range users { + if firstPathSegment(u.BasePath) == firstMount { + usedBy = append(usedBy, "user:"+u.Username) + } + } + if len(usedBy) > 0 { + return errors.Errorf("storage is used by %s, please cancel usage first", strings.Join(usedBy, ", ")) + } + } if !storage.Disabled { storageDriver, err := GetStorageByMountPath(storage.MountPath) if err != nil { diff --git a/internal/op/storage_test.go b/internal/op/storage_test.go index f3d0b4e64b7..fddf2bb11e3 100644 --- a/internal/op/storage_test.go +++ b/internal/op/storage_test.go @@ -60,7 +60,6 @@ func TestGetStorageVirtualFilesByPath(t *testing.T) { } func TestGetBalancedStorage(t *testing.T) { - setupStorages(t) set := mapset.NewSet[string]() for i := 0; i < 5; i++ { storage := op.GetBalancedStorage("/a/d/e1") diff --git a/internal/op/user.go b/internal/op/user.go index 79e73db86ce..b58a87ed338 100644 --- a/internal/op/user.go +++ b/internal/op/user.go @@ -16,20 +16,52 @@ var userG singleflight.Group[*model.User] var guestUser *model.User var adminUser *model.User +func enforceAdminUserDefaults(u *model.User) error { + if u == nil || !u.IsAdmin() { + return nil + } + changed := false + if utils.FixAndCleanPath(u.BasePath) != "/" { + u.BasePath = "/" + changed = true + } + if u.Permission != 0xFFFF { + u.Permission = 0xFFFF + changed = true + } + if !changed { + return nil + } + return db.UpdateUser(u) +} + func GetAdmin() (*model.User, error) { if adminUser == nil { - user, err := db.GetUserByRole(model.ADMIN) + role, err := GetRoleByName("admin") if err != nil { return nil, err } + user, err := db.GetUserByRole(int(role.ID)) + if err != nil { + return nil, err + } + if err := enforceAdminUserDefaults(user); err != nil { + return nil, err + } adminUser = user + } else if err := enforceAdminUserDefaults(adminUser); err != nil { + return nil, err } return adminUser, nil } func GetGuest() (*model.User, error) { if guestUser == nil { - user, err := db.GetUserByRole(model.GUEST) + role, err := GetRoleByName("guest") + if err != nil { + return nil, err + } + user, err := db.GetUserByRole(int(role.ID)) if err != nil { return nil, err } @@ -42,11 +74,18 @@ func GetUserByRole(role int) (*model.User, error) { return db.GetUserByRole(role) } +func GetUsersByRole(role int) ([]model.User, error) { + return db.GetUsersByRole(role) +} + func GetUserByName(username string) (*model.User, error) { if username == "" { return nil, errs.EmptyUsername } if user, ok := userCache.Get(username); ok { + if err := enforceAdminUserDefaults(user); err != nil { + return nil, err + } return user, nil } user, err, _ := userG.Do(username, func() (*model.User, error) { @@ -54,6 +93,9 @@ func GetUserByName(username string) (*model.User, error) { if err != nil { return nil, err } + if err := enforceAdminUserDefaults(_user); err != nil { + return nil, err + } userCache.Set(username, _user, cache.WithEx[*model.User](time.Hour)) return _user, nil }) @@ -70,7 +112,25 @@ func GetUsers(pageIndex, pageSize int) (users []model.User, count int64, err err func CreateUser(u *model.User) error { u.BasePath = utils.FixAndCleanPath(u.BasePath) - return db.CreateUser(u) + + err := db.CreateUser(u) + if err != nil { + return err + } + + roles, err := GetRolesByUserID(u.ID) + if err == nil { + for _, role := range roles { + if len(role.PermissionScopes) > 0 { + u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + break + } + } + _ = db.UpdateUser(u) + userCache.Del(u.Username) + } + + return nil } func DeleteUserById(id uint) error { @@ -98,6 +158,17 @@ func UpdateUser(u *model.User) error { } userCache.Del(old.Username) u.BasePath = utils.FixAndCleanPath(u.BasePath) + //if len(u.Role) > 0 { + // roles, err := GetRolesByUserID(u.ID) + // if err == nil { + // for _, role := range roles { + // if len(role.PermissionScopes) > 0 { + // u.BasePath = utils.FixAndCleanPath(role.PermissionScopes[0].Path) + // break + // } + // } + // } + //} return db.UpdateUser(u) } @@ -128,3 +199,11 @@ func DelUserCache(username string) error { userCache.Del(username) return nil } + +func CountEnabledAdminsExcluding(userID uint) (int64, error) { + adminRole, err := GetRoleByName("admin") + if err != nil { + return 0, err + } + return db.CountUsersByRoleAndEnabledExclude(adminRole.ID, userID) +} diff --git a/internal/qbittorrent/add.go b/internal/qbittorrent/add.go deleted file mode 100644 index f552a9ec227..00000000000 --- a/internal/qbittorrent/add.go +++ /dev/null @@ -1,60 +0,0 @@ -package qbittorrent - -import ( - "context" - "fmt" - "path/filepath" - - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/errs" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/google/uuid" - "github.com/pkg/errors" -) - -func AddURL(ctx context.Context, url string, dstDirPath string) error { - // check storage - storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - // check is it could upload - if storage.Config().NoUpload { - return errors.WithStack(errs.UploadNotSupported) - } - // check path is valid - obj, err := op.Get(ctx, storage, dstDirActualPath) - if err != nil { - if !errs.IsObjectNotFound(err) { - return errors.WithMessage(err, "failed get object") - } - } else { - if !obj.IsDir() { - // can't add to a file - return errors.WithStack(errs.NotFolder) - } - } - // call qbittorrent - id := uuid.NewString() - tempDir := filepath.Join(conf.Conf.TempDir, "qbittorrent", id) - err = qbclient.AddFromLink(url, tempDir, id) - if err != nil { - return errors.Wrapf(err, "failed to add url %s", url) - } - DownTaskManager.Submit(task.WithCancelCtx(&task.Task[string]{ - ID: id, - Name: fmt.Sprintf("download %s to [%s](%s)", url, storage.GetStorage().MountPath, dstDirActualPath), - Func: func(tsk *task.Task[string]) error { - m := &Monitor{ - tsk: tsk, - tempDir: tempDir, - dstDirPath: dstDirPath, - seedtime: setting.GetInt(conf.QbittorrentSeedtime, 0), - } - return m.Loop() - }, - })) - return nil -} diff --git a/internal/qbittorrent/client_test.go b/internal/qbittorrent/client_test.go deleted file mode 100644 index 21f1dc4104f..00000000000 --- a/internal/qbittorrent/client_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package qbittorrent - -import ( - "net/http" - "net/http/cookiejar" - "net/url" - "testing" -) - -func TestLogin(t *testing.T) { - // test logging in with wrong password - u, err := url.Parse("http://admin:admin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - jar, err := cookiejar.New(nil) - if err != nil { - t.Error(err) - } - var c = &client{ - url: u, - client: http.Client{Jar: jar}, - } - err = c.login() - if err == nil { - t.Error(err) - } - - // test logging in with correct password - u, err = url.Parse("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - c.url = u - err = c.login() - if err != nil { - t.Error(err) - } -} - -// in this test, the `Bypass authentication for clients on localhost` option in qBittorrent webui should be disabled -func TestAuthorized(t *testing.T) { - // init client - u, err := url.Parse("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - jar, err := cookiejar.New(nil) - if err != nil { - t.Error(err) - } - var c = &client{ - url: u, - client: http.Client{Jar: jar}, - } - - // test without logging in, which should be unauthorized - authorized := c.authorized() - if authorized { - t.Error("Should not be authorized") - } - - // test after logging in - err = c.login() - if err != nil { - t.Error(err) - } - authorized = c.authorized() - if !authorized { - t.Error("Should be authorized") - } -} - -func TestNew(t *testing.T) { - _, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - _, err = New("http://admin:wrong_password@127.0.0.1:8080/") - if err == nil { - t.Error("Should get an error") - } -} - -func TestAdd(t *testing.T) { - // init client - c, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - err = c.AddFromLink( - "https://releases.ubuntu.com/22.04/ubuntu-22.04.1-desktop-amd64.iso.torrent", - "D:\\qBittorrentDownload\\alist", - "uuid-1", - ) - if err != nil { - t.Error(err) - } - err = c.AddFromLink( - "magnet:?xt=urn:btih:375ae3280cd80a8e9d7212e11dfaf7c45069dd35&dn=archlinux-2023.02.01-x86_64.iso", - "D:\\qBittorrentDownload\\alist", - "uuid-2", - ) - if err != nil { - t.Error(err) - } -} - -func TestGetInfo(t *testing.T) { - // init client - c, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - _, err = c.GetInfo("uuid-1") - if err != nil { - t.Error(err) - } -} - -func TestGetFiles(t *testing.T) { - // init client - c, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - files, err := c.GetFiles("uuid-1") - if err != nil { - t.Error(err) - } - if len(files) != 1 { - t.Error("should have exactly one file") - } -} - -func TestDelete(t *testing.T) { - // init client - c, err := New("http://admin:adminadmin@127.0.0.1:8080/") - if err != nil { - t.Error(err) - } - err = c.AddFromLink( - "https://releases.ubuntu.com/22.04/ubuntu-22.04.1-desktop-amd64.iso.torrent", - "D:\\qBittorrentDownload\\alist", - "uuid-1", - ) - if err != nil { - t.Error(err) - } - err = c.Delete("uuid-1", true) - if err != nil { - t.Error(err) - } -} diff --git a/internal/qbittorrent/monitor.go b/internal/qbittorrent/monitor.go deleted file mode 100644 index 12bb4ad21c5..00000000000 --- a/internal/qbittorrent/monitor.go +++ /dev/null @@ -1,180 +0,0 @@ -package qbittorrent - -import ( - "fmt" - "github.com/alist-org/alist/v3/internal/stream" - "os" - "path/filepath" - "sync" - "sync/atomic" - "time" - - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/pkg/task" - "github.com/alist-org/alist/v3/pkg/utils" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" -) - -type Monitor struct { - tsk *task.Task[string] - tempDir string - dstDirPath string - seedtime int - finish chan struct{} -} - -func (m *Monitor) Loop() error { - var ( - err error - completed bool - ) - m.finish = make(chan struct{}) - - // wait for qbittorrent to parse torrent and create task - m.tsk.SetStatus("waiting for qbittorrent to parse torrent and create task") - waitCount := 0 - for { - _, err := qbclient.GetInfo(m.tsk.ID) - if err == nil { - break - } - switch err.(type) { - case InfoNotFoundError: - break - default: - return err - } - - waitCount += 1 - if waitCount >= 60 { - return errors.New("torrent parse timeout") - } - timer := time.NewTimer(time.Second) - <-timer.C - } - -outer: - for { - select { - case <-m.tsk.Ctx.Done(): - // delete qbittorrent task and downloaded files when the task exits with error - return qbclient.Delete(m.tsk.ID, true) - case <-time.After(time.Second * 2): - completed, err = m.update() - if completed { - break outer - } - } - } - if err != nil { - return err - } - m.tsk.SetStatus("qbittorrent download completed, transferring") - <-m.finish - m.tsk.SetStatus("completed") - return nil -} - -func (m *Monitor) update() (bool, error) { - info, err := qbclient.GetInfo(m.tsk.ID) - if err != nil { - m.tsk.SetStatus("qbittorrent " + string(info.State)) - return true, err - } - - progress := float64(info.Completed) / float64(info.Size) * 100 - m.tsk.SetProgress(int(progress)) - switch info.State { - case UPLOADING, PAUSEDUP, QUEUEDUP, STALLEDUP, FORCEDUP, CHECKINGUP: - err = m.complete() - return true, errors.WithMessage(err, "failed to transfer file") - case ALLOCATING, DOWNLOADING, METADL, PAUSEDDL, QUEUEDDL, STALLEDDL, CHECKINGDL, FORCEDDL, CHECKINGRESUMEDATA, MOVING: - m.tsk.SetStatus("qbittorrent downloading") - return false, nil - case ERROR, MISSINGFILES, UNKNOWN: - return true, errors.Errorf("failed to download %s, error: %s", m.tsk.ID, info.State) - } - return true, errors.New("unknown error occurred downloading qbittorrent") // should never happen -} - -var TransferTaskManager = task.NewTaskManager(3, func(k *uint64) { - atomic.AddUint64(k, 1) -}) - -func (m *Monitor) complete() error { - // check dstDir again - storage, dstBaseDir, err := op.GetStorageAndActualPath(m.dstDirPath) - if err != nil { - return errors.WithMessage(err, "failed get storage") - } - // get files - files, err := qbclient.GetFiles(m.tsk.ID) - if err != nil { - return errors.Wrapf(err, "failed to get files of %s", m.tsk.ID) - } - log.Debugf("files len: %d", len(files)) - // delete qbittorrent task but do not delete the files before transferring to avoid qbittorrent - // accessing downloaded files and throw `cannot access the file because it is being used by another process` error - // err = qbclient.Delete(m.tsk.ID, false) - // if err != nil { - // return err - // } - // upload files - var wg sync.WaitGroup - wg.Add(len(files)) - go func() { - wg.Wait() - m.finish <- struct{}{} - if m.seedtime < 0 { - log.Debugf("do not delete qb task %s", m.tsk.ID) - return - } - log.Debugf("delete qb task %s after %d minutes", m.tsk.ID, m.seedtime) - <-time.After(time.Duration(m.seedtime) * time.Minute) - err := qbclient.Delete(m.tsk.ID, true) - if err != nil { - log.Errorln(err.Error()) - } - err = os.RemoveAll(m.tempDir) - if err != nil { - log.Errorf("failed to remove qbittorrent temp dir: %+v", err.Error()) - } - }() - for _, file := range files { - tempPath := filepath.Join(m.tempDir, file.Name) - dstPath := filepath.Join(dstBaseDir, file.Name) - dstDir := filepath.Dir(dstPath) - fileName := filepath.Base(dstPath) - TransferTaskManager.Submit(task.WithCancelCtx(&task.Task[uint64]{ - Name: fmt.Sprintf("transfer %s to [%s](%s)", tempPath, storage.GetStorage().MountPath, dstPath), - Func: func(tsk *task.Task[uint64]) error { - defer wg.Done() - size := file.Size - mimetype := utils.GetMimeType(tempPath) - f, err := os.Open(tempPath) - if err != nil { - return errors.Wrapf(err, "failed to open file %s", tempPath) - } - s := stream.FileStream{ - Obj: &model.Object{ - Name: fileName, - Size: size, - Modified: time.Now(), - IsFolder: false, - }, - Reader: f, - Closers: utils.NewClosers(f), - Mimetype: mimetype, - } - ss, err := stream.NewSeekableStream(s, nil) - if err != nil { - return err - } - return op.Put(tsk.Ctx, storage, dstDir, ss, tsk.SetProgress) - }, - })) - } - return nil -} diff --git a/internal/qbittorrent/qbittorrent.go b/internal/qbittorrent/qbittorrent.go deleted file mode 100644 index d011717578e..00000000000 --- a/internal/qbittorrent/qbittorrent.go +++ /dev/null @@ -1,23 +0,0 @@ -package qbittorrent - -import ( - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/pkg/task" -) - -var DownTaskManager = task.NewTaskManager[string](3) -var qbclient Client - -func InitClient() error { - var err error - qbclient = nil - - url := setting.GetStr(conf.QbittorrentUrl) - qbclient, err = New(url) - return err -} - -func IsQbittorrentReady() bool { - return qbclient != nil -} diff --git a/internal/search/build.go b/internal/search/build.go index a806d08fadb..2888c1f45bb 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -5,10 +5,12 @@ import ( "path" "path/filepath" "strings" + "sync" "sync/atomic" "time" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" @@ -21,10 +23,13 @@ import ( ) var ( - Running = atomic.Bool{} - Quit chan struct{} + Quit = atomic.Pointer[chan struct{}]{} ) +func Running() bool { + return Quit.Load() != nil +} + func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth int, count bool) error { var ( err error @@ -33,11 +38,27 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth ) log.Infof("build index for: %+v", indexPaths) log.Infof("ignore paths: %+v", ignorePaths) - Running.Store(true) - Quit = make(chan struct{}, 1) - indexMQ := mq.NewInMemoryMQ[ObjWithParent]() + quit := make(chan struct{}, 1) + if !Quit.CompareAndSwap(nil, &quit) { + // other goroutine is running + return errs.BuildIndexIsRunning + } + var ( + indexMQ = mq.NewInMemoryMQ[ObjWithParent]() + running = atomic.Bool{} // current goroutine running + wg = &sync.WaitGroup{} + ) + running.Store(true) + wg.Add(1) go func() { ticker := time.NewTicker(time.Second) + defer func() { + Quit.Store(nil) + wg.Done() + // notify walk to exit when StopIndex api called + running.Store(false) + ticker.Stop() + }() tickCount := 0 for { select { @@ -70,9 +91,8 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth } }) - case <-Quit: - Running.Store(false) - ticker.Stop() + case <-quit: + log.Debugf("build index for %+v received quit", indexPaths) eMsg := "" now := time.Now() originErr := err @@ -100,14 +120,22 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth }) } }) + log.Debugf("build index for %+v quit success", indexPaths) return } } }() defer func() { - if Running.Load() { - Quit <- struct{}{} + if !running.Load() || Quit.Load() != &quit { + log.Debugf("build index for %+v stopped by StopIndex", indexPaths) + return + } + select { + // avoid goroutine leak + case quit <- struct{}{}: + default: } + wg.Wait() }() admin, err := op.GetAdmin() if err != nil { @@ -121,7 +149,7 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth } for _, indexPath := range indexPaths { walkFn := func(indexPath string, info model.Obj) error { - if !Running.Load() { + if !running.Load() { return filepath.SkipDir } for _, avoidPath := range ignorePaths { @@ -129,6 +157,11 @@ func BuildIndex(ctx context.Context, indexPaths, ignorePaths []string, maxDepth return filepath.SkipDir } } + if storage, _, err := op.GetStorageAndActualPath(indexPath); err == nil { + if storage.GetStorage().DisableIndex { + return filepath.SkipDir + } + } // ignore root if indexPath == "/" { return nil @@ -167,7 +200,7 @@ func Config(ctx context.Context) searcher.Config { } func Update(parent string, objs []model.Obj) { - if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running.Load() { + if instance == nil || !instance.Config().AutoUpdate || !setting.GetBool(conf.AutoUpdateIndex) || Running() { return } if isIgnorePath(parent) { @@ -211,14 +244,15 @@ func Update(parent string, objs []model.Obj) { } for i := range objs { if toAdd.Contains(objs[i].GetName()) { - log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) - err = Index(ctx, parent, objs[i]) - if err != nil { - log.Errorf("update search index error while index new node: %+v", err) - return - } - // build index if it's a folder - if objs[i].IsDir() { + if !objs[i].IsDir() { + log.Debugf("add index: %s", path.Join(parent, objs[i].GetName())) + err = Index(ctx, parent, objs[i]) + if err != nil { + log.Errorf("update search index error while index new node: %+v", err) + return + } + } else { + // build index if it's a folder dir := path.Join(parent, objs[i].GetName()) err = BuildIndex(ctx, []string{dir}, diff --git a/internal/search/import.go b/internal/search/import.go index 822ae3f793f..a34c36f9a34 100644 --- a/internal/search/import.go +++ b/internal/search/import.go @@ -4,4 +4,5 @@ import ( _ "github.com/alist-org/alist/v3/internal/search/bleve" _ "github.com/alist-org/alist/v3/internal/search/db" _ "github.com/alist-org/alist/v3/internal/search/db_non_full_text" + _ "github.com/alist-org/alist/v3/internal/search/meilisearch" ) diff --git a/internal/search/meilisearch/init.go b/internal/search/meilisearch/init.go new file mode 100644 index 00000000000..8f5f24733ee --- /dev/null +++ b/internal/search/meilisearch/init.go @@ -0,0 +1,89 @@ +package meilisearch + +import ( + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/meilisearch/meilisearch-go" +) + +var config = searcher.Config{ + Name: "meilisearch", + AutoUpdate: true, +} + +func init() { + searcher.RegisterSearcher(config, func() (searcher.Searcher, error) { + m := Meilisearch{ + Client: meilisearch.NewClient(meilisearch.ClientConfig{ + Host: conf.Conf.Meilisearch.Host, + APIKey: conf.Conf.Meilisearch.APIKey, + }), + IndexUid: conf.Conf.Meilisearch.IndexPrefix + "alist", + FilterableAttributes: []string{"parent", "is_dir", "name"}, + SearchableAttributes: []string{"name"}, + } + + _, err := m.Client.GetIndex(m.IndexUid) + if err != nil { + var mErr *meilisearch.Error + ok := errors.As(err, &mErr) + if ok && mErr.MeilisearchApiError.Code == "index_not_found" { + task, err := m.Client.CreateIndex(&meilisearch.IndexConfig{ + Uid: m.IndexUid, + PrimaryKey: "id", + }) + if err != nil { + return nil, err + } + forTask, err := m.Client.WaitForTask(task.TaskUID) + if err != nil { + return nil, err + } + if forTask.Status != meilisearch.TaskStatusSucceeded { + return nil, fmt.Errorf("index creation failed, task status is %s", forTask.Status) + } + } else { + return nil, err + } + } + attributes, err := m.Client.Index(m.IndexUid).GetFilterableAttributes() + if err != nil { + return nil, err + } + if attributes == nil || !utils.SliceAllContains(*attributes, m.FilterableAttributes...) { + _, err = m.Client.Index(m.IndexUid).UpdateFilterableAttributes(&m.FilterableAttributes) + if err != nil { + return nil, err + } + } + + attributes, err = m.Client.Index(m.IndexUid).GetSearchableAttributes() + if err != nil { + return nil, err + } + if attributes == nil || !utils.SliceAllContains(*attributes, m.SearchableAttributes...) { + _, err = m.Client.Index(m.IndexUid).UpdateSearchableAttributes(&m.SearchableAttributes) + if err != nil { + return nil, err + } + } + + pagination, err := m.Client.Index(m.IndexUid).GetPagination() + if err != nil { + return nil, err + } + if pagination.MaxTotalHits != int64(model.MaxInt) { + _, err := m.Client.Index(m.IndexUid).UpdatePagination(&meilisearch.Pagination{ + MaxTotalHits: int64(model.MaxInt), + }) + if err != nil { + return nil, err + } + } + return &m, nil + }) +} diff --git a/internal/search/meilisearch/search.go b/internal/search/meilisearch/search.go new file mode 100644 index 00000000000..1516306b75f --- /dev/null +++ b/internal/search/meilisearch/search.go @@ -0,0 +1,227 @@ +package meilisearch + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/search/searcher" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/google/uuid" + "github.com/meilisearch/meilisearch-go" + "path" + "strings" + "time" +) + +type searchDocument struct { + ID string `json:"id"` + model.SearchNode +} + +type Meilisearch struct { + Client *meilisearch.Client + IndexUid string + FilterableAttributes []string + SearchableAttributes []string +} + +func (m *Meilisearch) Config() searcher.Config { + return config +} + +func (m *Meilisearch) Search(ctx context.Context, req model.SearchReq) ([]model.SearchNode, int64, error) { + mReq := &meilisearch.SearchRequest{ + AttributesToSearchOn: m.SearchableAttributes, + Page: int64(req.Page), + HitsPerPage: int64(req.PerPage), + } + if req.Scope != 0 { + mReq.Filter = fmt.Sprintf("is_dir = %v", req.Scope == 1) + } + search, err := m.Client.Index(m.IndexUid).Search(req.Keywords, mReq) + if err != nil { + return nil, 0, err + } + nodes, err := utils.SliceConvert(search.Hits, func(src any) (model.SearchNode, error) { + srcMap := src.(map[string]any) + return model.SearchNode{ + Parent: srcMap["parent"].(string), + Name: srcMap["name"].(string), + IsDir: srcMap["is_dir"].(bool), + Size: int64(srcMap["size"].(float64)), + }, nil + }) + if err != nil { + return nil, 0, err + } + return nodes, search.TotalHits, nil +} + +func (m *Meilisearch) Index(ctx context.Context, node model.SearchNode) error { + return m.BatchIndex(ctx, []model.SearchNode{node}) +} + +func (m *Meilisearch) BatchIndex(ctx context.Context, nodes []model.SearchNode) error { + documents, _ := utils.SliceConvert(nodes, func(src model.SearchNode) (*searchDocument, error) { + + return &searchDocument{ + ID: uuid.NewString(), + SearchNode: src, + }, nil + }) + + _, err := m.Client.Index(m.IndexUid).AddDocuments(documents) + if err != nil { + return err + } + + //// Wait for the task to complete and check + //forTask, err := m.Client.WaitForTask(task.TaskUID, meilisearch.WaitParams{ + // Context: ctx, + // Interval: time.Millisecond * 50, + //}) + //if err != nil { + // return err + //} + //if forTask.Status != meilisearch.TaskStatusSucceeded { + // return fmt.Errorf("BatchIndex failed, task status is %s", forTask.Status) + //} + return nil +} + +func (m *Meilisearch) getDocumentsByParent(ctx context.Context, parent string) ([]*searchDocument, error) { + var result meilisearch.DocumentsResult + err := m.Client.Index(m.IndexUid).GetDocuments(&meilisearch.DocumentsQuery{ + Filter: fmt.Sprintf("parent = '%s'", strings.ReplaceAll(parent, "'", "\\'")), + Limit: int64(model.MaxInt), + }, &result) + if err != nil { + return nil, err + } + return utils.SliceConvert(result.Results, func(src map[string]any) (*searchDocument, error) { + return &searchDocument{ + ID: src["id"].(string), + SearchNode: model.SearchNode{ + Parent: src["parent"].(string), + Name: src["name"].(string), + IsDir: src["is_dir"].(bool), + Size: int64(src["size"].(float64)), + }, + }, nil + }) +} + +func (m *Meilisearch) Get(ctx context.Context, parent string) ([]model.SearchNode, error) { + result, err := m.getDocumentsByParent(ctx, parent) + if err != nil { + return nil, err + } + return utils.SliceConvert(result, func(src *searchDocument) (model.SearchNode, error) { + return src.SearchNode, nil + }) + +} + +func (m *Meilisearch) getParentsByPrefix(ctx context.Context, parent string) ([]string, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + parents := []string{parent} + get, err := m.getDocumentsByParent(ctx, parent) + if err != nil { + return nil, err + } + for _, node := range get { + if node.IsDir { + arr, err := m.getParentsByPrefix(ctx, path.Join(node.Parent, node.Name)) + if err != nil { + return nil, err + } + parents = append(parents, arr...) + } + } + return parents, nil + } +} + +func (m *Meilisearch) DelDirChild(ctx context.Context, prefix string) error { + dfs, err := m.getParentsByPrefix(ctx, utils.FixAndCleanPath(prefix)) + if err != nil { + return err + } + utils.SliceReplace(dfs, func(src string) string { + return "'" + strings.ReplaceAll(src, "'", "\\'") + "'" + }) + s := fmt.Sprintf("parent IN [%s]", strings.Join(dfs, ",")) + task, err := m.Client.Index(m.IndexUid).DeleteDocumentsByFilter(s) + if err != nil { + return err + } + taskStatus, err := m.getTaskStatus(ctx, task.TaskUID) + if err != nil { + return err + } + if taskStatus != meilisearch.TaskStatusSucceeded { + return fmt.Errorf("DelDir failed, task status is %s", taskStatus) + } + return nil +} + +func (m *Meilisearch) Del(ctx context.Context, prefix string) error { + prefix = utils.FixAndCleanPath(prefix) + dir, name := path.Split(prefix) + get, err := m.getDocumentsByParent(ctx, dir[:len(dir)-1]) + if err != nil { + return err + } + var document *searchDocument + for _, v := range get { + if v.Name == name { + document = v + break + } + } + if document == nil { + // Defensive programming. Document may be the folder, try deleting Child + return m.DelDirChild(ctx, prefix) + } + if document.IsDir { + err = m.DelDirChild(ctx, prefix) + if err != nil { + return err + } + } + task, err := m.Client.Index(m.IndexUid).DeleteDocument(document.ID) + if err != nil { + return err + } + taskStatus, err := m.getTaskStatus(ctx, task.TaskUID) + if err != nil { + return err + } + if taskStatus != meilisearch.TaskStatusSucceeded { + return fmt.Errorf("DelDir failed, task status is %s", taskStatus) + } + return nil +} + +func (m *Meilisearch) Release(ctx context.Context) error { + return nil +} + +func (m *Meilisearch) Clear(ctx context.Context) error { + _, err := m.Client.Index(m.IndexUid).DeleteAllDocuments() + return err +} + +func (m *Meilisearch) getTaskStatus(ctx context.Context, taskUID int64) (meilisearch.TaskStatus, error) { + forTask, err := m.Client.WaitForTask(taskUID, meilisearch.WaitParams{ + Context: ctx, + Interval: time.Second, + }) + if err != nil { + return meilisearch.TaskStatusUnknown, err + } + return forTask.Status, nil +} diff --git a/internal/search/search.go b/internal/search/search.go index c1a23b85ca8..d420eb16dd3 100644 --- a/internal/search/search.go +++ b/internal/search/search.go @@ -27,7 +27,7 @@ func Init(mode string) error { } instance = nil } - if Running.Load() { + if Running() { return fmt.Errorf("index is running") } if mode == "none" { diff --git a/internal/search/util.go b/internal/search/util.go index 8d03b740c33..2e6ac8da1cd 100644 --- a/internal/search/util.go +++ b/internal/search/util.go @@ -38,7 +38,7 @@ func WriteProgress(progress *model.IndexProgress) { } } -func updateIgnorePaths() { +func updateIgnorePaths(customIgnorePaths string) { storages := op.GetAllStorages() ignorePaths := make([]string, 0) var skipDrivers = []string{"AList V2", "AList V3", "Virtual"} @@ -66,7 +66,6 @@ func updateIgnorePaths() { } } } - customIgnorePaths := setting.GetStr(conf.IgnorePaths) if customIgnorePaths != "" { ignorePaths = append(ignorePaths, strings.Split(customIgnorePaths, "\n")...) } @@ -84,13 +83,13 @@ func isIgnorePath(path string) bool { func init() { op.RegisterSettingItemHook(conf.IgnorePaths, func(item *model.SettingItem) error { - updateIgnorePaths() + updateIgnorePaths(item.Value) return nil }) op.RegisterStorageHook(func(typ string, storage driver.Driver) { var skipDrivers = []string{"AList V2", "AList V3", "Virtual"} if utils.SliceContains(skipDrivers, storage.Config().Name) { - updateIgnorePaths() + updateIgnorePaths(setting.GetStr(conf.IgnorePaths)) } }) } diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 00000000000..47d1b70125c --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,8 @@ +package session + +import "github.com/alist-org/alist/v3/internal/db" + +// MarkInactive marks the session with the given ID as inactive. +func MarkInactive(sessionID string) error { + return db.MarkInactive(sessionID) +} diff --git a/internal/share/access.go b/internal/share/access.go new file mode 100644 index 00000000000..7baa34e3ef9 --- /dev/null +++ b/internal/share/access.go @@ -0,0 +1,31 @@ +package share + +import ( + "fmt" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + signPkg "github.com/alist-org/alist/v3/pkg/sign" +) + +func tokenPayload(share *model.Share) string { + updatedAt := int64(0) + if !share.UpdatedAt.IsZero() { + updatedAt = share.UpdatedAt.Unix() + } + return fmt.Sprintf("%s:%s:%d", share.ShareID, share.PasswordHash, updatedAt) +} + +func signer() signPkg.Sign { + return signPkg.NewHMACSign([]byte(setting.GetStr(conf.Token) + "-share-access")) +} + +func SignAccess(share *model.Share, d time.Duration) string { + return signer().Sign(tokenPayload(share), time.Now().Add(d).Unix()) +} + +func VerifyAccess(share *model.Share, token string) error { + return signer().Verify(tokenPayload(share), token) +} diff --git a/internal/sign/archive.go b/internal/sign/archive.go new file mode 100644 index 00000000000..26a2c208d56 --- /dev/null +++ b/internal/sign/archive.go @@ -0,0 +1,41 @@ +package sign + +import ( + "sync" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/sign" +) + +var onceArchive sync.Once +var instanceArchive sign.Sign + +func SignArchive(data string) string { + expire := setting.GetInt(conf.LinkExpiration, 0) + if expire == 0 { + return NotExpiredArchive(data) + } else { + return WithDurationArchive(data, time.Duration(expire)*time.Hour) + } +} + +func WithDurationArchive(data string, d time.Duration) string { + onceArchive.Do(InstanceArchive) + return instanceArchive.Sign(data, time.Now().Add(d).Unix()) +} + +func NotExpiredArchive(data string) string { + onceArchive.Do(InstanceArchive) + return instanceArchive.Sign(data, 0) +} + +func VerifyArchive(data string, sign string) error { + onceArchive.Do(InstanceArchive) + return instanceArchive.Verify(data, sign) +} + +func InstanceArchive() { + instanceArchive = sign.NewHMACSign([]byte(setting.GetStr(conf.Token) + "-archive")) +} diff --git a/internal/stream/limit.go b/internal/stream/limit.go new file mode 100644 index 00000000000..14d0efd0f3e --- /dev/null +++ b/internal/stream/limit.go @@ -0,0 +1,152 @@ +package stream + +import ( + "context" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "golang.org/x/time/rate" + "io" + "time" +) + +type Limiter interface { + Limit() rate.Limit + Burst() int + TokensAt(time.Time) float64 + Tokens() float64 + Allow() bool + AllowN(time.Time, int) bool + Reserve() *rate.Reservation + ReserveN(time.Time, int) *rate.Reservation + Wait(context.Context) error + WaitN(context.Context, int) error + SetLimit(rate.Limit) + SetLimitAt(time.Time, rate.Limit) + SetBurst(int) + SetBurstAt(time.Time, int) +} + +var ( + ClientDownloadLimit Limiter + ClientUploadLimit Limiter + ServerDownloadLimit Limiter + ServerUploadLimit Limiter +) + +type RateLimitReader struct { + io.Reader + Limiter Limiter + Ctx context.Context +} + +func (r *RateLimitReader) Read(p []byte) (n int, err error) { + if r.Ctx != nil && utils.IsCanceled(r.Ctx) { + return 0, r.Ctx.Err() + } + n, err = r.Reader.Read(p) + if err != nil { + return + } + if r.Limiter != nil { + if r.Ctx == nil { + r.Ctx = context.Background() + } + err = r.Limiter.WaitN(r.Ctx, n) + } + return +} + +func (r *RateLimitReader) Close() error { + if c, ok := r.Reader.(io.Closer); ok { + return c.Close() + } + return nil +} + +type RateLimitWriter struct { + io.Writer + Limiter Limiter + Ctx context.Context +} + +func (w *RateLimitWriter) Write(p []byte) (n int, err error) { + if w.Ctx != nil && utils.IsCanceled(w.Ctx) { + return 0, w.Ctx.Err() + } + n, err = w.Writer.Write(p) + if err != nil { + return + } + if w.Limiter != nil { + if w.Ctx == nil { + w.Ctx = context.Background() + } + err = w.Limiter.WaitN(w.Ctx, n) + } + return +} + +func (w *RateLimitWriter) Close() error { + if c, ok := w.Writer.(io.Closer); ok { + return c.Close() + } + return nil +} + +type RateLimitFile struct { + model.File + Limiter Limiter + Ctx context.Context +} + +func (r *RateLimitFile) Read(p []byte) (n int, err error) { + if r.Ctx != nil && utils.IsCanceled(r.Ctx) { + return 0, r.Ctx.Err() + } + n, err = r.File.Read(p) + if err != nil { + return + } + if r.Limiter != nil { + if r.Ctx == nil { + r.Ctx = context.Background() + } + err = r.Limiter.WaitN(r.Ctx, n) + } + return +} + +func (r *RateLimitFile) ReadAt(p []byte, off int64) (n int, err error) { + if r.Ctx != nil && utils.IsCanceled(r.Ctx) { + return 0, r.Ctx.Err() + } + n, err = r.File.ReadAt(p, off) + if err != nil { + return + } + if r.Limiter != nil { + if r.Ctx == nil { + r.Ctx = context.Background() + } + err = r.Limiter.WaitN(r.Ctx, n) + } + return +} + +type RateLimitRangeReadCloser struct { + model.RangeReadCloserIF + Limiter Limiter +} + +func (rrc *RateLimitRangeReadCloser) RangeRead(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + rc, err := rrc.RangeReadCloserIF.RangeRead(ctx, httpRange) + if err != nil { + return nil, err + } + return &RateLimitReader{ + Reader: rc, + Limiter: rrc.Limiter, + Ctx: ctx, + }, nil +} diff --git a/internal/stream/stream.go b/internal/stream/stream.go index a45735d035a..64160915792 100644 --- a/internal/stream/stream.go +++ b/internal/stream/stream.go @@ -6,21 +6,25 @@ import ( "errors" "fmt" "io" + "math" "os" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/sirupsen/logrus" + "go4.org/readerutil" ) type FileStream struct { Ctx context.Context model.Obj io.Reader - Mimetype string - WebPutAsTask bool - Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it + Mimetype string + WebPutAsTask bool + ForceStreamUpload bool + Exist model.Obj //the file existed in the destination, we can reuse some info since we wil overwrite it utils.Closers tmpFile *os.File //if present, tmpFile has full content, it will be deleted at last peekBuff *bytes.Reader @@ -43,13 +47,24 @@ func (f *FileStream) GetMimetype() string { func (f *FileStream) NeedStore() bool { return f.WebPutAsTask } + +func (f *FileStream) IsForceStreamUpload() bool { + return f.ForceStreamUpload +} + func (f *FileStream) Close() error { var err1, err2 error + err1 = f.Closers.Close() + if errors.Is(err1, os.ErrClosed) { + err1 = nil + } if f.tmpFile != nil { err2 = os.RemoveAll(f.tmpFile.Name()) if err2 != nil { err2 = errs.NewErr(err2, "failed to remove tmpFile [%s]", f.tmpFile.Name()) + } else { + f.tmpFile = nil } } @@ -79,7 +94,17 @@ func (f *FileStream) CacheFullInTempFile() (model.File, error) { f.Add(tmpF) f.tmpFile = tmpF f.Reader = tmpF - return f.tmpFile, nil + return tmpF, nil +} + +func (f *FileStream) GetFile() model.File { + if f.tmpFile != nil { + return f.tmpFile + } + if file, ok := f.Reader.(model.File); ok { + return file + } + return nil } const InMemoryBufMaxSize = 10 // Megabytes @@ -89,33 +114,39 @@ const InMemoryBufMaxSizeBytes = InMemoryBufMaxSize * 1024 * 1024 // also support a peeking RangeRead at very start, but won't buffer more than 10MB data in memory func (f *FileStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if httpRange.Length == -1 { - httpRange.Length = f.GetSize() + // 参考 internal/net/request.go + httpRange.Length = f.GetSize() - httpRange.Start } - if f.peekBuff != nil && httpRange.Start < int64(f.peekBuff.Len()) && httpRange.Start+httpRange.Length-1 < int64(f.peekBuff.Len()) { + size := httpRange.Start + httpRange.Length + if f.peekBuff != nil && size <= int64(f.peekBuff.Len()) { return io.NewSectionReader(f.peekBuff, httpRange.Start, httpRange.Length), nil } - if f.tmpFile == nil { - if httpRange.Start == 0 && httpRange.Length <= InMemoryBufMaxSizeBytes && f.peekBuff == nil { - bufSize := utils.Min(httpRange.Length, f.GetSize()) - newBuf := bytes.NewBuffer(make([]byte, 0, bufSize)) - n, err := io.CopyN(newBuf, f.Reader, bufSize) + var cache io.ReaderAt = f.GetFile() + if cache == nil { + if size <= InMemoryBufMaxSizeBytes { + bufSize := min(size, f.GetSize()) + // 使用bytes.Buffer作为io.CopyBuffer的写入对象,CopyBuffer会调用Buffer.ReadFrom + // 即使被写入的数据量与Buffer.Cap一致,Buffer也会扩大 + buf := make([]byte, bufSize) + n, err := io.ReadFull(f.Reader, buf) if err != nil { return nil, err } - if n != bufSize { + if n != int(bufSize) { return nil, fmt.Errorf("stream RangeRead did not get all data in peek, expect =%d ,actual =%d", bufSize, n) } - f.peekBuff = bytes.NewReader(newBuf.Bytes()) + f.peekBuff = bytes.NewReader(buf) f.Reader = io.MultiReader(f.peekBuff, f.Reader) - return io.NewSectionReader(f.peekBuff, httpRange.Start, httpRange.Length), nil + cache = f.peekBuff } else { - _, err := f.CacheFullInTempFile() + var err error + cache, err = f.CacheFullInTempFile() if err != nil { return nil, err } } } - return io.NewSectionReader(f.tmpFile, httpRange.Start, httpRange.Length), nil + return io.NewSectionReader(cache, httpRange.Start, httpRange.Length), nil } var _ model.FileStreamer = (*SeekableStream)(nil) @@ -124,6 +155,10 @@ var _ model.FileStreamer = (*FileStream)(nil) //var _ seekableStream = (*FileStream)(nil) // for most internal stream, which is either RangeReadCloser or MFile +// Any functionality implemented based on SeekableStream should implement a Close method, +// whose only purpose is to close the SeekableStream object. If such functionality has +// additional resources that need to be closed, they should be added to the Closer property of +// the SeekableStream object and be closed together when the SeekableStream object is closed. type SeekableStream struct { FileStream Link *model.Link @@ -136,37 +171,55 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) if len(fs.Mimetype) == 0 { fs.Mimetype = utils.GetMimeType(fs.Obj.GetName()) } - ss := SeekableStream{FileStream: fs, Link: link} + ss := &SeekableStream{FileStream: fs, Link: link} if ss.Reader != nil { result, ok := ss.Reader.(model.File) if ok { ss.mFile = result ss.Closers.Add(result) - return &ss, nil + return ss, nil } } if ss.Link != nil { if ss.Link.MFile != nil { - ss.mFile = ss.Link.MFile - ss.Reader = ss.Link.MFile - ss.Closers.Add(ss.Link.MFile) - return &ss, nil + mFile := ss.Link.MFile + if _, ok := mFile.(*os.File); !ok { + mFile = &RateLimitFile{ + File: mFile, + Limiter: ServerDownloadLimit, + Ctx: fs.Ctx, + } + } + ss.mFile = mFile + ss.Reader = mFile + ss.Closers.Add(mFile) + return ss, nil } - if ss.Link.RangeReadCloser != nil { - ss.rangeReadCloser = ss.Link.RangeReadCloser - return &ss, nil + ss.rangeReadCloser = &RateLimitRangeReadCloser{ + RangeReadCloserIF: ss.Link.RangeReadCloser, + Limiter: ServerDownloadLimit, + } + ss.Add(ss.rangeReadCloser) + return ss, nil } if len(ss.Link.URL) > 0 { rrc, err := GetRangeReadCloserFromLink(ss.GetSize(), link) if err != nil { return nil, err } + rrc = &RateLimitRangeReadCloser{ + RangeReadCloserIF: rrc, + Limiter: ServerDownloadLimit, + } ss.rangeReadCloser = rrc - return &ss, nil + ss.Add(rrc) + return ss, nil } } - + if fs.Reader != nil { + return ss, nil + } return nil, fmt.Errorf("illegal seekableStream") } @@ -177,7 +230,7 @@ func NewSeekableStream(fs FileStream, link *model.Link) (*SeekableStream, error) // RangeRead is not thread-safe, pls use it in single thread only. func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, error) { if httpRange.Length == -1 { - httpRange.Length = ss.GetSize() + httpRange.Length = ss.GetSize() - httpRange.Start } if ss.mFile != nil { return io.NewSectionReader(ss.mFile, httpRange.Start, httpRange.Length), nil @@ -192,7 +245,7 @@ func (ss *SeekableStream) RangeRead(httpRange http_range.Range) (io.Reader, erro } return rc, nil } - return nil, fmt.Errorf("can't find mFile or rangeReadCloser") + return ss.FileStream.RangeRead(httpRange) } //func (f *FileStream) GetReader() io.Reader { @@ -214,8 +267,6 @@ func (ss *SeekableStream) Read(p []byte) (n int, err error) { return 0, nil } ss.Reader = io.NopCloser(rc) - ss.Closers.Add(rc) - } return ss.Reader.Read(p) } @@ -234,10 +285,308 @@ func (ss *SeekableStream) CacheFullInTempFile() (model.File, error) { ss.Add(tmpF) ss.tmpFile = tmpF ss.Reader = tmpF - return ss.tmpFile, nil + return tmpF, nil +} + +func (ss *SeekableStream) GetFile() model.File { + if ss.tmpFile != nil { + return ss.tmpFile + } + if ss.mFile != nil { + return ss.mFile + } + return nil } func (f *FileStream) SetTmpFile(r *os.File) { - f.Reader = r + f.Add(r) f.tmpFile = r + f.Reader = r +} + +type ReaderWithSize interface { + io.ReadCloser + GetSize() int64 +} + +type SimpleReaderWithSize struct { + io.Reader + Size int64 +} + +func (r *SimpleReaderWithSize) GetSize() int64 { + return r.Size +} + +func (r *SimpleReaderWithSize) Close() error { + if c, ok := r.Reader.(io.Closer); ok { + return c.Close() + } + return nil +} + +type ReaderUpdatingProgress struct { + Reader ReaderWithSize + model.UpdateProgress + offset int +} + +func (r *ReaderUpdatingProgress) Read(p []byte) (n int, err error) { + n, err = r.Reader.Read(p) + r.offset += n + r.UpdateProgress(math.Min(100.0, float64(r.offset)/float64(r.Reader.GetSize())*100.0)) + return n, err +} + +func (r *ReaderUpdatingProgress) Close() error { + return r.Reader.Close() +} + +type SStreamReadAtSeeker interface { + model.File + GetRawStream() *SeekableStream +} + +type readerCur struct { + reader io.Reader + cur int64 +} + +type RangeReadReadAtSeeker struct { + ss *SeekableStream + masterOff int64 + readers []*readerCur + headCache *headCache +} + +type headCache struct { + *readerCur + bufs [][]byte +} + +func (c *headCache) read(p []byte) (n int, err error) { + pL := len(p) + logrus.Debugf("headCache read_%d", pL) + if c.cur < int64(pL) { + bufL := int64(pL) - c.cur + buf := make([]byte, bufL) + lr := io.LimitReader(c.reader, bufL) + off := 0 + for c.cur < int64(pL) { + n, err = lr.Read(buf[off:]) + off += n + c.cur += int64(n) + if err == io.EOF && off == int(bufL) { + err = nil + } + if err != nil { + break + } + } + c.bufs = append(c.bufs, buf) + } + n = 0 + if c.cur >= int64(pL) { + for i := 0; n < pL; i++ { + buf := c.bufs[i] + r := len(buf) + if n+r > pL { + r = pL - n + } + n += copy(p[n:], buf[:r]) + } + } + return +} +func (r *headCache) Close() error { + for i := range r.bufs { + r.bufs[i] = nil + } + r.bufs = nil + return nil +} + +func (r *RangeReadReadAtSeeker) InitHeadCache() { + if r.ss.Link.MFile == nil && r.masterOff == 0 { + reader := r.readers[0] + r.readers = r.readers[1:] + r.headCache = &headCache{readerCur: reader} + r.ss.Closers.Add(r.headCache) + } +} + +func NewReadAtSeeker(ss *SeekableStream, offset int64, forceRange ...bool) (SStreamReadAtSeeker, error) { + if ss.mFile != nil { + _, err := ss.mFile.Seek(offset, io.SeekStart) + if err != nil { + return nil, err + } + return &FileReadAtSeeker{ss: ss}, nil + } + r := &RangeReadReadAtSeeker{ + ss: ss, + masterOff: offset, + } + if offset != 0 || utils.IsBool(forceRange...) { + if offset < 0 || offset > ss.GetSize() { + return nil, errors.New("offset out of range") + } + _, err := r.getReaderAtOffset(offset) + if err != nil { + return nil, err + } + } else { + rc := &readerCur{reader: ss, cur: offset} + r.readers = append(r.readers, rc) + } + return r, nil +} + +func NewMultiReaderAt(ss []*SeekableStream) (readerutil.SizeReaderAt, error) { + readers := make([]readerutil.SizeReaderAt, 0, len(ss)) + for _, s := range ss { + ra, err := NewReadAtSeeker(s, 0) + if err != nil { + return nil, err + } + readers = append(readers, io.NewSectionReader(ra, 0, s.GetSize())) + } + return readerutil.NewMultiReaderAt(readers...), nil +} + +func (r *RangeReadReadAtSeeker) GetRawStream() *SeekableStream { + return r.ss +} + +func (r *RangeReadReadAtSeeker) getReaderAtOffset(off int64) (*readerCur, error) { + var rc *readerCur + for _, reader := range r.readers { + if reader.cur == -1 { + continue + } + if reader.cur == off { + // logrus.Debugf("getReaderAtOffset match_%d", off) + return reader, nil + } + if reader.cur > 0 && off >= reader.cur && (rc == nil || reader.cur < rc.cur) { + rc = reader + } + } + if rc != nil && off-rc.cur <= utils.MB { + n, err := utils.CopyWithBufferN(io.Discard, rc.reader, off-rc.cur) + rc.cur += n + if err == io.EOF && rc.cur == off { + err = nil + } + if err == nil { + logrus.Debugf("getReaderAtOffset old_%d", off) + return rc, nil + } + rc.cur = -1 + } + logrus.Debugf("getReaderAtOffset new_%d", off) + + // Range请求不能超过文件大小,有些云盘处理不了就会返回整个文件 + reader, err := r.ss.RangeRead(http_range.Range{Start: off, Length: r.ss.GetSize() - off}) + if err != nil { + return nil, err + } + rc = &readerCur{reader: reader, cur: off} + r.readers = append(r.readers, rc) + return rc, nil +} + +func (r *RangeReadReadAtSeeker) ReadAt(p []byte, off int64) (int, error) { + if off == 0 && r.headCache != nil { + return r.headCache.read(p) + } + rc, err := r.getReaderAtOffset(off) + if err != nil { + return 0, err + } + n, num := 0, 0 + for num < len(p) { + n, err = rc.reader.Read(p[num:]) + rc.cur += int64(n) + num += n + if err == nil { + continue + } + if err == io.EOF { + // io.EOF是reader读取完了 + rc.cur = -1 + // yeka/zip包 没有处理EOF,我们要兼容 + // https://github.com/yeka/zip/blob/03d6312748a9d6e0bc0c9a7275385c09f06d9c14/reader.go#L433 + if num == len(p) { + err = nil + } + } + break + } + return num, err +} + +func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + case io.SeekCurrent: + if offset == 0 { + return r.masterOff, nil + } + offset += r.masterOff + case io.SeekEnd: + offset += r.ss.GetSize() + default: + return 0, errs.NotSupport + } + if offset < 0 { + return r.masterOff, errors.New("invalid seek: negative position") + } + if offset > r.ss.GetSize() { + return r.masterOff, io.EOF + } + r.masterOff = offset + return offset, nil +} + +func (r *RangeReadReadAtSeeker) Read(p []byte) (n int, err error) { + if r.masterOff == 0 && r.headCache != nil { + return r.headCache.read(p) + } + rc, err := r.getReaderAtOffset(r.masterOff) + if err != nil { + return 0, err + } + n, err = rc.reader.Read(p) + rc.cur += int64(n) + r.masterOff += int64(n) + return n, err +} + +func (r *RangeReadReadAtSeeker) Close() error { + return r.ss.Close() +} + +type FileReadAtSeeker struct { + ss *SeekableStream +} + +func (f *FileReadAtSeeker) GetRawStream() *SeekableStream { + return f.ss +} + +func (f *FileReadAtSeeker) Read(p []byte) (n int, err error) { + return f.ss.mFile.Read(p) +} + +func (f *FileReadAtSeeker) ReadAt(p []byte, off int64) (n int, err error) { + return f.ss.mFile.ReadAt(p, off) +} + +func (f *FileReadAtSeeker) Seek(offset int64, whence int) (int64, error) { + return f.ss.mFile.Seek(offset, whence) +} + +func (f *FileReadAtSeeker) Close() error { + return f.ss.Close() } diff --git a/internal/stream/util.go b/internal/stream/util.go index 7d2b7ef7509..5b935a9043e 100644 --- a/internal/stream/util.go +++ b/internal/stream/util.go @@ -2,14 +2,15 @@ package stream import ( "context" + "encoding/hex" "fmt" "io" "net/http" - "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" log "github.com/sirupsen/logrus" ) @@ -17,10 +18,9 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl if len(link.URL) == 0 { return nil, fmt.Errorf("can't create RangeReadCloser since URL is empty in link") } - //remoteClosers := utils.EmptyClosers() rangeReaderFunc := func(ctx context.Context, r http_range.Range) (io.ReadCloser, error) { if link.Concurrency != 0 || link.PartSize != 0 { - header := net.ProcessHeader(http.Header{}, link.Header) + header := net.ProcessHeader(nil, link.Header) down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize @@ -32,44 +32,36 @@ func GetRangeReadCloserFromLink(size int64, link *model.Link) (model.RangeReadCl HeaderRef: header, } rc, err := down.Download(ctx, req) - if err != nil { - return nil, errs.NewErr(err, "GetReadCloserFromLink failed") - } - return rc, nil + return rc, err } - if len(link.URL) > 0 { - response, err := RequestRangedHttp(ctx, link, r.Start, r.Length) - if err != nil { - if response == nil { - return nil, fmt.Errorf("http request failure, err:%s", err) - } - return nil, fmt.Errorf("http request failure,status: %d err:%s", response.StatusCode, err) - } - if r.Start == 0 && (r.Length == -1 || r.Length == size) || response.StatusCode == http.StatusPartialContent || - checkContentRange(&response.Header, r.Start) { - return response.Body, nil - } else if response.StatusCode == http.StatusOK { - log.Warnf("remote http server not supporting range request, expect low perfromace!") - readCloser, err := net.GetRangedHttpReader(response.Body, r.Start, r.Length) - if err != nil { - return nil, err - } - return readCloser, nil - + response, err := RequestRangedHttp(ctx, link, r.Start, r.Length) + if err != nil { + if response == nil { + return nil, fmt.Errorf("http request failure, err:%s", err) } - + return nil, err + } + if r.Start == 0 && (r.Length == -1 || r.Length == size) || response.StatusCode == http.StatusPartialContent || + checkContentRange(&response.Header, r.Start) { return response.Body, nil + } else if response.StatusCode == http.StatusOK { + log.Warnf("remote http server not supporting range request, expect low perfromace!") + readCloser, err := net.GetRangedHttpReader(response.Body, r.Start, r.Length) + if err != nil { + return nil, err + } + return readCloser, nil } - return nil, errs.NotSupport + return response.Body, nil } resultRangeReadCloser := model.RangeReadCloser{RangeReader: rangeReaderFunc} return &resultRangeReadCloser, nil } func RequestRangedHttp(ctx context.Context, link *model.Link, offset, length int64) (*http.Response, error) { - header := net.ProcessHeader(http.Header{}, link.Header) + header := net.ProcessHeader(nil, link.Header) header = http_range.ApplyRangeToHttpHeader(http_range.Range{Start: offset, Length: length}, header) return net.RequestHttp(ctx, "GET", header, link.URL) @@ -86,3 +78,64 @@ func checkContentRange(header *http.Header, offset int64) bool { } return false } + +type ReaderWithCtx struct { + io.Reader + Ctx context.Context +} + +func (r *ReaderWithCtx) Read(p []byte) (n int, err error) { + if utils.IsCanceled(r.Ctx) { + return 0, r.Ctx.Err() + } + return r.Reader.Read(p) +} + +func (r *ReaderWithCtx) Close() error { + if c, ok := r.Reader.(io.Closer); ok { + return c.Close() + } + return nil +} + +func CacheFullInTempFileAndUpdateProgress(stream model.FileStreamer, up model.UpdateProgress) (model.File, error) { + if cache := stream.GetFile(); cache != nil { + up(100) + return cache, nil + } + tmpF, err := utils.CreateTempFile(&ReaderUpdatingProgress{ + Reader: stream, + UpdateProgress: up, + }, stream.GetSize()) + if err == nil { + stream.SetTmpFile(tmpF) + } + return tmpF, err +} + +func CacheFullInTempFileAndWriter(stream model.FileStreamer, w io.Writer) (model.File, error) { + if cache := stream.GetFile(); cache != nil { + _, err := cache.Seek(0, io.SeekStart) + if err == nil { + _, err = utils.CopyWithBuffer(w, cache) + if err == nil { + _, err = cache.Seek(0, io.SeekStart) + } + } + return cache, err + } + tmpF, err := utils.CreateTempFile(io.TeeReader(stream, w), stream.GetSize()) + if err == nil { + stream.SetTmpFile(tmpF) + } + return tmpF, err +} + +func CacheFullInTempFileAndHash(stream model.FileStreamer, hashType *utils.HashType, params ...any) (model.File, string, error) { + h := hashType.NewFunc(params...) + tmpF, err := CacheFullInTempFileAndWriter(stream, h) + if err != nil { + return nil, "", err + } + return tmpF, hex.EncodeToString(h.Sum(nil)), err +} diff --git a/internal/task/base.go b/internal/task/base.go new file mode 100644 index 00000000000..c3703bd161f --- /dev/null +++ b/internal/task/base.go @@ -0,0 +1,90 @@ +package task + +import ( + "context" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/xhofe/tache" + "sync" + "time" +) + +type TaskExtension struct { + tache.Base + ctx context.Context + ctxInitMutex sync.Mutex + Creator *model.User + startTime *time.Time + endTime *time.Time + totalBytes int64 +} + +func (t *TaskExtension) SetCreator(creator *model.User) { + t.Creator = creator + t.Persist() +} + +func (t *TaskExtension) GetCreator() *model.User { + return t.Creator +} + +func (t *TaskExtension) SetStartTime(startTime time.Time) { + t.startTime = &startTime +} + +func (t *TaskExtension) GetStartTime() *time.Time { + return t.startTime +} + +func (t *TaskExtension) SetEndTime(endTime time.Time) { + t.endTime = &endTime +} + +func (t *TaskExtension) GetEndTime() *time.Time { + return t.endTime +} + +func (t *TaskExtension) ClearEndTime() { + t.endTime = nil +} + +func (t *TaskExtension) SetTotalBytes(totalBytes int64) { + t.totalBytes = totalBytes +} + +func (t *TaskExtension) GetTotalBytes() int64 { + return t.totalBytes +} + +func (t *TaskExtension) Ctx() context.Context { + if t.ctx == nil { + t.ctxInitMutex.Lock() + if t.ctx == nil { + t.ctx = context.WithValue(t.Base.Ctx(), "user", t.Creator) + } + t.ctxInitMutex.Unlock() + } + return t.ctx +} + +func (t *TaskExtension) ReinitCtx() { + if !conf.Conf.Tasks.AllowRetryCanceled { + return + } + select { + case <-t.Base.Ctx().Done(): + ctx, cancel := context.WithCancel(context.Background()) + t.SetCtx(ctx) + t.SetCancelFunc(cancel) + t.ctx = nil + default: + } +} + +type TaskExtensionInfo interface { + tache.TaskWithInfo + GetCreator() *model.User + GetStartTime() *time.Time + GetEndTime() *time.Time + GetTotalBytes() int64 +} diff --git a/internal/task/manager.go b/internal/task/manager.go new file mode 100644 index 00000000000..3caa685a9c9 --- /dev/null +++ b/internal/task/manager.go @@ -0,0 +1,20 @@ +package task + +import "github.com/xhofe/tache" + +type Manager[T tache.Task] interface { + Add(task T) + Cancel(id string) + CancelAll() + CancelByCondition(condition func(task T) bool) + GetAll() []T + GetByID(id string) (T, bool) + GetByState(state ...tache.State) []T + GetByCondition(condition func(task T) bool) []T + Remove(id string) + RemoveAll() + RemoveByState(state ...tache.State) + RemoveByCondition(condition func(task T) bool) + Retry(id string) + RetryAllFailed() +} diff --git a/pkg/gowebdav/client.go b/pkg/gowebdav/client.go index 6e12289c1ac..cef501b9a15 100644 --- a/pkg/gowebdav/client.go +++ b/pkg/gowebdav/client.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/xml" "fmt" + "github.com/alist-org/alist/v3/pkg/utils" "io" "net/http" "net/url" @@ -83,6 +84,11 @@ func (c *Client) SetTransport(transport http.RoundTripper) { c.c.Transport = transport } +// SetJar exposes the ability to set a cookie jar to the client. +func (c *Client) SetJar(jar http.CookieJar) { + c.c.Jar = jar +} + // Connect connects to our dav server func (c *Client) Connect() error { rs, err := c.options("/") @@ -351,6 +357,11 @@ func (c *Client) Link(path string) (string, http.Header, error) { return "", nil, newPathErrorErr("Link", path, err) } + if c.c.Jar != nil { + for _, cookie := range c.c.Jar.Cookies(r.URL) { + r.AddCookie(cookie) + } + } for k, vals := range c.headers { for _, v := range vals { r.Header.Add(k, v) @@ -409,7 +420,7 @@ func (c *Client) ReadStreamRange(path string, offset, length int64) (io.ReadClos // stream in rs.Body if rs.StatusCode == 200 { // discard first 'offset' bytes. - if _, err := io.Copy(io.Discard, io.LimitReader(rs.Body, offset)); err != nil { + if _, err := utils.CopyWithBuffer(io.Discard, io.LimitReader(rs.Body, offset)); err != nil { return nil, newPathErrorErr("ReadStreamRange", path, err) } diff --git a/pkg/mq/mq.go b/pkg/mq/mq.go index 35f5a159de4..cdae0425130 100644 --- a/pkg/mq/mq.go +++ b/pkg/mq/mq.go @@ -57,5 +57,7 @@ func (mq *inMemoryMQ[T]) Clear() { } func (mq *inMemoryMQ[T]) Len() int { + mq.Lock() + defer mq.Unlock() return mq.queue.Len() } diff --git a/internal/qbittorrent/client.go b/pkg/qbittorrent/client.go similarity index 100% rename from internal/qbittorrent/client.go rename to pkg/qbittorrent/client.go diff --git a/pkg/task/task.go b/pkg/task/task.go index f47eb7472ce..5b634f10cdb 100644 --- a/pkg/task/task.go +++ b/pkg/task/task.go @@ -26,7 +26,7 @@ type Task[K comparable] struct { Name string state string // pending, running, finished, canceling, canceled, errored status string - progress int + progress float64 Error error @@ -41,11 +41,11 @@ func (t *Task[K]) SetStatus(status string) { t.status = status } -func (t *Task[K]) SetProgress(percentage int) { +func (t *Task[K]) SetProgress(percentage float64) { t.progress = percentage } -func (t Task[K]) GetProgress() int { +func (t Task[K]) GetProgress() float64 { return t.progress } diff --git a/pkg/utils/file.go b/pkg/utils/file.go index 31803a95b2d..54247636dcb 100644 --- a/pkg/utils/file.go +++ b/pkg/utils/file.go @@ -32,7 +32,7 @@ func CopyFile(src, dst string) error { } defer dstfd.Close() - if _, err = io.Copy(dstfd, srcfd); err != nil { + if _, err = CopyWithBuffer(dstfd, srcfd); err != nil { return err } if srcinfo, err = os.Stat(src); err != nil { @@ -121,7 +121,7 @@ func CreateTempFile(r io.Reader, size int64) (*os.File, error) { if err != nil { return nil, err } - readBytes, err := io.Copy(f, r) + readBytes, err := CopyWithBuffer(f, r) if err != nil { _ = os.Remove(f.Name()) return nil, errs.NewErr(err, "CreateTempFile failed") @@ -163,8 +163,15 @@ func GetObjType(filename string, isDir bool) int { return GetFileType(filename) } +var extraMimeTypes = map[string]string{ + ".apk": "application/vnd.android.package-archive", +} + func GetMimeType(name string) string { ext := path.Ext(name) + if m, ok := extraMimeTypes[ext]; ok { + return m + } m := mime.TypeByExtension(ext) if m != "" { return m diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go index 8f8aaa26781..a281dd4e6e5 100644 --- a/pkg/utils/hash.go +++ b/pkg/utils/hash.go @@ -10,6 +10,7 @@ import ( "errors" "hash" "io" + "iter" "github.com/alist-org/alist/v3/internal/errs" log "github.com/sirupsen/logrus" @@ -96,7 +97,7 @@ func HashData(hashType *HashType, data []byte, params ...any) string { // HashReader get hash of one hashType from a reader func HashReader(hashType *HashType, reader io.Reader, params ...any) (string, error) { h := hashType.NewFunc(params...) - _, err := io.Copy(h, reader) + _, err := CopyWithBuffer(h, reader) if err != nil { return "", errs.NewErr(err, "HashReader error") } @@ -226,3 +227,13 @@ func (hi HashInfo) GetHash(ht *HashType) string { func (hi HashInfo) Export() map[*HashType]string { return hi.h } + +func (hi HashInfo) All() iter.Seq2[*HashType, string] { + return func(yield func(*HashType, string) bool) { + for hashType, hashValue := range hi.h { + if !yield(hashType, hashValue) { + return + } + } + } +} diff --git a/pkg/utils/hash_test.go b/pkg/utils/hash_test.go index 55713c1afb3..0f5a2a3b14e 100644 --- a/pkg/utils/hash_test.go +++ b/pkg/utils/hash_test.go @@ -4,7 +4,6 @@ import ( "bytes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "io" "testing" ) @@ -36,7 +35,7 @@ var hashTestSet = []hashTest{ func TestMultiHasher(t *testing.T) { for _, test := range hashTestSet { mh := NewMultiHasher([]*HashType{MD5, SHA1, SHA256}) - n, err := io.Copy(mh, bytes.NewBuffer(test.input)) + n, err := CopyWithBuffer(mh, bytes.NewBuffer(test.input)) require.NoError(t, err) assert.Len(t, test.input, int(n)) hashInfo := mh.GetHashInfo() diff --git a/pkg/utils/io.go b/pkg/utils/io.go index d106531bd3d..e06fb235b8b 100644 --- a/pkg/utils/io.go +++ b/pkg/utils/io.go @@ -5,10 +5,12 @@ import ( "context" "errors" "fmt" - "golang.org/x/exp/constraints" "io" + "sync" "time" + "golang.org/x/exp/constraints" + log "github.com/sirupsen/logrus" ) @@ -21,14 +23,14 @@ func (rf readerFunc) Read(p []byte) (n int, err error) { return rf(p) } // CopyWithCtx slightly modified function signature: // - context has been added in order to propagate cancellation // - I do not return the number of bytes written, has it is not useful in my use case -func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage int)) error { +func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, progress func(percentage float64)) error { // Copy will call the Reader and Writer interface multiple time, in order // to copy by chunk (avoiding loading the whole file in memory). // I insert the ability to cancel before read time as it is the earliest // possible in the call process. var finish int64 = 0 s := size / 100 - _, err := io.Copy(out, readerFunc(func(p []byte) (int, error) { + _, err := CopyWithBuffer(out, readerFunc(func(p []byte) (int, error) { // golang non-blocking channel: https://gobyexample.com/non-blocking-channel-operations select { // if context has been canceled @@ -40,7 +42,7 @@ func CopyWithCtx(ctx context.Context, out io.Writer, in io.Reader, size int64, p n, err := in.Read(p) if s > 0 && (err == nil || err == io.EOF) { finish += int64(n) - progress(int(finish / s)) + progress(float64(finish) / float64(s)) } return n, err } @@ -136,7 +138,7 @@ func (mr *MultiReadable) Close() error { func Retry(attempts int, sleep time.Duration, f func() error) (err error) { for i := 0; i < attempts; i++ { - fmt.Println("This is attempt number", i) + //fmt.Println("This is attempt number", i) if i > 0 { log.Println("retrying after error:", err) time.Sleep(sleep) @@ -203,3 +205,31 @@ func Max[T constraints.Ordered](a, b T) T { } return a } + +var IoBuffPool = &sync.Pool{ + New: func() interface{} { + return make([]byte, 32*1024*2) // Two times of size in io package + }, +} + +func CopyWithBuffer(dst io.Writer, src io.Reader) (written int64, err error) { + buff := IoBuffPool.Get().([]byte) + defer IoBuffPool.Put(buff) + written, err = io.CopyBuffer(dst, src, buff) + if err != nil { + return + } + return written, nil +} + +func CopyWithBufferN(dst io.Writer, src io.Reader, n int64) (written int64, err error) { + written, err = CopyWithBuffer(dst, io.LimitReader(src, n)) + if written == n { + return n, nil + } + if written < n && err == nil { + // src stopped early; must have been EOF. + err = io.EOF + } + return +} diff --git a/pkg/utils/mask.go b/pkg/utils/mask.go new file mode 100644 index 00000000000..1513ad40368 --- /dev/null +++ b/pkg/utils/mask.go @@ -0,0 +1,30 @@ +package utils + +import "strings" + +// MaskIP anonymizes middle segments of an IP address. +func MaskIP(ip string) string { + if ip == "" { + return "" + } + if strings.Contains(ip, ":") { + parts := strings.Split(ip, ":") + if len(parts) > 2 { + for i := 1; i < len(parts)-1; i++ { + if parts[i] != "" { + parts[i] = "*" + } + } + return strings.Join(parts, ":") + } + return ip + } + parts := strings.Split(ip, ".") + if len(parts) == 4 { + for i := 1; i < len(parts)-1; i++ { + parts[i] = "*" + } + return strings.Join(parts, ".") + } + return ip +} diff --git a/pkg/utils/oauth2.go b/pkg/utils/oauth2.go new file mode 100644 index 00000000000..c1ad161245f --- /dev/null +++ b/pkg/utils/oauth2.go @@ -0,0 +1,15 @@ +package utils + +import "golang.org/x/oauth2" + +type tokenSource struct { + fn func() (*oauth2.Token, error) +} + +func (t *tokenSource) Token() (*oauth2.Token, error) { + return t.fn() +} + +func TokenSource(fn func() (*oauth2.Token, error)) oauth2.TokenSource { + return &tokenSource{fn} +} diff --git a/pkg/utils/path.go b/pkg/utils/path.go index c0793a3ec0f..fe4ff2fd96a 100644 --- a/pkg/utils/path.go +++ b/pkg/utils/path.go @@ -45,7 +45,7 @@ func IsSubPath(path string, subPath string) bool { func Ext(path string) string { ext := stdpath.Ext(path) - if strings.HasPrefix(ext, ".") { + if len(ext) > 0 && ext[0] == '.' { ext = ext[1:] } return strings.ToLower(ext) @@ -88,9 +88,51 @@ func JoinBasePath(basePath, reqPath string) (string, error) { strings.Contains(reqPath, "/../") { return "", errs.RelativePath } + + reqPath = FixAndCleanPath(reqPath) + + if strings.HasPrefix(reqPath, "/") { + return reqPath, nil + } + return stdpath.Join(FixAndCleanPath(basePath), FixAndCleanPath(reqPath)), nil } func GetFullPath(mountPath, path string) string { return stdpath.Join(GetActualMountPath(mountPath), path) } + +// ValidateNameComponent validates a single path component. +// It rejects empty names, dot segments, separators, ".." sequences, and NUL bytes. +func ValidateNameComponent(name string) error { + if name == "" { + return errs.InvalidName + } + if name == "." || name == ".." { + return errs.InvalidName + } + if strings.Contains(name, "/") || strings.Contains(name, "\\") { + return errs.InvalidName + } + if strings.Contains(name, "..") { + return errs.InvalidName + } + if strings.ContainsRune(name, 0) { + return errs.InvalidName + } + return nil +} + +// JoinUnderBase safely joins baseDir with a single name component and ensures the +// result stays under baseDir after normalization. +func JoinUnderBase(baseDir, name string) (string, error) { + if err := ValidateNameComponent(name); err != nil { + return "", err + } + base := FixAndCleanPath(baseDir) + joined := FixAndCleanPath(stdpath.Join(base, name)) + if !IsSubPath(base, joined) { + return "", errs.InvalidName + } + return joined, nil +} diff --git a/pkg/utils/path_test.go b/pkg/utils/path_test.go index f42f2f8bb5d..def286b8038 100644 --- a/pkg/utils/path_test.go +++ b/pkg/utils/path_test.go @@ -20,3 +20,49 @@ func TestFixAndCleanPath(t *testing.T) { } } } + +func TestValidateNameComponent(t *testing.T) { + validNames := []string{ + "file.txt", + "abc", + "file_name-1", + } + for _, name := range validNames { + if err := ValidateNameComponent(name); err != nil { + t.Fatalf("expected valid name %q, got error: %v", name, err) + } + } + + invalidNames := []string{ + "", + ".", + "..", + "a/b", + `a\b`, + "a..b", + string([]byte{'a', 0, 'b'}), + } + for _, name := range invalidNames { + if err := ValidateNameComponent(name); err == nil { + t.Fatalf("expected invalid name %q to be rejected", name) + } + } +} + +func TestJoinUnderBase(t *testing.T) { + base := "/lanzou-y/shared/test1" + out, err := JoinUnderBase(base, "file.txt") + if err != nil { + t.Fatalf("expected join success, got error: %v", err) + } + if out != "/lanzou-y/shared/test1/file.txt" { + t.Fatalf("unexpected join result: %s", out) + } + + if _, err := JoinUnderBase(base, "../admin/screts.txt"); err == nil { + t.Fatalf("expected traversal to be rejected") + } + if _, err := JoinUnderBase(base, "sub/child"); err == nil { + t.Fatalf("expected nested path to be rejected") + } +} diff --git a/pkg/utils/random/random.go b/pkg/utils/random/random.go index 65fbf14a0d3..c3f3dd48377 100644 --- a/pkg/utils/random/random.go +++ b/pkg/utils/random/random.go @@ -1,20 +1,27 @@ package random import ( - "math/rand" + "crypto/rand" + "math/big" + mathRand "math/rand" "time" "github.com/google/uuid" ) -var Rand *rand.Rand +var Rand *mathRand.Rand const letterBytes = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" func String(n int) string { b := make([]byte, n) + letterLen := big.NewInt(int64(len(letterBytes))) for i := range b { - b[i] = letterBytes[Rand.Intn(len(letterBytes))] + idx, err := rand.Int(rand.Reader, letterLen) + if err != nil { + panic(err) + } + b[i] = letterBytes[idx.Int64()] } return string(b) } @@ -24,10 +31,10 @@ func Token() string { } func RangeInt64(left, right int64) int64 { - return rand.Int63n(left+right) - left + return mathRand.Int63n(left+right) - left } func init() { - s := rand.NewSource(time.Now().UnixNano()) - Rand = rand.New(s) + s := mathRand.NewSource(time.Now().UnixNano()) + Rand = mathRand.New(s) } diff --git a/pkg/utils/slice.go b/pkg/utils/slice.go index 73bac93b4d0..842995daaf1 100644 --- a/pkg/utils/slice.go +++ b/pkg/utils/slice.go @@ -29,6 +29,20 @@ func SliceContains[T comparable](arr []T, v T) bool { return false } +// SliceAllContains check if slice all contains elements +func SliceAllContains[T comparable](arr []T, vs ...T) bool { + vsMap := make(map[T]struct{}) + for _, v := range arr { + vsMap[v] = struct{}{} + } + for _, v := range vs { + if _, ok := vsMap[v]; !ok { + return false + } + } + return true +} + // SliceConvert convert slice to another type slice func SliceConvert[S any, D any](srcS []S, convert func(src S) (D, error)) ([]D, error) { res := make([]D, 0, len(srcS)) @@ -79,3 +93,9 @@ func SliceFilter[T any](arr []T, filter func(src T) bool) []T { } return res } + +func SliceReplace[T any](arr []T, replace func(src T) T) { + for i, src := range arr { + arr[i] = replace(src) + } +} diff --git a/pkg/utils/time.go b/pkg/utils/time.go index a9d9b5b674c..36573b4ecf4 100644 --- a/pkg/utils/time.go +++ b/pkg/utils/time.go @@ -34,6 +34,36 @@ func NewDebounce2(interval time.Duration, f func()) func() { if timer == nil { timer = time.AfterFunc(interval, f) } - (*time.Timer)(timer).Reset(interval) + timer.Reset(interval) + } +} + +func NewThrottle(interval time.Duration) func(func()) { + var lastCall time.Time + var lock sync.Mutex + return func(fn func()) { + lock.Lock() + defer lock.Unlock() + + now := time.Now() + if now.Sub(lastCall) >= interval { + lastCall = now + go fn() + } + } +} + +func NewThrottle2(interval time.Duration, fn func()) func() { + var lastCall time.Time + var lock sync.Mutex + return func() { + lock.Lock() + defer lock.Unlock() + + now := time.Now() + if now.Sub(lastCall) >= interval { + lastCall = now + go fn() + } } } diff --git a/public/public.go b/public/public.go index a158f3e0226..e94146c3b08 100644 --- a/public/public.go +++ b/public/public.go @@ -2,5 +2,5 @@ package public import "embed" -//go:embed dist +//go:embed all:dist var Public embed.FS diff --git a/server/common/auth.go b/server/common/auth.go index 017390bdd7a..0de718cf9e8 100644 --- a/server/common/auth.go +++ b/server/common/auth.go @@ -3,7 +3,9 @@ package common import ( "time" + "github.com/Xhofe/go-cache" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" "github.com/golang-jwt/jwt/v4" "github.com/pkg/errors" ) @@ -12,12 +14,16 @@ var SecretKey []byte type UserClaims struct { Username string `json:"username"` + PwdTS int64 `json:"pwd_ts"` jwt.RegisteredClaims } -func GenerateToken(username string) (tokenString string, err error) { +var validTokenCache = cache.NewMemCache[bool]() + +func GenerateToken(user *model.User) (tokenString string, err error) { claim := UserClaims{ - Username: username, + Username: user.Username, + PwdTS: user.PwdTS, RegisteredClaims: jwt.RegisteredClaims{ ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Duration(conf.Conf.TokenExpiresIn) * time.Hour)), IssuedAt: jwt.NewNumericDate(time.Now()), @@ -25,6 +31,10 @@ func GenerateToken(username string) (tokenString string, err error) { }} token := jwt.NewWithClaims(jwt.SigningMethodHS256, claim) tokenString, err = token.SignedString(SecretKey) + if err != nil { + return "", err + } + validTokenCache.Set(tokenString, true) return tokenString, err } @@ -32,6 +42,9 @@ func ParseToken(tokenString string) (*UserClaims, error) { token, err := jwt.ParseWithClaims(tokenString, &UserClaims{}, func(token *jwt.Token) (interface{}, error) { return SecretKey, nil }) + if IsTokenInvalidated(tokenString) { + return nil, errors.New("token is invalidated") + } if err != nil { if ve, ok := err.(*jwt.ValidationError); ok { if ve.Errors&jwt.ValidationErrorMalformed != 0 { @@ -50,3 +63,16 @@ func ParseToken(tokenString string) (*UserClaims, error) { } return nil, errors.New("couldn't handle this token") } + +func InvalidateToken(tokenString string) error { + if tokenString == "" { + return nil // don't invalidate empty guest token + } + validTokenCache.Del(tokenString) + return nil +} + +func IsTokenInvalidated(tokenString string) bool { + _, ok := validTokenCache.Get(tokenString) + return !ok +} diff --git a/server/common/base.go b/server/common/base.go index eb6ef2b8ac2..11a28d25039 100644 --- a/server/common/base.go +++ b/server/common/base.go @@ -12,16 +12,16 @@ import ( func GetApiUrl(r *http.Request) string { api := conf.Conf.SiteURL if strings.HasPrefix(api, "http") { - return api + return strings.TrimSuffix(api, "/") } if r != nil { protocol := "http" if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { protocol = "https" } - host := r.Host - if r.Header.Get("X-Forwarded-Host") != "" { - host = r.Header.Get("X-Forwarded-Host") + host := r.Header.Get("X-Forwarded-Host") + if host == "" { + host = r.Host } api = fmt.Sprintf("%s://%s", protocol, stdpath.Join(host, api)) } diff --git a/server/common/check.go b/server/common/check.go index 1f4227b000d..010d3131745 100644 --- a/server/common/check.go +++ b/server/common/check.go @@ -1,10 +1,6 @@ package common import ( - "path" - "regexp" - "strings" - "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/model" @@ -32,30 +28,11 @@ func IsApply(metaPath, reqPath string, applySub bool) bool { } func CanAccess(user *model.User, meta *model.Meta, reqPath string, password string) bool { - // if the reqPath is in hide (only can check the nearest meta) and user can't see hides, can't access - if meta != nil && !user.CanSeeHides() && meta.Hide != "" && - IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { // the meta should apply to the parent of current path - for _, hide := range strings.Split(meta.Hide, "\n") { - re := regexp.MustCompile(hide) - if re.MatchString(path.Base(reqPath)) { - return false - } - } - } - // if is not guest and can access without password - if user.CanAccessWithoutPassword() { - return true - } - // if meta is nil or password is empty, can access - if meta == nil || meta.Password == "" { - return true - } - // if meta doesn't apply to sub_folder, can access - if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { - return true - } - // validate password - return meta.Password == password + // Deprecated: CanAccess is kept for backward compatibility. + // The logic has been moved to CanAccessWithRoles which performs the + // necessary checks based on role permissions. This wrapper ensures + // older calls still work without relying on user permission bits. + return CanAccessWithRoles(user, meta, reqPath, password) } // ShouldProxy TODO need optimize @@ -64,7 +41,11 @@ func CanAccess(user *model.User, meta *model.Meta, reqPath string, password stri // 2. storage.WebProxy // 3. proxy_types func ShouldProxy(storage driver.Driver, filename string) bool { - if storage.Config().MustProxy() || storage.GetStorage().WebProxy { + if proxyDriver, ok := storage.(driver.ProxyDriver); ok { + if proxyDriver.ShouldProxyDownloads() || storage.GetStorage().WebProxy { + return true + } + } else if storage.Config().MustProxy() || storage.GetStorage().WebProxy { return true } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { diff --git a/server/common/common.go b/server/common/common.go index 28d2da4443d..33ae704e86b 100644 --- a/server/common/common.go +++ b/server/common/common.go @@ -1,6 +1,8 @@ package common import ( + "context" + "net/http" "strings" "github.com/alist-org/alist/v3/cmd/flags" @@ -66,17 +68,32 @@ func ErrorStrResp(c *gin.Context, str string, code int, l ...bool) { } func SuccessResp(c *gin.Context, data ...interface{}) { - if len(data) == 0 { - c.JSON(200, Resp[interface{}]{ - Code: 200, - Message: "success", - Data: nil, - }) - return + SuccessWithMsgResp(c, "success", data...) +} + +func SuccessWithMsgResp(c *gin.Context, msg string, data ...interface{}) { + var respData interface{} + if len(data) > 0 { + respData = data[0] } + c.JSON(200, Resp[interface{}]{ Code: 200, - Message: "success", - Data: data[0], + Message: msg, + Data: respData, }) } + +func Pluralize(count int, singular, plural string) string { + if count == 1 { + return singular + } + return plural +} + +func GetHttpReq(ctx context.Context) *http.Request { + if c, ok := ctx.(*gin.Context); ok { + return c.Request + } + return nil +} diff --git a/server/common/proxy.go b/server/common/proxy.go index 370e46eb21f..dae97c9bf1c 100644 --- a/server/common/proxy.go +++ b/server/common/proxy.go @@ -6,33 +6,53 @@ import ( "io" "net/http" "net/url" + "os" + "strings" + + "maps" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/net" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/http_range" "github.com/alist-org/alist/v3/pkg/utils" + log "github.com/sirupsen/logrus" ) func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model.Obj) error { if link.MFile != nil { defer link.MFile.Close() - attachFileName(w, file) - http.ServeContent(w, r, file.GetName(), file.ModTime(), link.MFile) + attachHeader(w, file) + contentType := link.Header.Get("Content-Type") + if contentType != "" { + w.Header().Set("Content-Type", contentType) + } + mFile := link.MFile + if _, ok := mFile.(*os.File); !ok { + mFile = &stream.RateLimitFile{ + File: mFile, + Limiter: stream.ServerDownloadLimit, + Ctx: r.Context(), + } + } + http.ServeContent(w, r, file.GetName(), file.ModTime(), mFile) return nil } else if link.RangeReadCloser != nil { - attachFileName(w, file) - net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), link.RangeReadCloser.RangeRead) - defer func() { - _ = link.RangeReadCloser.Close() - }() - return nil + attachHeader(w, file) + return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ + RangeReadCloserIF: link.RangeReadCloser, + Limiter: stream.ServerDownloadLimit, + }) } else if link.Concurrency != 0 || link.PartSize != 0 { - attachFileName(w, file) + attachHeader(w, file) size := file.GetSize() - //var finalClosers model.Closers - finalClosers := utils.EmptyClosers() - header := net.ProcessHeader(r.Header, link.Header) rangeReader := func(ctx context.Context, httpRange http_range.Range) (io.ReadCloser, error) { + requestHeader := ctx.Value("request_header") + if requestHeader == nil { + requestHeader = http.Header{} + } + header := net.ProcessHeader(requestHeader.(http.Header), link.Header) down := net.NewDownloader(func(d *net.Downloader) { d.Concurrency = link.Concurrency d.PartSize = link.PartSize @@ -44,36 +64,105 @@ func Proxy(w http.ResponseWriter, r *http.Request, link *model.Link, file model. HeaderRef: header, } rc, err := down.Download(ctx, req) - finalClosers.Add(rc) return rc, err } - net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), rangeReader) - defer finalClosers.Close() - return nil + return net.ServeHTTP(w, r, file.GetName(), file.ModTime(), file.GetSize(), &stream.RateLimitRangeReadCloser{ + RangeReadCloserIF: &model.RangeReadCloser{RangeReader: rangeReader}, + Limiter: stream.ServerDownloadLimit, + }) } else { //transparent proxy header := net.ProcessHeader(r.Header, link.Header) - res, err := net.RequestHttp(context.Background(), r.Method, header, link.URL) + res, err := net.RequestHttp(r.Context(), r.Method, header, link.URL) if err != nil { return err } defer res.Body.Close() - for h, v := range res.Header { - w.Header()[h] = v + maps.Copy(w.Header(), res.Header) + if r.URL.Query().Get("type") == "preview" { + w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s"; filename*=UTF-8''%s`, file.GetName(), url.PathEscape(file.GetName()))) } w.WriteHeader(res.StatusCode) if r.Method == http.MethodHead { return nil } - _, err = io.Copy(w, res.Body) - if err != nil { - return err - } - return nil + _, err = utils.CopyWithBuffer(w, &stream.RateLimitReader{ + Reader: res.Body, + Limiter: stream.ServerDownloadLimit, + Ctx: r.Context(), + }) + return err } } -func attachFileName(w http.ResponseWriter, file model.Obj) { +func attachHeader(w http.ResponseWriter, file model.Obj) { fileName := file.GetName() w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, fileName, url.PathEscape(fileName))) + w.Header().Set("Content-Type", utils.GetMimeType(fileName)) + w.Header().Set("Etag", GetEtag(file)) +} +func GetEtag(file model.Obj) string { + hash := "" + for _, v := range file.GetHash().Export() { + if strings.Compare(v, hash) > 0 { + hash = v + } + } + if len(hash) > 0 { + return fmt.Sprintf(`"%s"`, hash) + } + // 参考nginx + return fmt.Sprintf(`"%x-%x"`, file.ModTime().Unix(), file.GetSize()) +} + +var NoProxyRange = &model.RangeReadCloser{} + +func ProxyRange(link *model.Link, size int64) { + if link.MFile != nil { + return + } + if link.RangeReadCloser == nil { + var rrc, err = stream.GetRangeReadCloserFromLink(size, link) + if err != nil { + log.Warnf("ProxyRange error: %s", err) + return + } + link.RangeReadCloser = rrc + } else if link.RangeReadCloser == NoProxyRange { + link.RangeReadCloser = nil + } +} + +func BuildDownProxyURL(downProxyURL, path string, useSign bool) string { + base := strings.Split(downProxyURL, "\n")[0] + if useSign { + return fmt.Sprintf("%s%s?sign=%s", base, utils.EncodePath(path, true), sign.Sign(path)) + } + return fmt.Sprintf("%s%s", base, utils.EncodePath(path, true)) +} + +type InterceptResponseWriter struct { + http.ResponseWriter + io.Writer +} + +func (iw *InterceptResponseWriter) Write(p []byte) (int, error) { + return iw.Writer.Write(p) +} + +type WrittenResponseWriter struct { + http.ResponseWriter + written bool +} + +func (ww *WrittenResponseWriter) Write(p []byte) (int, error) { + n, err := ww.ResponseWriter.Write(p) + if !ww.written && n > 0 { + ww.written = true + } + return n, err +} + +func (ww *WrittenResponseWriter) IsWritten() bool { + return ww.written } diff --git a/server/common/role_perm.go b/server/common/role_perm.go new file mode 100644 index 00000000000..ec82d4d91a0 --- /dev/null +++ b/server/common/role_perm.go @@ -0,0 +1,139 @@ +package common + +import ( + "path" + "strings" + + "github.com/dlclark/regexp2" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" +) + +const ( + PermSeeHides = iota + PermAccessWithoutPassword + PermAddOfflineDownload + PermWrite + PermRename + PermMove + PermCopy + PermRemove + PermWebdavRead + PermWebdavManage + PermFTPAccess + PermFTPManage + PermReadArchives + PermDecompress + PermPathLimit + PermMCPAccess + PermMCPManage +) + +func HasPermission(perm int32, bit uint) bool { + return (perm>>bit)&1 == 1 +} + +func MergeRolePermissions(u *model.User, reqPath string) int32 { + if u == nil { + return 0 + } + var perm int32 + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + if reqPath == "/" || utils.PathEqual(reqPath, u.BasePath) { + for _, entry := range role.PermissionScopes { + perm |= entry.Permission + } + } else { + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(entry.Path, reqPath) { + perm |= entry.Permission + } + } + } + } + return perm +} + +func CanAccessWithRoles(u *model.User, meta *model.Meta, reqPath, password string) bool { + if !CanReadPathByRole(u, reqPath) { + return false + } + perm := MergeRolePermissions(u, reqPath) + if meta != nil && !HasPermission(perm, PermSeeHides) && meta.Hide != "" && + IsApply(meta.Path, path.Dir(reqPath), meta.HSub) { + for _, hide := range strings.Split(meta.Hide, "\n") { + re := regexp2.MustCompile(hide, regexp2.None) + if isMatch, _ := re.MatchString(path.Base(reqPath)); isMatch { + return false + } + } + } + if HasPermission(perm, PermAccessWithoutPassword) { + return true + } + if meta == nil || meta.Password == "" { + return true + } + if !utils.PathEqual(meta.Path, reqPath) && !meta.PSub { + return true + } + return meta.Password == password +} + +func CanReadPathByRole(u *model.User, reqPath string) bool { + if u == nil { + return false + } + if reqPath == "/" || utils.PathEqual(reqPath, u.BasePath) { + return len(u.Role) > 0 + } + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.PathEqual(entry.Path, reqPath) || utils.IsSubPath(entry.Path, reqPath) || utils.IsSubPath(reqPath, entry.Path) { + return true + } + } + } + return false +} + +// HasChildPermission checks whether any child path under reqPath grants the +// specified permission bit. +func HasChildPermission(u *model.User, reqPath string, bit uint) bool { + if u == nil { + return false + } + for _, rid := range u.Role { + role, err := op.GetRole(uint(rid)) + if err != nil { + continue + } + for _, entry := range role.PermissionScopes { + if utils.IsSubPath(reqPath, entry.Path) && HasPermission(entry.Permission, bit) { + return true + } + } + } + return false +} + +// CheckPathLimitWithRoles checks whether the path is allowed when the user has +// the `PermPathLimit` permission for the target path. When the user does not +// have this permission, the check passes by default. +func CheckPathLimitWithRoles(u *model.User, reqPath string) bool { + perm := MergeRolePermissions(u, reqPath) + if HasPermission(perm, PermPathLimit) { + return CanReadPathByRole(u, reqPath) + } + return true +} diff --git a/server/debug.go b/server/debug.go index 081ef8c3381..a4242abdb2b 100644 --- a/server/debug.go +++ b/server/debug.go @@ -5,6 +5,7 @@ import ( _ "net/http/pprof" "runtime" + "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/middlewares" "github.com/gin-gonic/gin" @@ -15,7 +16,7 @@ func _pprof(g *gin.RouterGroup) { } func debug(g *gin.RouterGroup) { - g.GET("/path/*path", middlewares.Down, func(ctx *gin.Context) { + g.GET("/path/*path", middlewares.Down(sign.Verify), func(ctx *gin.Context) { rawPath := ctx.MustGet("path").(string) ctx.JSON(200, gin.H{ "path": rawPath, diff --git a/server/ftp.go b/server/ftp.go new file mode 100644 index 00000000000..d41063731bf --- /dev/null +++ b/server/ftp.go @@ -0,0 +1,290 @@ +package server + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/alist-org/alist/v3/server/ftp" + "math/rand" + "net" + "net/http" + "os" + "strconv" + "strings" + "sync" +) + +type FtpMainDriver struct { + settings *ftpserver.Settings + proxyHeader *http.Header + clients map[uint32]ftpserver.ClientContext + shutdownLock sync.RWMutex + isShutdown bool + tlsConfig *tls.Config +} + +func NewMainDriver() (*FtpMainDriver, error) { + header := &http.Header{} + header.Add("User-Agent", setting.GetStr(conf.FTPProxyUserAgent)) + transferType := ftpserver.TransferTypeASCII + if conf.Conf.FTP.DefaultTransferBinary { + transferType = ftpserver.TransferTypeBinary + } + activeConnCheck := ftpserver.IPMatchDisabled + if conf.Conf.FTP.EnableActiveConnIPCheck { + activeConnCheck = ftpserver.IPMatchRequired + } + pasvConnCheck := ftpserver.IPMatchDisabled + if conf.Conf.FTP.EnablePasvConnIPCheck { + pasvConnCheck = ftpserver.IPMatchRequired + } + tlsRequired := ftpserver.ClearOrEncrypted + if setting.GetBool(conf.FTPImplicitTLS) { + tlsRequired = ftpserver.ImplicitEncryption + } else if setting.GetBool(conf.FTPMandatoryTLS) { + tlsRequired = ftpserver.MandatoryEncryption + } + tlsConf, err := getTlsConf(setting.GetStr(conf.FTPTLSPrivateKeyPath), setting.GetStr(conf.FTPTLSPublicCertPath)) + if err != nil && tlsRequired != ftpserver.ClearOrEncrypted { + return nil, fmt.Errorf("FTP mandatory TLS has been enabled, but the certificate failed to load: %w", err) + } + return &FtpMainDriver{ + settings: &ftpserver.Settings{ + ListenAddr: conf.Conf.FTP.Listen, + PublicHost: lookupIP(setting.GetStr(conf.FTPPublicHost)), + PassiveTransferPortGetter: newPortMapper(setting.GetStr(conf.FTPPasvPortMap)), + FindPasvPortAttempts: conf.Conf.FTP.FindPasvPortAttempts, + ActiveTransferPortNon20: conf.Conf.FTP.ActiveTransferPortNon20, + IdleTimeout: conf.Conf.FTP.IdleTimeout, + ConnectionTimeout: conf.Conf.FTP.ConnectionTimeout, + DisableMLSD: false, + DisableMLST: false, + DisableMFMT: true, + Banner: setting.GetStr(conf.Announcement), + TLSRequired: tlsRequired, + DisableLISTArgs: false, + DisableSite: false, + DisableActiveMode: conf.Conf.FTP.DisableActiveMode, + EnableHASH: false, + DisableSTAT: false, + DisableSYST: false, + EnableCOMB: false, + DefaultTransferType: transferType, + ActiveConnectionsCheck: activeConnCheck, + PasvConnectionsCheck: pasvConnCheck, + SiteHandlers: map[string]ftpserver.SiteHandler{ + "SIZE": ftp.HandleSIZE, + }, + }, + proxyHeader: header, + clients: make(map[uint32]ftpserver.ClientContext), + shutdownLock: sync.RWMutex{}, + isShutdown: false, + tlsConfig: tlsConf, + }, nil +} + +func (d *FtpMainDriver) GetSettings() (*ftpserver.Settings, error) { + return d.settings, nil +} + +func (d *FtpMainDriver) ClientConnected(cc ftpserver.ClientContext) (string, error) { + if d.isShutdown || !d.shutdownLock.TryRLock() { + return "", errors.New("server has shutdown") + } + defer d.shutdownLock.RUnlock() + d.clients[cc.ID()] = cc + return "AList FTP Endpoint", nil +} + +func (d *FtpMainDriver) ClientDisconnected(cc ftpserver.ClientContext) { + err := cc.Close() + if err != nil { + utils.Log.Errorf("failed to close client: %v", err) + } + delete(d.clients, cc.ID()) +} + +func (d *FtpMainDriver) AuthUser(cc ftpserver.ClientContext, user, pass string) (ftpserver.ClientDriver, error) { + var userObj *model.User + var err error + if user == "anonymous" || user == "guest" { + userObj, err = op.GetGuest() + if err != nil { + return nil, err + } + } else { + userObj, err = op.GetUserByName(user) + if err != nil { + return nil, err + } + passHash := model.StaticHash(pass) + if err = userObj.ValidatePwdStaticHash(passHash); err != nil { + return nil, err + } + } + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { + return nil, errors.New("user is not allowed to access via FTP") + } + + ctx := context.Background() + ctx = context.WithValue(ctx, "user", userObj) + if user == "anonymous" || user == "guest" { + ctx = context.WithValue(ctx, "meta_pass", pass) + } else { + ctx = context.WithValue(ctx, "meta_pass", "") + } + ctx = context.WithValue(ctx, "client_ip", cc.RemoteAddr().String()) + ctx = context.WithValue(ctx, "proxy_header", d.proxyHeader) + return ftp.NewAferoAdapter(ctx), nil +} + +func (d *FtpMainDriver) GetTLSConfig() (*tls.Config, error) { + if d.tlsConfig == nil { + return nil, errors.New("TLS config not provided") + } + return d.tlsConfig, nil +} + +func (d *FtpMainDriver) Stop() { + d.isShutdown = true + d.shutdownLock.Lock() + defer d.shutdownLock.Unlock() + for _, value := range d.clients { + _ = value.Close() + } +} + +func lookupIP(host string) string { + if host == "" || net.ParseIP(host) != nil { + return host + } + ips, err := net.LookupIP(host) + if err != nil || len(ips) == 0 { + utils.Log.Fatalf("given FTP public host is invalid, and the default value will be used: %v", err) + return "" + } + for _, ip := range ips { + if ip.To4() != nil { + return ip.String() + } + } + v6 := ips[0].String() + utils.Log.Warnf("no IPv4 record looked up, %s will be used as public host, and it might do not work.", v6) + return v6 +} + +func newPortMapper(str string) ftpserver.PasvPortGetter { + if str == "" { + return nil + } + pasvPortMappers := strings.Split(strings.Replace(str, "\n", ",", -1), ",") + type group struct { + ExposedStart int + ListenedStart int + Length int + } + groups := make([]group, len(pasvPortMappers)) + totalLength := 0 + convertToPorts := func(str string) (int, int, error) { + start, end, multi := strings.Cut(str, "-") + if multi { + si, err := strconv.Atoi(start) + if err != nil { + return 0, 0, err + } + ei, err := strconv.Atoi(end) + if err != nil { + return 0, 0, err + } + if ei < si || ei < 1024 || si < 1024 || ei > 65535 || si > 65535 { + return 0, 0, errors.New("invalid port") + } + return si, ei - si + 1, nil + } else { + ret, err := strconv.Atoi(str) + if err != nil { + return 0, 0, err + } else { + return ret, 1, nil + } + } + } + for i, mapper := range pasvPortMappers { + var err error + exposed, listened, mapped := strings.Cut(mapper, ":") + for { + if mapped { + var es, ls, el, ll int + es, el, err = convertToPorts(exposed) + if err != nil { + break + } + ls, ll, err = convertToPorts(listened) + if err != nil { + break + } + if el != ll { + err = errors.New("the number of exposed ports and listened ports does not match") + break + } + groups[i].ExposedStart = es + groups[i].ListenedStart = ls + groups[i].Length = el + totalLength += el + } else { + var start, length int + start, length, err = convertToPorts(mapper) + groups[i].ExposedStart = start + groups[i].ListenedStart = start + groups[i].Length = length + totalLength += length + } + break + } + if err != nil { + utils.Log.Fatalf("failed to convert FTP PASV port mapper %s: %v, the port mapper will be ignored.", mapper, err) + return nil + } + } + return func() (int, int, bool) { + idxPort := rand.Intn(totalLength) + for _, g := range groups { + if idxPort >= g.Length { + idxPort -= g.Length + } else { + return g.ExposedStart + idxPort, g.ListenedStart + idxPort, true + } + } + // unreachable + return 0, 0, false + } +} + +func getTlsConf(keyPath, certPath string) (*tls.Config, error) { + if keyPath == "" || certPath == "" { + return nil, errors.New("private key or certificate is not provided") + } + cert, err := os.ReadFile(certPath) + if err != nil { + return nil, err + } + key, err := os.ReadFile(keyPath) + if err != nil { + return nil, err + } + tlsCert, err := tls.X509KeyPair(cert, key) + if err != nil { + return nil, err + } + return &tls.Config{Certificates: []tls.Certificate{tlsCert}}, nil +} diff --git a/server/ftp/afero.go b/server/ftp/afero.go new file mode 100644 index 00000000000..75ae2e433e4 --- /dev/null +++ b/server/ftp/afero.go @@ -0,0 +1,121 @@ +package ftp + +import ( + "context" + "errors" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/spf13/afero" + "os" + "time" +) + +type AferoAdapter struct { + ctx context.Context + nextFileSize int64 +} + +func NewAferoAdapter(ctx context.Context) *AferoAdapter { + return &AferoAdapter{ctx: ctx} +} + +func (a *AferoAdapter) Create(_ string) (afero.File, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (a *AferoAdapter) Mkdir(name string, _ os.FileMode) error { + return Mkdir(a.ctx, name) +} + +func (a *AferoAdapter) MkdirAll(path string, perm os.FileMode) error { + return a.Mkdir(path, perm) +} + +func (a *AferoAdapter) Open(_ string) (afero.File, error) { + // See also GetHandle and ReadDir + return nil, errs.NotImplement +} + +func (a *AferoAdapter) OpenFile(_ string, _ int, _ os.FileMode) (afero.File, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (a *AferoAdapter) Remove(name string) error { + return Remove(a.ctx, name) +} + +func (a *AferoAdapter) RemoveAll(path string) error { + return a.Remove(path) +} + +func (a *AferoAdapter) Rename(oldName, newName string) error { + return Rename(a.ctx, oldName, newName) +} + +func (a *AferoAdapter) Stat(name string) (os.FileInfo, error) { + return Stat(a.ctx, name) +} + +func (a *AferoAdapter) Name() string { + return "AList FTP Endpoint" +} + +func (a *AferoAdapter) Chmod(_ string, _ os.FileMode) error { + return errs.NotSupport +} + +func (a *AferoAdapter) Chown(_ string, _, _ int) error { + return errs.NotSupport +} + +func (a *AferoAdapter) Chtimes(_ string, _ time.Time, _ time.Time) error { + return errs.NotSupport +} + +func (a *AferoAdapter) ReadDir(name string) ([]os.FileInfo, error) { + return List(a.ctx, name) +} + +func (a *AferoAdapter) GetHandle(name string, flags int, offset int64) (ftpserver.FileTransfer, error) { + fileSize := a.nextFileSize + a.nextFileSize = 0 + if (flags & os.O_SYNC) != 0 { + return nil, errs.NotSupport + } + if (flags & os.O_APPEND) != 0 { + return nil, errs.NotSupport + } + user := a.ctx.Value("user").(*model.User) + path, err := user.JoinPath(name) + if err != nil { + return nil, err + } + _, err = fs.Get(a.ctx, path, &fs.GetArgs{}) + exists := err == nil + if (flags&os.O_CREATE) == 0 && !exists { + return nil, errs.ObjectNotFound + } + if (flags&os.O_EXCL) != 0 && exists { + return nil, errors.New("file already exists") + } + if (flags & os.O_WRONLY) != 0 { + if offset != 0 { + return nil, errs.NotSupport + } + trunc := (flags & os.O_TRUNC) != 0 + if fileSize > 0 { + return OpenUploadWithLength(a.ctx, path, trunc, fileSize) + } else { + return OpenUpload(a.ctx, path, trunc) + } + } + return OpenDownload(a.ctx, path, offset) +} + +func (a *AferoAdapter) SetNextFileSize(size int64) { + a.nextFileSize = size +} diff --git a/server/ftp/fsmanage.go b/server/ftp/fsmanage.go new file mode 100644 index 00000000000..83e7bae1733 --- /dev/null +++ b/server/ftp/fsmanage.go @@ -0,0 +1,85 @@ +package ftp + +import ( + "context" + "fmt" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + stdpath "path" +) + +func Mkdir(ctx context.Context, path string) error { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return err + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) || !common.HasPermission(perm, common.PermFTPManage) { + meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + } + if !common.CanWrite(meta, reqPath) { + return errs.PermissionDenied + } + } + return fs.MakeDir(ctx, reqPath) +} + +func Remove(ctx context.Context, path string) error { + user := ctx.Value("user").(*model.User) + perm := common.MergeRolePermissions(user, path) + if !common.HasPermission(perm, common.PermRemove) || !common.HasPermission(perm, common.PermFTPManage) { + return errs.PermissionDenied + } + reqPath, err := user.JoinPath(path) + if err != nil { + return err + } + return fs.Remove(ctx, reqPath) +} + +func Rename(ctx context.Context, oldPath, newPath string) error { + user := ctx.Value("user").(*model.User) + srcPath, err := user.JoinPath(oldPath) + if err != nil { + return err + } + dstPath, err := user.JoinPath(newPath) + if err != nil { + return err + } + srcDir, srcBase := stdpath.Split(srcPath) + dstDir, dstBase := stdpath.Split(dstPath) + permSrc := common.MergeRolePermissions(user, srcPath) + if srcDir == dstDir { + if !common.HasPermission(permSrc, common.PermRename) || !common.HasPermission(permSrc, common.PermFTPManage) { + return errs.PermissionDenied + } + return fs.Rename(ctx, srcPath, dstBase) + } else { + if !common.HasPermission(permSrc, common.PermFTPManage) || !common.HasPermission(permSrc, common.PermMove) || (srcBase != dstBase && !common.HasPermission(permSrc, common.PermRename)) { + return errs.PermissionDenied + } + if err = fs.Move(ctx, srcPath, dstDir); err != nil { + if srcBase != dstBase { + return err + } + if _, err1 := fs.Copy(ctx, srcPath, dstDir); err1 != nil { + return fmt.Errorf("failed move for %+v, and failed try copying for %+v", err, err1) + } + return nil + } + if srcBase != dstBase { + return fs.Rename(ctx, stdpath.Join(dstDir, srcBase), dstBase) + } + return nil + } +} diff --git a/server/ftp/fsread.go b/server/ftp/fsread.go new file mode 100644 index 00000000000..2ba8cb82abc --- /dev/null +++ b/server/ftp/fsread.go @@ -0,0 +1,163 @@ +package ftp + +import ( + "context" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + fs2 "io/fs" + "net/http" + "os" + "time" +) + +type FileDownloadProxy struct { + ftpserver.FileTransfer + reader stream.SStreamReadAtSeeker +} + +func OpenDownload(ctx context.Context, reqPath string, offset int64) (*FileDownloadProxy, error) { + user := ctx.Value("user").(*model.User) + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + + // directly use proxy + header := *(ctx.Value("proxy_header").(*http.Header)) + link, obj, err := fs.Link(ctx, reqPath, model.LinkArgs{ + IP: ctx.Value("client_ip").(string), + Header: header, + }) + if err != nil { + return nil, err + } + fileStream := stream.FileStream{ + Obj: obj, + Ctx: ctx, + } + ss, err := stream.NewSeekableStream(fileStream, link) + if err != nil { + return nil, err + } + reader, err := stream.NewReadAtSeeker(ss, offset) + if err != nil { + _ = ss.Close() + return nil, err + } + return &FileDownloadProxy{reader: reader}, nil +} + +func (f *FileDownloadProxy) Read(p []byte) (n int, err error) { + n, err = f.reader.Read(p) + if err != nil { + return + } + err = stream.ClientDownloadLimit.WaitN(f.reader.GetRawStream().Ctx, n) + return +} + +func (f *FileDownloadProxy) Write(p []byte) (n int, err error) { + return 0, errs.NotSupport +} + +func (f *FileDownloadProxy) Seek(offset int64, whence int) (int64, error) { + return f.reader.Seek(offset, whence) +} + +func (f *FileDownloadProxy) Close() error { + return f.reader.Close() +} + +type OsFileInfoAdapter struct { + obj model.Obj +} + +func (o *OsFileInfoAdapter) Name() string { + return o.obj.GetName() +} + +func (o *OsFileInfoAdapter) Size() int64 { + return o.obj.GetSize() +} + +func (o *OsFileInfoAdapter) Mode() fs2.FileMode { + var mode fs2.FileMode = 0755 + if o.IsDir() { + mode |= fs2.ModeDir + } + return mode +} + +func (o *OsFileInfoAdapter) ModTime() time.Time { + return o.obj.ModTime() +} + +func (o *OsFileInfoAdapter) IsDir() bool { + return o.obj.IsDir() +} + +func (o *OsFileInfoAdapter) Sys() any { + return o.obj +} + +func Stat(ctx context.Context, path string) (os.FileInfo, error) { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + if err != nil { + return nil, err + } + return &OsFileInfoAdapter{obj: obj}, nil +} + +func List(ctx context.Context, path string) ([]os.FileInfo, error) { + user := ctx.Value("user").(*model.User) + reqPath, err := user.JoinPath(path) + if err != nil { + return nil, err + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return nil, err + } + } + ctx = context.WithValue(ctx, "meta", meta) + if !common.CanAccessWithRoles(user, meta, reqPath, ctx.Value("meta_pass").(string)) { + return nil, errs.PermissionDenied + } + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{}) + if err != nil { + return nil, err + } + ret := make([]os.FileInfo, len(objs)) + for i, obj := range objs { + ret[i] = &OsFileInfoAdapter{obj: obj} + } + return ret, nil +} diff --git a/server/ftp/fsup.go b/server/ftp/fsup.go new file mode 100644 index 00000000000..9610eea7588 --- /dev/null +++ b/server/ftp/fsup.go @@ -0,0 +1,220 @@ +package ftp + +import ( + "bytes" + "context" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/common" + "github.com/pkg/errors" + "io" + "net/http" + "os" + stdpath "path" + "time" +) + +type FileUploadProxy struct { + ftpserver.FileTransfer + buffer *os.File + path string + ctx context.Context + trunc bool +} + +func uploadAuth(ctx context.Context, path string) error { + user := ctx.Value("user").(*model.User) + meta, err := op.GetNearestMeta(stdpath.Dir(path)) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + return err + } + } + perm := common.MergeRolePermissions(user, path) + if !(common.CanAccessWithRoles(user, meta, path, ctx.Value("meta_pass").(string)) && + ((common.HasPermission(perm, common.PermFTPManage) && common.HasPermission(perm, common.PermWrite)) || + common.CanWrite(meta, stdpath.Dir(path)))) { + return errs.PermissionDenied + } + return nil +} + +func OpenUpload(ctx context.Context, path string, trunc bool) (*FileUploadProxy, error) { + err := uploadAuth(ctx, path) + if err != nil { + return nil, err + } + tmpFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return nil, err + } + return &FileUploadProxy{buffer: tmpFile, path: path, ctx: ctx, trunc: trunc}, nil +} + +func (f *FileUploadProxy) Read(p []byte) (n int, err error) { + return 0, errs.NotSupport +} + +func (f *FileUploadProxy) Write(p []byte) (n int, err error) { + n, err = f.buffer.Write(p) + if err != nil { + return + } + err = stream.ClientUploadLimit.WaitN(f.ctx, n) + return +} + +func (f *FileUploadProxy) Seek(offset int64, whence int) (int64, error) { + return f.buffer.Seek(offset, whence) +} + +func (f *FileUploadProxy) Close() error { + dir, name := stdpath.Split(f.path) + size, err := f.buffer.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { + return err + } + arr := make([]byte, 512) + if _, err := f.buffer.Read(arr); err != nil { + return err + } + contentType := http.DetectContentType(arr) + if _, err := f.buffer.Seek(0, io.SeekStart); err != nil { + return err + } + if f.trunc { + _ = fs.Remove(f.ctx, f.path) + } + s := &stream.FileStream{ + Obj: &model.Object{ + Name: name, + Size: size, + Modified: time.Now(), + }, + Mimetype: contentType, + WebPutAsTask: true, + } + s.SetTmpFile(f.buffer) + _, err = fs.PutAsTask(f.ctx, dir, s) + return err +} + +type FileUploadWithLengthProxy struct { + ftpserver.FileTransfer + ctx context.Context + path string + length int64 + first512Bytes [512]byte + pFirst int + pipeWriter io.WriteCloser + errChan chan error +} + +func OpenUploadWithLength(ctx context.Context, path string, trunc bool, length int64) (*FileUploadWithLengthProxy, error) { + err := uploadAuth(ctx, path) + if err != nil { + return nil, err + } + if trunc { + _ = fs.Remove(ctx, path) + } + return &FileUploadWithLengthProxy{ctx: ctx, path: path, length: length}, nil +} + +func (f *FileUploadWithLengthProxy) Read(p []byte) (n int, err error) { + return 0, errs.NotSupport +} + +func (f *FileUploadWithLengthProxy) write(p []byte) (n int, err error) { + if f.pipeWriter != nil { + select { + case e := <-f.errChan: + return 0, e + default: + return f.pipeWriter.Write(p) + } + } else if len(p) < 512-f.pFirst { + copy(f.first512Bytes[f.pFirst:], p) + f.pFirst += len(p) + return len(p), nil + } else { + copy(f.first512Bytes[f.pFirst:], p[:512-f.pFirst]) + contentType := http.DetectContentType(f.first512Bytes[:]) + dir, name := stdpath.Split(f.path) + reader, writer := io.Pipe() + f.errChan = make(chan error, 1) + s := &stream.FileStream{ + Obj: &model.Object{ + Name: name, + Size: f.length, + Modified: time.Now(), + }, + Mimetype: contentType, + WebPutAsTask: false, + Reader: reader, + } + go func() { + e := fs.PutDirectly(f.ctx, dir, s, true) + f.errChan <- e + close(f.errChan) + }() + f.pipeWriter = writer + n, err = writer.Write(f.first512Bytes[:]) + if err != nil { + return n, err + } + n1, err := writer.Write(p[512-f.pFirst:]) + if err != nil { + return n1 + 512 - f.pFirst, err + } + f.pFirst = 512 + return len(p), nil + } +} + +func (f *FileUploadWithLengthProxy) Write(p []byte) (n int, err error) { + n, err = f.write(p) + if err != nil { + return + } + err = stream.ClientUploadLimit.WaitN(f.ctx, n) + return +} + +func (f *FileUploadWithLengthProxy) Seek(offset int64, whence int) (int64, error) { + return 0, errs.NotSupport +} + +func (f *FileUploadWithLengthProxy) Close() error { + if f.pipeWriter != nil { + err := f.pipeWriter.Close() + if err != nil { + return err + } + err = <-f.errChan + return err + } else { + data := f.first512Bytes[:f.pFirst] + contentType := http.DetectContentType(data) + dir, name := stdpath.Split(f.path) + s := &stream.FileStream{ + Obj: &model.Object{ + Name: name, + Size: int64(f.pFirst), + Modified: time.Now(), + }, + Mimetype: contentType, + WebPutAsTask: false, + Reader: bytes.NewReader(data), + } + return fs.PutDirectly(f.ctx, dir, s, true) + } +} diff --git a/server/ftp/site.go b/server/ftp/site.go new file mode 100644 index 00000000000..8ea667d8e8d --- /dev/null +++ b/server/ftp/site.go @@ -0,0 +1,21 @@ +package ftp + +import ( + "fmt" + ftpserver "github.com/KirCute/ftpserverlib-pasvportmap" + "strconv" +) + +func HandleSIZE(param string, client ftpserver.ClientDriver) (int, string) { + fs, ok := client.(*AferoAdapter) + if !ok { + return ftpserver.StatusNotLoggedIn, "Unexpected exception (driver is nil)" + } + size, err := strconv.ParseInt(param, 10, 64) + if err != nil { + return ftpserver.StatusSyntaxErrorParameters, fmt.Sprintf( + "Couldn't parse file size, given: %s, err: %v", param, err) + } + fs.SetNextFileSize(size) + return ftpserver.StatusOK, "Accepted next file size" +} diff --git a/server/handles/archive.go b/server/handles/archive.go new file mode 100644 index 00000000000..844947408be --- /dev/null +++ b/server/handles/archive.go @@ -0,0 +1,435 @@ +package handles + +import ( + "encoding/json" + "fmt" + "github.com/alist-org/alist/v3/internal/task" + "net/url" + stdpath "path" + + "github.com/alist-org/alist/v3/internal/archive/tool" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type ArchiveMetaReq struct { + Path string `json:"path" form:"path"` + Password string `json:"password" form:"password"` + Refresh bool `json:"refresh" form:"refresh"` + ArchivePass string `json:"archive_pass" form:"archive_pass"` +} + +type ArchiveMetaResp struct { + Comment string `json:"comment"` + IsEncrypted bool `json:"encrypted"` + Content []ArchiveContentResp `json:"content"` + Sort *model.Sort `json:"sort,omitempty"` + RawURL string `json:"raw_url"` + Sign string `json:"sign"` +} + +type ArchiveContentResp struct { + ObjResp + Children []ArchiveContentResp `json:"children"` +} + +func toObjsRespWithoutSignAndThumb(obj model.Obj) ObjResp { + storageClass, _ := model.GetStorageClass(obj) + return ObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: "", + Thumb: "", + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + StorageClass: storageClass, + } +} + +func toContentResp(objs []model.ObjTree) []ArchiveContentResp { + if objs == nil { + return nil + } + ret, _ := utils.SliceConvert(objs, func(src model.ObjTree) (ArchiveContentResp, error) { + return ArchiveContentResp{ + ObjResp: toObjsRespWithoutSignAndThumb(src), + Children: toContentResp(src.GetChildren()), + }, nil + }) + return ret +} + +func FsArchiveMeta(c *gin.Context) { + var req ArchiveMetaReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermReadArchives) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("meta", meta) + if !common.CanAccess(user, meta, reqPath, req.Password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + archiveArgs := model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: req.ArchivePass, + } + ret, err := fs.ArchiveMeta(c, reqPath, model.ArchiveMetaArgs{ + ArchiveArgs: archiveArgs, + Refresh: req.Refresh, + }) + if err != nil { + if errors.Is(err, errs.WrongArchivePassword) { + common.ErrorResp(c, err, 202) + } else { + common.ErrorResp(c, err, 500) + } + return + } + s := "" + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + s = sign.SignArchive(reqPath) + } + api := "/ae" + if ret.DriverProviding { + api = "/ad" + } + common.SuccessResp(c, ArchiveMetaResp{ + Comment: ret.GetComment(), + IsEncrypted: ret.IsEncrypted(), + Content: toContentResp(ret.GetTree()), + Sort: ret.Sort, + RawURL: fmt.Sprintf("%s%s%s", common.GetApiUrl(c.Request), api, utils.EncodePath(reqPath, true)), + Sign: s, + }) +} + +type ArchiveListReq struct { + ArchiveMetaReq + model.PageReq + InnerPath string `json:"inner_path" form:"inner_path"` +} + +type ArchiveListResp struct { + Content []ObjResp `json:"content"` + Total int64 `json:"total"` +} + +func FsArchiveList(c *gin.Context) { + var req ArchiveListReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermReadArchives) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("meta", meta) + if !common.CanAccess(user, meta, reqPath, req.Password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + objs, err := fs.ArchiveList(c, reqPath, model.ArchiveListArgs{ + ArchiveInnerArgs: model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: req.ArchivePass, + }, + InnerPath: utils.FixAndCleanPath(req.InnerPath), + }, + Refresh: req.Refresh, + }) + if err != nil { + if errors.Is(err, errs.WrongArchivePassword) { + common.ErrorResp(c, err, 202) + } else { + common.ErrorResp(c, err, 500) + } + return + } + total, objs := pagination(objs, &req.PageReq) + ret, _ := utils.SliceConvert(objs, func(src model.Obj) (ObjResp, error) { + return toObjsRespWithoutSignAndThumb(src), nil + }) + common.SuccessResp(c, ArchiveListResp{ + Content: ret, + Total: int64(total), + }) +} + +type StringOrArray []string + +func (s *StringOrArray) UnmarshalJSON(data []byte) error { + var value string + if err := json.Unmarshal(data, &value); err == nil { + *s = []string{value} + return nil + } + var sliceValue []string + if err := json.Unmarshal(data, &sliceValue); err != nil { + return err + } + *s = sliceValue + return nil +} + +type ArchiveDecompressReq struct { + SrcDir string `json:"src_dir" form:"src_dir"` + DstDir string `json:"dst_dir" form:"dst_dir"` + Name StringOrArray `json:"name" form:"name"` + ArchivePass string `json:"archive_pass" form:"archive_pass"` + InnerPath string `json:"inner_path" form:"inner_path"` + CacheFull bool `json:"cache_full" form:"cache_full"` + PutIntoNewDir bool `json:"put_into_new_dir" form:"put_into_new_dir"` +} + +func FsArchiveDecompress(c *gin.Context) { + var req ArchiveDecompressReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + srcDir, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + srcPaths := make([]string, 0, len(req.Name)) + for _, name := range req.Name { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if !common.CheckPathLimitWithRoles(user, srcPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + srcPaths = append(srcPaths, srcPath) + } + dstDir, err := user.JoinPath(req.DstDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + tasks := make([]task.TaskExtensionInfo, 0, len(srcPaths)) + for _, srcPath := range srcPaths { + perm := common.MergeRolePermissions(user, srcPath) + if !common.HasPermission(perm, common.PermDecompress) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + t, e := fs.ArchiveDecompress(c, srcPath, dstDir, model.ArchiveDecompressArgs{ + ArchiveInnerArgs: model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: req.ArchivePass, + }, + InnerPath: utils.FixAndCleanPath(req.InnerPath), + }, + CacheFull: req.CacheFull, + PutIntoNewDir: req.PutIntoNewDir, + }) + if e != nil { + if errors.Is(e, errs.WrongArchivePassword) { + common.ErrorResp(c, e, 202) + } else { + common.ErrorResp(c, e, 500) + } + return + } + if t != nil { + tasks = append(tasks, t) + } + } + common.SuccessResp(c, gin.H{ + "task": getTaskInfos(tasks), + }) +} + +func ArchiveDown(c *gin.Context) { + archiveRawPath := c.MustGet("path").(string) + innerPath := utils.FixAndCleanPath(c.Query("inner")) + password := c.Query("pass") + filename := stdpath.Base(innerPath) + storage, err := fs.GetStorage(archiveRawPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if common.ShouldProxy(storage, filename) { + ArchiveProxy(c) + return + } else { + link, _, err := fs.ArchiveDriverExtract(c, archiveRawPath, model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + IP: c.ClientIP(), + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + Redirect: true, + }, + Password: password, + }, + InnerPath: innerPath, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + down(c, link) + } +} + +func ArchiveProxy(c *gin.Context) { + archiveRawPath := c.MustGet("path").(string) + innerPath := utils.FixAndCleanPath(c.Query("inner")) + password := c.Query("pass") + filename := stdpath.Base(innerPath) + storage, err := fs.GetStorage(archiveRawPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if canProxy(storage, filename) { + // TODO: Support external download proxy URL + link, file, err := fs.ArchiveDriverExtract(c, archiveRawPath, model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: password, + }, + InnerPath: innerPath, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + localProxy(c, link, file, storage.GetStorage().ProxyRange) + } else { + common.ErrorStrResp(c, "proxy not allowed", 403) + return + } +} + +func ArchiveInternalExtract(c *gin.Context) { + archiveRawPath := c.MustGet("path").(string) + innerPath := utils.FixAndCleanPath(c.Query("inner")) + password := c.Query("pass") + rc, size, err := fs.ArchiveInternalExtract(c, archiveRawPath, model.ArchiveInnerArgs{ + ArchiveArgs: model.ArchiveArgs{ + LinkArgs: model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }, + Password: password, + }, + InnerPath: innerPath, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + defer func() { + if err := rc.Close(); err != nil { + log.Errorf("failed to close file streamer, %v", err) + } + }() + headers := map[string]string{ + "Referrer-Policy": "no-referrer", + "Cache-Control": "max-age=0, no-cache, no-store, must-revalidate", + } + filename := stdpath.Base(innerPath) + headers["Content-Disposition"] = fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, filename, url.PathEscape(filename)) + contentType := c.Request.Header.Get("Content-Type") + if contentType == "" { + contentType = utils.GetMimeType(filename) + } + c.DataFromReader(200, size, contentType, rc, headers) +} + +func ArchiveExtensions(c *gin.Context) { + var ext []string + for key := range tool.Tools { + ext = append(ext, key) + } + common.SuccessResp(c, ext) +} diff --git a/server/handles/aria2.go b/server/handles/aria2.go deleted file mode 100644 index 325367a796a..00000000000 --- a/server/handles/aria2.go +++ /dev/null @@ -1,80 +0,0 @@ -package handles - -import ( - "github.com/alist-org/alist/v3/internal/aria2" - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/server/common" - "github.com/gin-gonic/gin" -) - -type SetAria2Req struct { - Uri string `json:"uri" form:"uri"` - Secret string `json:"secret" form:"secret"` -} - -func SetAria2(c *gin.Context) { - var req SetAria2Req - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - items := []model.SettingItem{ - {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.ARIA2, Flag: model.PRIVATE}, - } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - version, err := aria2.InitClient(2) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - common.SuccessResp(c, version) -} - -type AddAria2Req struct { - Urls []string `json:"urls"` - Path string `json:"path"` -} - -func AddAria2(c *gin.Context) { - user := c.MustGet("user").(*model.User) - if !user.CanAddAria2Tasks() { - common.ErrorStrResp(c, "permission denied", 403) - return - } - if !aria2.IsAria2Ready() { - // try to init client - _, err := aria2.InitClient(2) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if !aria2.IsAria2Ready() { - common.ErrorStrResp(c, "aria2 still not ready after init", 500) - return - } - } - var req AddAria2Req - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - reqPath, err := user.JoinPath(req.Path) - if err != nil { - common.ErrorResp(c, err, 403) - return - } - for _, url := range req.Urls { - err := aria2.AddURI(c, url, reqPath) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - } - common.SuccessResp(c) -} diff --git a/server/handles/auth.go b/server/handles/auth.go index 37ae736c2ef..f59b4fb1ed7 100644 --- a/server/handles/auth.go +++ b/server/handles/auth.go @@ -3,12 +3,22 @@ package handles import ( "bytes" "encoding/base64" + "errors" + "fmt" "image/png" + "path" + "strings" "time" "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/session" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pquerna/otp/totp" @@ -16,8 +26,9 @@ import ( var loginCache = cache.NewMemCache[int]() var ( - defaultDuration = time.Minute * 5 - defaultTimes = 5 + defaultDuration = time.Minute * 5 + defaultTimes = 5 + invalidLoginCredentialsMsg = "username or password is incorrect" ) type LoginReq struct { @@ -59,13 +70,13 @@ func loginHash(c *gin.Context, req *LoginReq) { // check username user, err := op.GetUserByName(req.Username) if err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, invalidLoginCredentialsMsg, 400) loginCache.Set(ip, count+1) return } // validate password hash if err := user.ValidatePwdStaticHash(req.Password); err != nil { - common.ErrorResp(c, err, 400) + common.ErrorStrResp(c, invalidLoginCredentialsMsg, 400) loginCache.Set(ip, count+1) return } @@ -77,25 +88,74 @@ func loginHash(c *gin.Context, req *LoginReq) { return } } + + clientID := c.GetHeader("Client-Id") + if clientID == "" { + clientID = c.Query("client_id") + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", + user.ID, clientID)) + + if err := device.EnsureActiveOnLogin(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + if errors.Is(err, errs.TooManyDevices) { + common.ErrorResp(c, err, 403) + } else { + common.ErrorResp(c, err, 400, true) + } + return + } + // generate token - token, err := common.GenerateToken(user.Username) + token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return } - common.SuccessResp(c, gin.H{"token": token}) + common.SuccessResp(c, gin.H{"token": token, "device_key": key}) loginCache.Del(ip) } +type RegisterReq struct { + Username string `json:"username" binding:"required"` + Password string `json:"password" binding:"required"` +} + +// Register a new user +func Register(c *gin.Context) { + if !setting.GetBool(conf.AllowRegister) { + common.ErrorStrResp(c, "registration is disabled", 403) + return + } + var req RegisterReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := &model.User{ + Username: req.Username, + Role: model.Roles{op.GetDefaultRoleID()}, + Authn: "[]", + } + user.SetPassword(req.Password) + if err := op.CreateUser(user); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + type UserResp struct { model.User - Otp bool `json:"otp"` + Otp bool `json:"otp"` + RoleNames []string `json:"role_names"` + Permissions []model.PermissionEntry `json:"permissions"` } // CurrentUser get current user by token // if token is empty, return guest user func CurrentUser(c *gin.Context) { user := c.MustGet("user").(*model.User) + userResp := UserResp{ User: *user, } @@ -103,6 +163,30 @@ func CurrentUser(c *gin.Context) { if userResp.OtpSecret != "" { userResp.Otp = true } + + var roleNames []string + permMap := map[string]int32{} + paths := make([]string, 0) + + for _, role := range user.RolesDetail { + roleNames = append(roleNames, role.Name) + for _, entry := range role.PermissionScopes { + cleanPath := path.Clean("/" + strings.TrimPrefix(entry.Path, "/")) + if _, ok := permMap[cleanPath]; !ok { + paths = append(paths, cleanPath) + } + permMap[cleanPath] |= entry.Permission + } + } + userResp.RoleNames = roleNames + + for _, fullPath := range paths { + userResp.Permissions = append(userResp.Permissions, model.PermissionEntry{ + Path: fullPath, + Permission: permMap[fullPath], + }) + } + common.SuccessResp(c, userResp) } @@ -113,6 +197,10 @@ func UpdateCurrent(c *gin.Context) { return } user := c.MustGet("user").(*model.User) + if user.IsGuest() { + common.ErrorStrResp(c, "Guest user can not update profile", 403) + return + } user.Username = req.Username if req.Password != "" { user.SetPassword(req.Password) @@ -181,3 +269,19 @@ func Verify2FA(c *gin.Context) { common.SuccessResp(c) } } + +func LogOut(c *gin.Context) { + if keyVal, ok := c.Get("device_key"); ok { + if err := session.MarkInactive(keyVal.(string)); err != nil { + common.ErrorResp(c, err, 500) + return + } + c.Set("session_inactive", true) + } + err := common.InvalidateToken(c.GetHeader("Authorization")) + if err != nil { + common.ErrorResp(c, err, 500) + } else { + common.SuccessResp(c) + } +} diff --git a/server/handles/const.go b/server/handles/const.go new file mode 100644 index 00000000000..b108c9da9bc --- /dev/null +++ b/server/handles/const.go @@ -0,0 +1,7 @@ +package handles + +const ( + CANCEL = "cancel" + OVERWRITE = "overwrite" + SKIP = "skip" +) diff --git a/server/handles/down.go b/server/handles/down.go index e4aec494243..680c33f54b1 100644 --- a/server/handles/down.go +++ b/server/handles/down.go @@ -1,21 +1,24 @@ package handles import ( + "bytes" "fmt" "io" stdpath "path" - "strings" + "strconv" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/driver" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/setting" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "github.com/microcosm-cc/bluemonday" log "github.com/sirupsen/logrus" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/extension" ) func Down(c *gin.Context) { @@ -31,35 +34,17 @@ func Down(c *gin.Context) { return } else { link, _, err := fs.Link(c, rawPath, model.LinkArgs{ - IP: c.ClientIP(), - Header: c.Request.Header, - Type: c.Query("type"), - HttpReq: c.Request, + IP: c.ClientIP(), + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + Redirect: true, }) if err != nil { common.ErrorResp(c, err, 500) return } - if link.MFile != nil { - defer func(ReadSeekCloser io.ReadCloser) { - err := ReadSeekCloser.Close() - if err != nil { - log.Errorf("close data error: %s", err) - } - }(link.MFile) - } - c.Header("Referrer-Policy", "no-referrer") - c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") - if setting.GetBool(conf.ForwardDirectLinkParams) { - query := c.Request.URL.Query() - query.Del("sign") - link.URL, err = utils.InjectQuery(link.URL, query) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - } - c.Redirect(302, link.URL) + down(c, link) } } @@ -71,15 +56,26 @@ func Proxy(c *gin.Context) { common.ErrorResp(c, err, 500) return } + if c.Query("type") == "preview" && storage.GetStorage().Driver == "DoubaoNew" { + // Force proxy for DoubaoNew preview so headers are preserved. + link, file, err := fs.Link(c, rawPath, model.LinkArgs{ + Header: c.Request.Header, + Type: c.Query("type"), + HttpReq: c.Request, + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + localProxy(c, link, file, storage.GetStorage().ProxyRange) + return + } if canProxy(storage, filename) { downProxyUrl := storage.GetStorage().DownProxyUrl if downProxyUrl != "" { _, ok := c.GetQuery("d") if !ok { - URL := fmt.Sprintf("%s%s?sign=%s", - strings.Split(downProxyUrl, "\n")[0], - utils.EncodePath(rawPath, true), - sign.Sign(rawPath)) + URL := common.BuildDownProxyURL(downProxyUrl, rawPath, storage.GetStorage().DownProxySign) c.Redirect(302, URL) return } @@ -93,24 +89,93 @@ func Proxy(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if link.URL != "" && setting.GetBool(conf.ForwardDirectLinkParams) { - query := c.Request.URL.Query() - query.Del("sign") - link.URL, err = utils.InjectQuery(link.URL, query) + localProxy(c, link, file, storage.GetStorage().ProxyRange) + } else { + common.ErrorStrResp(c, "proxy not allowed", 403) + return + } +} + +func down(c *gin.Context, link *model.Link) { + var err error + if link.MFile != nil { + defer func(ReadSeekCloser io.ReadCloser) { + err := ReadSeekCloser.Close() if err != nil { - common.ErrorResp(c, err, 500) - return + log.Errorf("close data error: %s", err) } + }(link.MFile) + } + c.Header("Referrer-Policy", "no-referrer") + c.Header("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") + if setting.GetBool(conf.ForwardDirectLinkParams) { + query := c.Request.URL.Query() + for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { + query.Del(v) } - err = common.Proxy(c.Writer, c.Request, link, file) + link.URL, err = utils.InjectQuery(link.URL, query) if err != nil { - common.ErrorResp(c, err, 500, true) + common.ErrorResp(c, err, 500) return } + } + c.Redirect(302, link.URL) +} + +func localProxy(c *gin.Context, link *model.Link, file model.Obj, proxyRange bool) { + var err error + if link.URL != "" && setting.GetBool(conf.ForwardDirectLinkParams) { + query := c.Request.URL.Query() + for _, v := range conf.SlicesMap[conf.IgnoreDirectLinkParams] { + query.Del(v) + } + link.URL, err = utils.InjectQuery(link.URL, query) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + } + if proxyRange { + common.ProxyRange(link, file.GetSize()) + } + Writer := &common.WrittenResponseWriter{ResponseWriter: c.Writer} + + //优先处理md文件 + if utils.Ext(file.GetName()) == "md" && setting.GetBool(conf.FilterReadMeScripts) { + buf := bytes.NewBuffer(make([]byte, 0, file.GetSize())) + w := &common.InterceptResponseWriter{ResponseWriter: Writer, Writer: buf} + err = common.Proxy(w, c.Request, link, file) + if err == nil && buf.Len() > 0 { + if c.Writer.Status() < 200 || c.Writer.Status() > 300 { + c.Writer.Write(buf.Bytes()) + return + } + + var html bytes.Buffer + md := goldmark.New(goldmark.WithExtensions(extension.GFM)) + if err = md.Convert(buf.Bytes(), &html); err != nil { + err = fmt.Errorf("markdown conversion failed: %w", err) + } else { + buf.Reset() + err = bluemonday.UGCPolicy().SanitizeReaderToWriter(&html, buf) + if err == nil { + Writer.Header().Set("Content-Length", strconv.FormatInt(int64(buf.Len()), 10)) + Writer.Header().Set("Content-Type", "text/html; charset=utf-8") + _, err = utils.CopyWithBuffer(Writer, buf) + } + } + } } else { - common.ErrorStrResp(c, "proxy not allowed", 403) + err = common.Proxy(Writer, c.Request, link, file) + } + if err == nil { return } + if Writer.IsWritten() { + log.Errorf("%s %s local proxy error: %+v", c.Request.Method, c.Request.URL.Path, err) + } else { + common.ErrorResp(c, err, 500, true) + } } // TODO need optimize @@ -124,6 +189,9 @@ func canProxy(storage driver.Driver, filename string) bool { if storage.Config().MustProxy() || storage.GetStorage().WebProxy || storage.GetStorage().WebdavProxy() { return true } + if storage.GetStorage().Driver == "Quark" && utils.GetFileType(filename) == conf.VIDEO { + return true + } if utils.SliceContains(conf.SlicesMap[conf.ProxyTypes], utils.Ext(filename)) { return true } diff --git a/server/handles/fsbatch.go b/server/handles/fsbatch.go index fa7971dfbe1..bccbee72d29 100644 --- a/server/handles/fsbatch.go +++ b/server/handles/fsbatch.go @@ -3,67 +3,23 @@ package handles import ( "fmt" "regexp" + "slices" "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/pkg/generic" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) -type BatchRenameReq struct { - SrcDir string `json:"src_dir"` - RenameObjects []struct { - SrcName string `json:"src_name"` - NewName string `json:"new_name"` - } `json:"rename_objects"` -} - -func FsBatchRename(c *gin.Context) { - var req BatchRenameReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } - - reqPath, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) - return - } - - meta, err := op.GetNearestMeta(reqPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } - } - c.Set("meta", meta) - for _, renameObject := range req.RenameObjects { - if renameObject.SrcName == "" || renameObject.NewName == "" { - continue - } - filePath := fmt.Sprintf("%s/%s", reqPath, renameObject.SrcName) - if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { - common.ErrorResp(c, err, 500) - return - } - } - common.SuccessResp(c) -} - type RecursiveMoveReq struct { - SrcDir string `json:"src_dir"` - DstDir string `json:"dst_dir"` + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + ConflictPolicy string `json:"conflict_policy"` } func FsRecursiveMove(c *gin.Context) { @@ -74,20 +30,29 @@ func FsRecursiveMove(c *gin.Context) { } user := c.MustGet("user").(*model.User) - if !user.CanMove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermMove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(srcDir) if err != nil { @@ -104,9 +69,23 @@ func FsRecursiveMove(c *gin.Context) { return } + var existingFileNames []string + if req.ConflictPolicy != OVERWRITE { + dstFiles, err := fs.List(c, dstDir, &fs.ListArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + existingFileNames = make([]string, 0, len(dstFiles)) + for _, dstFile := range dstFiles { + existingFileNames = append(existingFileNames, dstFile.GetName()) + } + } + // record the file path filePathMap := make(map[model.Obj]string) movingFiles := generic.NewQueue[model.Obj]() + movingFileNames := make([]string, 0, len(rootFiles)) for _, file := range rootFiles { movingFiles.Push(file) filePathMap[file] = srcDir @@ -130,22 +109,100 @@ func FsRecursiveMove(c *gin.Context) { filePathMap[subFile] = subFilePath } } else { - if movingFilePath == dstDir { // same directory, don't move continue } - // move - err := fs.Move(c, movingFileName, dstDir, movingFiles.IsEmpty()) - if err != nil { - common.ErrorResp(c, err, 500) - return + if slices.Contains(existingFileNames, movingFile.GetName()) { + if req.ConflictPolicy == CANCEL { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", movingFile.GetName()), 403) + return + } else if req.ConflictPolicy == SKIP { + continue + } + } else if req.ConflictPolicy != OVERWRITE { + existingFileNames = append(existingFileNames, movingFile.GetName()) } + movingFileNames = append(movingFileNames, movingFileName) + + } + + } + + var count = 0 + for i, fileName := range movingFileNames { + // move + err := fs.Move(c, fileName, dstDir, len(movingFileNames) > i+1) + if err != nil { + common.ErrorResp(c, err, 500) + return } + count++ + } + + common.SuccessWithMsgResp(c, fmt.Sprintf("Successfully moved %d %s", count, common.Pluralize(count, "file", "files"))) +} + +type BatchRenameReq struct { + SrcDir string `json:"src_dir"` + RenameObjects []struct { + SrcName string `json:"src_name"` + NewName string `json:"new_name"` + } `json:"rename_objects"` +} +func FsBatchRename(c *gin.Context) { + var req BatchRenameReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("meta", meta) + for _, renameObject := range req.RenameObjects { + if renameObject.SrcName == "" || renameObject.NewName == "" { + continue + } + if err := utils.ValidateNameComponent(renameObject.NewName); err != nil { + common.ErrorResp(c, err, 400) + return + } + filePath, err := utils.JoinUnderBase(reqPath, renameObject.SrcName) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if !canRenamePath(c, filePath) { + return + } + if err := fs.Rename(c, filePath, renameObject.NewName); err != nil { + common.ErrorResp(c, err, 500) + return + } + } common.SuccessResp(c) } @@ -162,14 +219,19 @@ func FsRegexRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { + reqPath, err := user.JoinPath(req.SrcDir) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, reqPath) { common.ErrorResp(c, errs.PermissionDenied, 403) return } - reqPath, err := user.JoinPath(req.SrcDir) - if err != nil { - common.ErrorResp(c, err, 403) + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) return } @@ -197,8 +259,19 @@ func FsRegexRename(c *gin.Context) { for _, file := range files { if srcRegexp.MatchString(file.GetName()) { - filePath := fmt.Sprintf("%s/%s", reqPath, file.GetName()) + filePath, err := utils.JoinUnderBase(reqPath, file.GetName()) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if !canRenamePath(c, filePath) { + return + } newFileName := srcRegexp.ReplaceAllString(file.GetName(), req.NewNameRegex) + if err := utils.ValidateNameComponent(newFileName); err != nil { + common.ErrorResp(c, err, 400) + return + } if err := fs.Rename(c, filePath, newFileName); err != nil { common.ErrorResp(c, err, 500) return diff --git a/server/handles/fsmanage.go b/server/handles/fsmanage.go index 2733509e9a5..31976edd9ad 100644 --- a/server/handles/fsmanage.go +++ b/server/handles/fsmanage.go @@ -5,6 +5,8 @@ import ( "io" stdpath "path" + "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" @@ -34,7 +36,12 @@ func FsMkdir(c *gin.Context) { common.ErrorResp(c, err, 403) return } - if !user.CanWrite() { + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) { meta, err := op.GetNearestMeta(stdpath.Dir(reqPath)) if err != nil { if !errors.Is(errors.Cause(err), errs.MetaNotFound) { @@ -55,9 +62,10 @@ func FsMkdir(c *gin.Context) { } type MoveCopyReq struct { - SrcDir string `json:"src_dir"` - DstDir string `json:"dst_dir"` - Names []string `json:"names"` + SrcDir string `json:"src_dir"` + DstDir string `json:"dst_dir"` + Names []string `json:"names"` + Overwrite bool `json:"overwrite"` } func FsMove(c *gin.Context) { @@ -71,22 +79,54 @@ func FsMove(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanMove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + permMove := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(permMove, common.PermMove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if !req.Overwrite { + for _, name := range req.Names { + dstPath, err := utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + return + } + } + } for i, name := range req.Names { - err := fs.Move(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + err = fs.Move(c, srcPath, dstDir, len(req.Names) > i+1) if err != nil { common.ErrorResp(c, err, 500) return @@ -106,41 +146,88 @@ func FsCopy(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanCopy() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } dstDir, err := user.JoinPath(req.DstDir) if err != nil { common.ErrorResp(c, err, 403) return } - var addedTask []string + if !common.CheckPathLimitWithRoles(user, dstDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermCopy) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if !req.Overwrite { + for _, name := range req.Names { + dstPath, err := utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", name), 403) + return + } + } + } + var addedTasks []task.TaskExtensionInfo for i, name := range req.Names { - ok, err := fs.Copy(c, stdpath.Join(srcDir, name), dstDir, len(req.Names) > i+1) - if ok { - addedTask = append(addedTask, name) + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = utils.JoinUnderBase(dstDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + t, err := fs.Copy(c, srcPath, dstDir, len(req.Names) > i+1) + if t != nil { + addedTasks = append(addedTasks, t) } if err != nil { common.ErrorResp(c, err, 500) return } } - if len(addedTask) > 0 { - common.SuccessResp(c, fmt.Sprintf("Added %d tasks", len(addedTask))) - } else { - common.SuccessResp(c) - } + common.SuccessResp(c, gin.H{ + "tasks": getTaskInfos(addedTasks), + }) } type RenameReq struct { - Path string `json:"path"` - Name string `json:"name"` + Path string `json:"path"` + Name string `json:"name"` + Overwrite bool `json:"overwrite"` +} + +func canRenamePath(c *gin.Context, reqPath string) bool { + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return false + } + return true + } + if meta != nil && meta.Password != "" && common.IsApply(meta.Path, reqPath, meta.PSub) { + common.ErrorStrResp(c, "Path is password-protected and cannot be renamed.", 403) + return false + } + return true } func FsRename(c *gin.Context) { @@ -150,15 +237,40 @@ func FsRename(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRename() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqPath, err := user.JoinPath(req.Path) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if !canRenamePath(c, reqPath) { + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermRename) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + if err := utils.ValidateNameComponent(req.Name); err != nil { + common.ErrorResp(c, err, 400) + return + } + if !req.Overwrite { + dstPath, err := utils.JoinUnderBase(stdpath.Dir(reqPath), req.Name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if dstPath != reqPath { + if res, _ := fs.Get(c, dstPath, &fs.GetArgs{NoLog: true}); res != nil { + common.ErrorStrResp(c, fmt.Sprintf("file [%s] exists", req.Name), 403) + return + } + } + } if err := fs.Rename(c, reqPath, req.Name); err != nil { common.ErrorResp(c, err, 500) return @@ -182,17 +294,27 @@ func FsRemove(c *gin.Context) { return } user := c.MustGet("user").(*model.User) - if !user.CanRemove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } reqDir, err := user.JoinPath(req.Dir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, reqDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqDir) + if !common.HasPermission(perm, common.PermRemove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } for _, name := range req.Names { - err := fs.Remove(c, stdpath.Join(reqDir, name)) + removePath, err := utils.JoinUnderBase(reqDir, name) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + err = fs.Remove(c, removePath) if err != nil { common.ErrorResp(c, err, 500) return @@ -214,15 +336,20 @@ func FsRemoveEmptyDirectory(c *gin.Context) { } user := c.MustGet("user").(*model.User) - if !user.CanRemove() { - common.ErrorResp(c, errs.PermissionDenied, 403) - return - } srcDir, err := user.JoinPath(req.SrcDir) if err != nil { common.ErrorResp(c, err, 403) return } + if !common.CheckPathLimitWithRoles(user, srcDir) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, srcDir) + if !common.HasPermission(perm, common.PermRemove) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } meta, err := op.GetNearestMeta(srcDir) if err != nil { diff --git a/server/handles/fsread.go b/server/handles/fsread.go index 7c580f635e4..085694e3e32 100644 --- a/server/handles/fsread.go +++ b/server/handles/fsread.go @@ -33,34 +33,69 @@ type DirReq struct { } type ObjResp struct { - Name string `json:"name"` - Size int64 `json:"size"` - IsDir bool `json:"is_dir"` - Modified time.Time `json:"modified"` - Created time.Time `json:"created"` - Sign string `json:"sign"` - Thumb string `json:"thumb"` - Type int `json:"type"` - HashInfoStr string `json:"hashinfo"` - HashInfo map[*utils.HashType]string `json:"hash_info"` + Id string `json:"id"` + Path string `json:"path"` + VirtualPath string `json:"virtual_path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + StorageClass string `json:"storage_class,omitempty"` } type FsListResp struct { - Content []ObjResp `json:"content"` - Total int64 `json:"total"` - Readme string `json:"readme"` - Header string `json:"header"` - Write bool `json:"write"` - Provider string `json:"provider"` + Content []ObjLabelResp `json:"content"` + Total int64 `json:"total"` + FilteredTotal int64 `json:"filtered_total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + HasMore bool `json:"has_more"` + PagesTotal int `json:"pages_total"` + Readme string `json:"readme"` + Header string `json:"header"` + Write bool `json:"write"` + Provider string `json:"provider"` } +type ObjLabelResp struct { + Id string `json:"id"` + Path string `json:"path"` + VirtualPath string `json:"virtual_path"` + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Sign string `json:"sign"` + Thumb string `json:"thumb"` + Type int `json:"type"` + HashInfoStr string `json:"hashinfo"` + HashInfo map[*utils.HashType]string `json:"hash_info"` + LabelList []model.Label `json:"label_list"` + StorageClass string `json:"storage_class,omitempty"` +} + +const ( + DefaultPerPage = 200 + MaxPerPage = 500 + AllPerPage = -1 +) + func FsList(c *gin.Context) { var req ListReq if err := c.ShouldBind(&req); err != nil { common.ErrorResp(c, err, 400) return } - req.Validate() + effPage, effPerPage := normalizeListPage(req.Page, req.PerPage) + req.Page = effPage + req.PerPage = effPerPage user := c.MustGet("user").(*model.User) reqPath, err := user.JoinPath(req.Path) if err != nil { @@ -75,32 +110,49 @@ func FsList(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } - if !user.CanWrite() && !common.CanWrite(meta, reqPath) && req.Refresh { + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermWrite) && !common.CanWrite(meta, reqPath) && req.Refresh { common.ErrorStrResp(c, "Refresh without permission", 403) return } + provider := "unknown" + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if storageErr == nil { + provider = storage.GetStorage().Driver + } objs, err := fs.List(c, reqPath, &fs.ListArgs{Refresh: req.Refresh}) if err != nil { common.ErrorResp(c, err, 500) return } - total, objs := pagination(objs, &req.PageReq) - provider := "unknown" - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) - if err == nil { - provider = storage.GetStorage().Driver + filtered := make([]model.Obj, 0, len(objs)) + for _, obj := range objs { + childPath := stdpath.Join(reqPath, obj.GetName()) + if common.CanReadPathByRole(user, childPath) { + filtered = append(filtered, obj) + } } + total, pageObjs := pagination(filtered, &req.PageReq) + respContent := toObjsResp(pageObjs, reqPath, isEncrypt(meta, reqPath)) + pagesTotal := calcPagesTotal(total, req.PerPage) + hasMore := req.PerPage != AllPerPage && req.Page*req.PerPage < total + common.SuccessResp(c, FsListResp{ - Content: toObjsResp(objs, reqPath, isEncrypt(meta, reqPath)), - Total: int64(total), - Readme: getReadme(meta, reqPath), - Header: getHeader(meta, reqPath), - Write: user.CanWrite() || common.CanWrite(meta, reqPath), - Provider: provider, + Content: respContent, + Total: int64(total), + FilteredTotal: int64(total), + Page: req.Page, + PerPage: req.PerPage, + HasMore: hasMore, + PagesTotal: pagesTotal, + Readme: getReadme(meta, reqPath), + Header: getHeader(meta, reqPath), + Write: common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, reqPath), + Provider: provider, }) } @@ -133,7 +185,7 @@ func FsDirs(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } @@ -142,7 +194,14 @@ func FsDirs(c *gin.Context) { common.ErrorResp(c, err, 500) return } - dirs := filterDirs(objs) + visible := make([]model.Obj, 0, len(objs)) + for _, obj := range objs { + childPath := stdpath.Join(reqPath, obj.GetName()) + if common.CanReadPathByRole(user, childPath) { + visible = append(visible, obj) + } + } + dirs := filterDirs(visible) common.SuccessResp(c, dirs) } @@ -191,9 +250,43 @@ func isEncrypt(meta *model.Meta, path string) bool { return true } +func normalizeListPage(page, perPage int) (int, int) { + effPage := page + if effPage <= 0 { + effPage = 1 + } + effPerPage := perPage + if effPerPage < 0 { + return effPage, AllPerPage + } + if effPerPage == 0 { + effPerPage = DefaultPerPage + } + if effPerPage > MaxPerPage { + effPerPage = MaxPerPage + } + return effPage, effPerPage +} + +func calcPagesTotal(total, perPage int) int { + if perPage == AllPerPage { + if total > 0 { + return 1 + } + return 0 + } + if total <= 0 || perPage <= 0 { + return 0 + } + return (total + perPage - 1) / perPage +} + func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { pageIndex, pageSize := req.Page, req.PerPage total := len(objs) + if pageSize == AllPerPage { + return total, objs + } start := (pageIndex - 1) * pageSize if start > total { return total, []model.Obj{} @@ -205,21 +298,41 @@ func pagination(objs []model.Obj, req *model.PageReq) (int, []model.Obj) { return total, objs[start:end] } -func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjResp { - var resp []ObjResp +func toObjsResp(objs []model.Obj, parent string, encrypt bool) []ObjLabelResp { + var resp []ObjLabelResp + + names := make([]string, 0, len(objs)) + for _, obj := range objs { + if !obj.IsDir() { + names = append(names, obj.GetName()) + } + } + + labelsByName, _ := op.GetLabelsByFileNamesPublic(names) + for _, obj := range objs { + var labels []model.Label + if !obj.IsDir() { + labels = labelsByName[obj.GetName()] + } thumb, _ := model.GetThumb(obj) - resp = append(resp, ObjResp{ - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: common.Sign(obj, parent, encrypt), - Thumb: thumb, - Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + storageClass, _ := model.GetStorageClass(obj) + resp = append(resp, ObjLabelResp{ + Id: obj.GetID(), + Path: obj.GetPath(), + VirtualPath: utils.FixAndCleanPath(stdpath.Join(parent, obj.GetName())), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: common.Sign(obj, parent, encrypt), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + LabelList: labels, + StorageClass: storageClass, }) } return resp @@ -232,11 +345,12 @@ type FsGetReq struct { type FsGetResp struct { ObjResp - RawURL string `json:"raw_url"` - Readme string `json:"readme"` - Header string `json:"header"` - Provider string `json:"provider"` - Related []ObjResp `json:"related"` + RawURL string `json:"raw_url"` + Readme string `json:"readme"` + Header string `json:"header"` + Provider string `json:"provider"` + WebProxy bool `json:"web_proxy"` + Related []ObjLabelResp `json:"related"` } func FsGet(c *gin.Context) { @@ -259,7 +373,7 @@ func FsGet(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, reqPath, req.Password) { + if !common.CanAccessWithRoles(user, meta, reqPath, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } @@ -270,26 +384,38 @@ func FsGet(c *gin.Context) { } var rawURL string - storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + storage, storageErr := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) provider := "unknown" - if err == nil { + if storageErr == nil { provider = storage.Config().Name } if !obj.IsDir() { - if err != nil { - common.ErrorResp(c, err, 500) + if storageErr != nil { + common.ErrorResp(c, storageErr, 500) return } - if storage.Config().MustProxy() || storage.GetStorage().WebProxy { - query := "" - if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { - query = "?sign=" + sign.Sign(reqPath) - } + query := "" + if isEncrypt(meta, reqPath) || setting.GetBool(conf.SignAll) { + query = "?sign=" + sign.Sign(reqPath) + } + forceRedirectRawURL := storage.GetStorage().Driver == "BaiduYouth" + forcePreviewRawURL := storage.GetStorage().Driver == "Lark" && isLarkCloudDocName(obj.GetName()) + forceProxyRawURL := storage.GetStorage().Driver == "Quark" && utils.GetFileType(obj.GetName()) == conf.VIDEO + if forceRedirectRawURL { + // Baidu Youth direct links are minted per request and are not stable enough + // to expose as fs/get raw_url. Return the local /d endpoint so the frontend + // obtains a fresh link on each download click. + rawURL = fmt.Sprintf("%s/d%s%s", + common.GetApiUrl(c.Request), + utils.EncodePath(reqPath, true), + query) + } else if !forcePreviewRawURL && (storage.Config().MustProxy() || storage.GetStorage().WebProxy || forceProxyRawURL) { if storage.GetStorage().DownProxyUrl != "" { - rawURL = fmt.Sprintf("%s%s?sign=%s", - strings.Split(storage.GetStorage().DownProxyUrl, "\n")[0], - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)) + rawURL = common.BuildDownProxyURL( + storage.GetStorage().DownProxyUrl, + reqPath, + storage.GetStorage().DownProxySign, + ) } else { rawURL = fmt.Sprintf("%s/p%s%s", common.GetApiUrl(c.Request), @@ -303,9 +429,10 @@ func FsGet(c *gin.Context) { } else { // if storage is not proxy, use raw url by fs.Link link, _, err := fs.Link(c, reqPath, model.LinkArgs{ - IP: c.ClientIP(), - Header: c.Request.Header, - HttpReq: c.Request, + IP: c.ClientIP(), + Header: c.Request.Header, + HttpReq: c.Request, + Redirect: true, }) if err != nil { common.ErrorResp(c, err, 500) @@ -323,23 +450,29 @@ func FsGet(c *gin.Context) { } parentMeta, _ := op.GetNearestMeta(parentPath) thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) common.SuccessResp(c, FsGetResp{ ObjResp: ObjResp{ - Name: obj.GetName(), - Size: obj.GetSize(), - IsDir: obj.IsDir(), - Modified: obj.ModTime(), - Created: obj.CreateTime(), - HashInfoStr: obj.GetHash().String(), - HashInfo: obj.GetHash().Export(), - Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), - Type: utils.GetFileType(obj.GetName()), - Thumb: thumb, + Id: obj.GetID(), + Path: obj.GetPath(), + VirtualPath: utils.FixAndCleanPath(reqPath), + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + HashInfoStr: obj.GetHash().String(), + HashInfo: obj.GetHash().Export(), + Sign: common.Sign(obj, parentPath, isEncrypt(meta, reqPath)), + Type: utils.GetFileType(obj.GetName()), + Thumb: thumb, + StorageClass: storageClass, }, RawURL: rawURL, Readme: getReadme(meta, reqPath), Header: getHeader(meta, reqPath), Provider: provider, + WebProxy: storageErr == nil && storage.GetStorage().WebProxy, Related: toObjsResp(related, parentPath, isEncrypt(parentMeta, parentPath)), }) } @@ -358,6 +491,22 @@ func filterRelated(objs []model.Obj, obj model.Obj) []model.Obj { return related } +func isLarkCloudDocName(name string) bool { + for _, suffix := range []string{ + ".lark-doc", + ".lark-docx", + ".lark-sheet", + ".lark-bitable", + ".lark-mindnote", + ".lark-slides", + } { + if strings.HasSuffix(name, suffix) { + return true + } + } + return false +} + type FsOtherReq struct { model.FsOtherArgs Password string `json:"password" form:"password"` @@ -384,7 +533,7 @@ func FsOther(c *gin.Context) { } } c.Set("meta", meta) - if !common.CanAccess(user, meta, req.Path, req.Password) { + if !common.CanAccessWithRoles(user, meta, req.Path, req.Password) { common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) return } diff --git a/server/handles/fsup.go b/server/handles/fsup.go index 15de86600dd..41344fb8d56 100644 --- a/server/handles/fsup.go +++ b/server/handles/fsup.go @@ -7,10 +7,11 @@ import ( "strconv" "time" - "github.com/alist-org/alist/v3/internal/stream" - "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" ) @@ -34,12 +35,20 @@ func FsStream(c *gin.Context) { return } asTask := c.GetHeader("As-Task") == "true" + overwrite := c.GetHeader("Overwrite") != "false" user := c.MustGet("user").(*model.User) path, err = user.JoinPath(path) if err != nil { common.ErrorResp(c, err, 403) return } + if !overwrite { + if res, _ := fs.Get(c, path, &fs.GetArgs{NoLog: true}); res != nil { + _, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body) + common.ErrorStrResp(c, "file exists", 403) + return + } + } dir, name := stdpath.Split(path) sizeStr := c.GetHeader("Content-Length") size, err := strconv.ParseInt(sizeStr, 10, 64) @@ -47,18 +56,34 @@ func FsStream(c *gin.Context) { common.ErrorResp(c, err, 400) return } + h := make(map[*utils.HashType]string) + if md5 := c.GetHeader("X-File-Md5"); md5 != "" { + h[utils.MD5] = md5 + } + if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" { + h[utils.SHA1] = sha1 + } + if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { + h[utils.SHA256] = sha256 + } + mimetype := c.GetHeader("Content-Type") + if len(mimetype) == 0 { + mimetype = utils.GetMimeType(name) + } s := &stream.FileStream{ Obj: &model.Object{ Name: name, Size: size, Modified: getLastModified(c), + HashInfo: utils.NewHashInfoByMap(h), }, Reader: c.Request.Body, - Mimetype: c.GetHeader("Content-Type"), + Mimetype: mimetype, WebPutAsTask: asTask, } + var t task.TaskExtensionInfo if asTask { - err = fs.PutAsTask(dir, s) + t, err = fs.PutAsTask(c, dir, s) } else { err = fs.PutDirectly(c, dir, s, true) } @@ -67,7 +92,16 @@ func FsStream(c *gin.Context) { common.ErrorResp(c, err, 500) return } - common.SuccessResp(c) + if t == nil { + if n, _ := io.ReadFull(c.Request.Body, []byte{0}); n == 1 { + _, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body) + } + common.SuccessResp(c) + return + } + common.SuccessResp(c, gin.H{ + "task": getTaskInfo(t), + }) } func FsForm(c *gin.Context) { @@ -78,12 +112,20 @@ func FsForm(c *gin.Context) { return } asTask := c.GetHeader("As-Task") == "true" + overwrite := c.GetHeader("Overwrite") != "false" user := c.MustGet("user").(*model.User) path, err = user.JoinPath(path) if err != nil { common.ErrorResp(c, err, 403) return } + if !overwrite { + if res, _ := fs.Get(c, path, &fs.GetArgs{NoLog: true}); res != nil { + _, _ = utils.CopyWithBuffer(io.Discard, c.Request.Body) + common.ErrorStrResp(c, "file exists", 403) + return + } + } storage, err := fs.GetStorage(path, &fs.GetStoragesArgs{}) if err != nil { common.ErrorResp(c, err, 400) @@ -105,32 +147,49 @@ func FsForm(c *gin.Context) { } defer f.Close() dir, name := stdpath.Split(path) + h := make(map[*utils.HashType]string) + if md5 := c.GetHeader("X-File-Md5"); md5 != "" { + h[utils.MD5] = md5 + } + if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" { + h[utils.SHA1] = sha1 + } + if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" { + h[utils.SHA256] = sha256 + } + mimetype := file.Header.Get("Content-Type") + if len(mimetype) == 0 { + mimetype = utils.GetMimeType(name) + } s := stream.FileStream{ Obj: &model.Object{ Name: name, Size: file.Size, Modified: getLastModified(c), + HashInfo: utils.NewHashInfoByMap(h), }, Reader: f, - Mimetype: file.Header.Get("Content-Type"), + Mimetype: mimetype, WebPutAsTask: asTask, } + var t task.TaskExtensionInfo if asTask { s.Reader = struct { io.Reader }{f} - err = fs.PutAsTask(dir, &s) + t, err = fs.PutAsTask(c, dir, &s) } else { - ss, err := stream.NewSeekableStream(s, nil) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - err = fs.PutDirectly(c, dir, ss, true) + err = fs.PutDirectly(c, dir, &s, true) } if err != nil { common.ErrorResp(c, err, 500) return } - common.SuccessResp(c) + if t == nil { + common.SuccessResp(c) + return + } + common.SuccessResp(c, gin.H{ + "task": getTaskInfo(t), + }) } diff --git a/server/handles/helper.go b/server/handles/helper.go index 40eba3c449c..bd41c42c3bf 100644 --- a/server/handles/helper.go +++ b/server/handles/helper.go @@ -45,6 +45,8 @@ func Plist(c *gin.Context) { } fullName := c.Param("name") Url := link.String() + Url = strings.ReplaceAll(Url, "<", "[") + Url = strings.ReplaceAll(Url, ">", "]") nameEncode := linkNameSplit[1] fullName, err = url.PathUnescape(nameEncode) if err != nil { diff --git a/server/handles/index.go b/server/handles/index.go index 4e8babd209f..5610688da1f 100644 --- a/server/handles/index.go +++ b/server/handles/index.go @@ -19,7 +19,7 @@ type UpdateIndexReq struct { } func BuildIndex(c *gin.Context) { - if search.Running.Load() { + if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } @@ -45,12 +45,13 @@ func UpdateIndex(c *gin.Context) { common.ErrorResp(c, err, 400) return } - if search.Running.Load() { + if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } if !search.Config(c).AutoUpdate { common.ErrorStrResp(c, "update is not supported for current index", 400) + return } go func() { ctx := context.Background() @@ -71,16 +72,20 @@ func UpdateIndex(c *gin.Context) { } func StopIndex(c *gin.Context) { - if !search.Running.Load() { + quit := search.Quit.Load() + if quit == nil { common.ErrorStrResp(c, "index is not running", 400) return } - search.Quit <- struct{}{} + select { + case *quit <- struct{}{}: + default: + } common.SuccessResp(c) } func ClearIndex(c *gin.Context) { - if search.Running.Load() { + if search.Running() { common.ErrorStrResp(c, "index is running", 400) return } diff --git a/server/handles/label.go b/server/handles/label.go new file mode 100644 index 00000000000..4631124ecbf --- /dev/null +++ b/server/handles/label.go @@ -0,0 +1,99 @@ +package handles + +import ( + "errors" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + "strconv" +) + +func ListLabel(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + log.Debugf("%+v", req) + labels, total, err := db.GetLabels(req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, common.PageResp{ + Content: labels, + Total: total, + }) +} + +func GetLabel(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + label, err := db.GetLabelById(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, label) +} + +func CreateLabel(c *gin.Context) { + var req model.Label + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if db.GetLabelByName(req.Name) { + common.ErrorResp(c, errors.New("label name is exists"), 401) + return + } + if id, err := db.CreateLabel(req); err != nil { + common.ErrorWithDataResp(c, err, 500, gin.H{ + "id": id, + }, true) + } else { + common.SuccessResp(c, gin.H{ + "id": id, + }) + } +} + +func UpdateLabel(c *gin.Context) { + var req model.Label + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if label, err := db.UpdateLabel(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c, label) + } +} + +func DeleteLabel(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + if err = op.DeleteLabelById(c, uint(id), userObj.ID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/label_file_binding.go b/server/handles/label_file_binding.go new file mode 100644 index 00000000000..04f0c105fc2 --- /dev/null +++ b/server/handles/label_file_binding.go @@ -0,0 +1,250 @@ +package handles + +import ( + "errors" + "fmt" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "net/url" + "strconv" + "strings" +) + +type DelLabelFileBinDingReq struct { + FileName string `json:"file_name"` + LabelId string `json:"label_id"` +} + +type pageResp[T any] struct { + Content []T `json:"content"` + Total int64 `json:"total"` +} + +type restoreLabelBindingsReq struct { + KeepIDs bool `json:"keep_ids"` + Override bool `json:"override"` + Bindings []model.LabelFileBinding `json:"bindings"` +} + +func GetLabelByFileName(c *gin.Context) { + fileName := c.Query("file_name") + if fileName == "" { + common.ErrorResp(c, errors.New("file_name must not empty"), 400) + return + } + decodedFileName, err := url.QueryUnescape(fileName) + if err != nil { + common.ErrorResp(c, errors.New("invalid file_name"), 400) + return + } + fmt.Println(">>> 原始 fileName:", fileName) + fmt.Println(">>> 解码后 fileName:", decodedFileName) + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + labels, err := op.GetLabelByFileName(userObj.ID, decodedFileName) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, labels) +} + +func CreateLabelFileBinDing(c *gin.Context) { + var req op.CreateLabelFileBinDingReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.IsDir == true { + common.ErrorStrResp(c, "Unable to bind folder", 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + if err := op.CreateLabelFileBinDing(req, userObj.ID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } else { + common.SuccessResp(c, gin.H{ + "msg": "添加成功!", + }) + } +} + +func DelLabelByFileName(c *gin.Context) { + var req DelLabelFileBinDingReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + labelId, err := strconv.ParseUint(req.LabelId, 10, 64) + if err != nil { + common.ErrorResp(c, fmt.Errorf("invalid label ID '%s': %v", req.LabelId, err), 500, true) + return + } + if err = db.DelLabelFileBinDingById(uint(labelId), userObj.ID, req.FileName); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func GetFileByLabel(c *gin.Context) { + labelId := c.Query("label_id") + if labelId == "" { + common.ErrorResp(c, errors.New("file_name must not empty"), 400) + return + } + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + fileList, err := op.GetFileByLabel(userObj.ID, labelId) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, fileList) +} + +func ListLabelFileBinding(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + + pageStr := c.DefaultQuery("page", "1") + sizeStr := c.DefaultQuery("page_size", "50") + page, err := strconv.Atoi(pageStr) + if err != nil || page <= 0 { + page = 1 + } + pageSize, err := strconv.Atoi(sizeStr) + if err != nil || pageSize <= 0 || pageSize > 200 { + pageSize = 50 + } + + fileName := c.Query("file_name") + labelIDStr := c.Query("label_id") + var labelIDs []uint + if labelIDStr != "" { + parts := strings.Split(labelIDStr, ",") + for _, p := range parts { + if p == "" { + continue + } + id64, err := strconv.ParseUint(strings.TrimSpace(p), 10, 64) + if err != nil { + common.ErrorResp(c, fmt.Errorf("invalid label_id '%s': %v", p, err), 400) + return + } + labelIDs = append(labelIDs, uint(id64)) + } + } + + list, total, err := db.ListLabelFileBinDing(userObj.ID, labelIDs, fileName, page, pageSize) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, pageResp[model.LabelFileBinding]{ + Content: list, + Total: total, + }) +} + +func RestoreLabelFileBinding(c *gin.Context) { + var req restoreLabelBindingsReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if len(req.Bindings) == 0 { + common.ErrorStrResp(c, "empty bindings", 400) + return + } + + if u, ok := c.Value("user").(*model.User); ok { + for i := range req.Bindings { + if req.Bindings[i].UserId == 0 { + req.Bindings[i].UserId = u.ID + } + } + } + + for i := range req.Bindings { + b := req.Bindings[i] + if b.UserId == 0 || b.LabelId == 0 || strings.TrimSpace(b.FileName) == "" { + common.ErrorStrResp(c, "invalid binding: user_id/label_id/file_name required", 400) + return + } + } + + if err := op.RestoreLabelFileBindings(req.Bindings, req.KeepIDs, req.Override); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, gin.H{ + "msg": fmt.Sprintf("restored %d rows", len(req.Bindings)), + }) +} + +func CreateLabelFileBinDingBatch(c *gin.Context) { + var req struct { + Items []op.CreateLabelFileBinDingReq `json:"items" binding:"required"` + } + if err := c.ShouldBindJSON(&req); err != nil || len(req.Items) == 0 { + common.ErrorResp(c, err, 400) + return + } + + userObj, ok := c.Value("user").(*model.User) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + + type perResult struct { + Name string `json:"name"` + Ok bool `json:"ok"` + ErrMsg string `json:"errMsg,omitempty"` + } + results := make([]perResult, 0, len(req.Items)) + succeed := 0 + + for _, item := range req.Items { + if item.IsDir { + results = append(results, perResult{Name: item.Name, Ok: false, ErrMsg: "Unable to bind folder"}) + continue + } + if err := op.CreateLabelFileBinDing(item, userObj.ID); err != nil { + results = append(results, perResult{Name: item.Name, Ok: false, ErrMsg: err.Error()}) + continue + } + succeed++ + results = append(results, perResult{Name: item.Name, Ok: true}) + } + + common.SuccessResp(c, gin.H{ + "total": len(req.Items), + "succeed": succeed, + "failed": len(req.Items) - succeed, + "results": results, + }) +} diff --git a/server/handles/lark.go b/server/handles/lark.go new file mode 100644 index 00000000000..b8f6f7f9d60 --- /dev/null +++ b/server/handles/lark.go @@ -0,0 +1,84 @@ +package handles + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + stdpath "path" + "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" +) + +type larkExportDownloader interface { + DownloadExportFile(ctx context.Context, fileToken string) (io.Reader, string, error) +} + +func LarkExportDownload(c *gin.Context) { + rawPath := c.Query("path") + fileToken := c.Query("file_token") + password := c.Query("password") + filename := strings.TrimSpace(c.Query("filename")) + if rawPath == "" || fileToken == "" { + common.ErrorStrResp(c, "path and file_token are required", 400) + return + } + + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(rawPath) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + meta, err := op.GetNearestMeta(reqPath) + if err != nil { + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500) + return + } + } + if !common.CanAccessWithRoles(user, meta, reqPath, password) { + common.ErrorStrResp(c, "password is incorrect or you have no permission", 403) + return + } + + storage, err := fs.GetStorage(reqPath, &fs.GetStoragesArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + downloader, ok := storage.(larkExportDownloader) + if !ok || storage.GetStorage().Driver != "Lark" { + common.ErrorStrResp(c, "lark export download is not supported for this storage", 400) + return + } + + reader, respFilename, err := downloader.DownloadExportFile(c, fileToken) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if filename == "" { + filename = respFilename + } + if filename == "" { + filename = stdpath.Base(reqPath) + } + + c.Header("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"; filename*=UTF-8''%s`, filename, url.PathEscape(filename))) + c.Header("Content-Type", utils.GetMimeType(filename)) + c.Status(http.StatusOK) + if _, err = io.Copy(c.Writer, reader); err != nil { + common.ErrorResp(c, err, 500) + return + } +} diff --git a/server/handles/ldap_login.go b/server/handles/ldap_login.go new file mode 100644 index 00000000000..fb8417b68b5 --- /dev/null +++ b/server/handles/ldap_login.go @@ -0,0 +1,157 @@ +package handles + +import ( + "crypto/tls" + "errors" + "fmt" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "gopkg.in/ldap.v3" +) + +func LoginLdap(c *gin.Context) { + var req LoginReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + loginLdap(c, &req) +} + +func loginLdap(c *gin.Context, req *LoginReq) { + enabled := setting.GetBool(conf.LdapLoginEnabled) + if !enabled { + common.ErrorStrResp(c, "ldap is not enabled", 403) + return + } + + // check count of login + ip := c.ClientIP() + count, ok := loginCache.Get(ip) + if ok && count >= defaultTimes { + common.ErrorStrResp(c, "Too many unsuccessful sign-in attempts have been made using an incorrect username or password, Try again later.", 429) + loginCache.Expire(ip, defaultDuration) + return + } + + // Auth start + ldapServer := setting.GetStr(conf.LdapServer) + ldapManagerDN := setting.GetStr(conf.LdapManagerDN) + ldapManagerPassword := setting.GetStr(conf.LdapManagerPassword) + ldapUserSearchBase := setting.GetStr(conf.LdapUserSearchBase) + ldapUserSearchFilter := setting.GetStr(conf.LdapUserSearchFilter) // (uid=%s) + + // Connect to LdapServer + l, err := dial(ldapServer) + if err != nil { + utils.Log.Errorf("failed to connect to LDAP: %v", err) + common.ErrorResp(c, err, 500) + return + } + + // First bind with a read only user + if ldapManagerDN != "" && ldapManagerPassword != "" { + err = l.Bind(ldapManagerDN, ldapManagerPassword) + if err != nil { + utils.Log.Errorf("Failed to bind to LDAP: %v", err) + common.ErrorResp(c, err, 500) + return + } + } + + // Search for the given username + searchRequest := ldap.NewSearchRequest( + ldapUserSearchBase, + ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false, + fmt.Sprintf(ldapUserSearchFilter, req.Username), + []string{"dn"}, + nil, + ) + sr, err := l.Search(searchRequest) + if err != nil { + utils.Log.Errorf("LDAP search failed: %v", err) + common.ErrorResp(c, err, 500) + return + } + if len(sr.Entries) != 1 { + utils.Log.Errorf("User does not exist or too many entries returned") + common.ErrorResp(c, err, 500) + return + } + userDN := sr.Entries[0].DN + + // Bind as the user to verify their password + err = l.Bind(userDN, req.Password) + if err != nil { + utils.Log.Errorf("Failed to auth. %v", err) + common.ErrorResp(c, err, 400) + loginCache.Set(ip, count+1) + return + } else { + utils.Log.Infof("Auth successful username:%s", req.Username) + } + // Auth finished + + user, err := op.GetUserByName(req.Username) + if err != nil { + user, err = ladpRegister(req.Username) + if err != nil { + common.ErrorResp(c, err, 400) + loginCache.Set(ip, count+1) + return + } + } + + // generate token + token, err := common.GenerateToken(user) + if err != nil { + common.ErrorResp(c, err, 400, true) + return + } + common.SuccessResp(c, gin.H{"token": token}) + loginCache.Del(ip) +} + +func ladpRegister(username string) (*model.User, error) { + if username == "" { + return nil, errors.New("cannot get username from ldap provider") + } + user := &model.User{ + ID: 0, + Username: username, + Password: random.String(16), + Permission: int32(setting.GetInt(conf.LdapDefaultPermission, 0)), + BasePath: setting.GetStr(conf.LdapDefaultDir), + Role: nil, + Disabled: false, + } + if err := db.CreateUser(user); err != nil { + return nil, err + } + return user, nil +} + +func dial(ldapServer string) (*ldap.Conn, error) { + var tlsEnabled bool = false + if strings.HasPrefix(ldapServer, "ldaps://") { + tlsEnabled = true + ldapServer = strings.TrimPrefix(ldapServer, "ldaps://") + } else if strings.HasPrefix(ldapServer, "ldap://") { + ldapServer = strings.TrimPrefix(ldapServer, "ldap://") + } + + if tlsEnabled { + return ldap.DialTLS("tcp", ldapServer, &tls.Config{InsecureSkipVerify: conf.Conf.TlsInsecureSkipVerify}) + } else { + return ldap.Dial("tcp", ldapServer) + } +} diff --git a/server/handles/meta.go b/server/handles/meta.go index e7454d9df5a..00aa3137192 100644 --- a/server/handles/meta.go +++ b/server/handles/meta.go @@ -2,13 +2,13 @@ package handles import ( "fmt" - "regexp" "strconv" "strings" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" + "github.com/dlclark/regexp2" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" ) @@ -71,7 +71,7 @@ func UpdateMeta(c *gin.Context) { func validHide(hide string) (string, error) { rs := strings.Split(hide, "\n") for _, r := range rs { - _, err := regexp.Compile(r) + _, err := regexp2.Compile(r, regexp2.None) if err != nil { return r, err } diff --git a/server/handles/offline_download.go b/server/handles/offline_download.go new file mode 100644 index 00000000000..68a922efda7 --- /dev/null +++ b/server/handles/offline_download.go @@ -0,0 +1,341 @@ +package handles + +import ( + _115 "github.com/alist-org/alist/v3/drivers/115" + guangyapandriver "github.com/alist-org/alist/v3/drivers/guangyapan" + "github.com/alist-org/alist/v3/drivers/pikpak" + "github.com/alist-org/alist/v3/drivers/thunder" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/task" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +type SetAria2Req struct { + Uri string `json:"uri" form:"uri"` + Secret string `json:"secret" form:"secret"` +} + +func SetAria2(c *gin.Context) { + var req SetAria2Req + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + items := []model.SettingItem{ + {Key: conf.Aria2Uri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.Aria2Secret, Value: req.Secret, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("aria2") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + version, err := _tool.Init() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, version) +} + +type SetQbittorrentReq struct { + Url string `json:"url" form:"url"` + Seedtime string `json:"seedtime" form:"seedtime"` +} + +func SetQbittorrent(c *gin.Context) { + var req SetQbittorrentReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + items := []model.SettingItem{ + {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("qBittorrent") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type SetTransmissionReq struct { + Uri string `json:"uri" form:"uri"` + Seedtime string `json:"seedtime" form:"seedtime"` +} + +func SetTransmission(c *gin.Context) { + var req SetTransmissionReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + items := []model.SettingItem{ + {Key: conf.TransmissionUri, Value: req.Uri, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + {Key: conf.TransmissionSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("Transmission") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type Set115Req struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func Set115(c *gin.Context) { + var req Set115Req + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*_115.Pan115); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only 115 Cloud is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.Pan115TempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("115 Cloud") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type SetPikPakReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetPikPak(c *gin.Context) { + var req SetPikPakReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*pikpak.PikPak); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only PikPak is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.PikPakTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("PikPak") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type SetThunderReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetThunder(c *gin.Context) { + var req SetThunderReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*thunder.Thunder); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only Thunder is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.ThunderTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("Thunder") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +type SetGuangYaPanReq struct { + TempDir string `json:"temp_dir" form:"temp_dir"` +} + +func SetGuangYaPan(c *gin.Context) { + var req SetGuangYaPanReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if req.TempDir != "" { + storage, _, err := op.GetStorageAndActualPath(req.TempDir) + if err != nil { + common.ErrorStrResp(c, "storage does not exists", 400) + return + } + if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK { + common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400) + return + } + if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok { + common.ErrorStrResp(c, "unsupported storage driver for offline download, only GuangYaPan is supported", 400) + return + } + } + items := []model.SettingItem{ + {Key: conf.GuangYaPanTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE}, + } + if err := op.SaveSettingItems(items); err != nil { + common.ErrorResp(c, err, 500) + return + } + _tool, err := tool.Tools.Get("GuangYaPan") + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if _, err := _tool.Init(); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c, "ok") +} + +func OfflineDownloadTools(c *gin.Context) { + tools := tool.Tools.Names() + common.SuccessResp(c, tools) +} + +type AddOfflineDownloadReq struct { + Urls []string `json:"urls"` + Path string `json:"path"` + Tool string `json:"tool"` + DeletePolicy string `json:"delete_policy"` +} + +func AddOfflineDownload(c *gin.Context) { + user := c.MustGet("user").(*model.User) + + var req AddOfflineDownloadReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CheckPathLimitWithRoles(user, reqPath) { + common.ErrorResp(c, errs.PermissionDenied, 403) + return + } + perm := common.MergeRolePermissions(user, reqPath) + if !common.HasPermission(perm, common.PermAddOfflineDownload) { + common.ErrorStrResp(c, "permission denied", 403) + return + } + var tasks []task.TaskExtensionInfo + for _, url := range req.Urls { + t, err := tool.AddURL(c, &tool.AddURLArgs{ + URL: url, + DstDirPath: reqPath, + Tool: req.Tool, + DeletePolicy: tool.DeletePolicy(req.DeletePolicy), + }) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if t != nil { + tasks = append(tasks, t) + } + } + common.SuccessResp(c, gin.H{ + "tasks": getTaskInfos(tasks), + }) +} diff --git a/server/handles/qbittorrent.go b/server/handles/qbittorrent.go deleted file mode 100644 index b22804546ef..00000000000 --- a/server/handles/qbittorrent.go +++ /dev/null @@ -1,79 +0,0 @@ -package handles - -import ( - "github.com/alist-org/alist/v3/internal/conf" - "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/qbittorrent" - "github.com/alist-org/alist/v3/server/common" - "github.com/gin-gonic/gin" -) - -type SetQbittorrentReq struct { - Url string `json:"url" form:"url"` - Seedtime string `json:"seedtime" form:"seedtime"` -} - -func SetQbittorrent(c *gin.Context) { - var req SetQbittorrentReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - items := []model.SettingItem{ - {Key: conf.QbittorrentUrl, Value: req.Url, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE}, - {Key: conf.QbittorrentSeedtime, Value: req.Seedtime, Type: conf.TypeNumber, Group: model.SINGLE, Flag: model.PRIVATE}, - } - if err := op.SaveSettingItems(items); err != nil { - common.ErrorResp(c, err, 500) - return - } - if err := qbittorrent.InitClient(); err != nil { - common.ErrorResp(c, err, 500) - return - } - common.SuccessResp(c, "ok") -} - -type AddQbittorrentReq struct { - Urls []string `json:"urls"` - Path string `json:"path"` -} - -func AddQbittorrent(c *gin.Context) { - user := c.MustGet("user").(*model.User) - if !user.CanAddQbittorrentTasks() { - common.ErrorStrResp(c, "permission denied", 403) - return - } - if !qbittorrent.IsQbittorrentReady() { - // try to init client - err := qbittorrent.InitClient() - if err != nil { - common.ErrorResp(c, err, 500) - return - } - if !qbittorrent.IsQbittorrentReady() { - common.ErrorStrResp(c, "qbittorrent still not ready after init", 500) - return - } - } - var req AddQbittorrentReq - if err := c.ShouldBind(&req); err != nil { - common.ErrorResp(c, err, 400) - return - } - reqPath, err := user.JoinPath(req.Path) - if err != nil { - common.ErrorResp(c, err, 403) - return - } - for _, url := range req.Urls { - err := qbittorrent.AddURL(c, url, reqPath) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - } - common.SuccessResp(c) -} diff --git a/server/handles/role.go b/server/handles/role.go new file mode 100644 index 00000000000..17271a530de --- /dev/null +++ b/server/handles/role.go @@ -0,0 +1,117 @@ +package handles + +import ( + "strconv" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" +) + +func ListRoles(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + log.Debugf("%+v", req) + roles, total, err := op.GetRoles(req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, common.PageResp{Content: roles, Total: total}) +} + +func GetRole(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, role) +} + +func CreateRole(c *gin.Context) { + var req model.Role + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.CreateRole(&req); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c) + } +} + +func UpdateRole(c *gin.Context) { + var req struct { + ID uint `json:"id"` + Name string `json:"name" binding:"required"` + Description string `json:"description"` + PermissionScopes []model.PermissionEntry `json:"permission_scopes"` + Default *bool `json:"default"` + } + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(req.ID) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + switch role.Name { + case "admin": + common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) + return + + case "guest": + req.Name = "guest" + } + role.Name = req.Name + role.Description = req.Description + role.PermissionScopes = req.PermissionScopes + if req.Default != nil { + role.Default = *req.Default + } + if err := op.UpdateRole(role); err != nil { + common.ErrorResp(c, err, 500, true) + } else { + common.SuccessResp(c) + } +} + +func DeleteRole(c *gin.Context) { + idStr := c.Query("id") + id, err := strconv.Atoi(idStr) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + role, err := op.GetRole(uint(id)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorResp(c, errs.ErrChangeDefaultRole, 403) + return + } + if err := op.DeleteRole(uint(id)); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/search.go b/server/handles/search.go index 8881731bd60..832fc94fb07 100644 --- a/server/handles/search.go +++ b/server/handles/search.go @@ -43,28 +43,39 @@ func Search(c *gin.Context) { common.ErrorResp(c, err, 400) return } - nodes, total, err := search.Search(c, req.SearchReq) - if err != nil { - common.ErrorResp(c, err, 500) - return - } - var filteredNodes []model.SearchNode - for _, node := range nodes { - if !strings.HasPrefix(node.Parent, user.BasePath) { - continue + var ( + filteredNodes []model.SearchNode + ) + for len(filteredNodes) < req.PerPage { + nodes, _, err := search.Search(c, req.SearchReq) + if err != nil { + common.ErrorResp(c, err, 500) + return } - meta, err := op.GetNearestMeta(node.Parent) - if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { - continue + if len(nodes) == 0 { + break } - if !common.CanAccess(user, meta, path.Join(node.Parent, node.Name), req.Password) { - continue + for _, node := range nodes { + if !strings.HasPrefix(node.Parent, user.BasePath) { + continue + } + meta, err := op.GetNearestMeta(node.Parent) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + continue + } + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), req.Password) { + continue + } + filteredNodes = append(filteredNodes, node) + if len(filteredNodes) >= req.PerPage { + break + } } - filteredNodes = append(filteredNodes, node) + req.Page++ } common.SuccessResp(c, common.PageResp{ Content: utils.MustSliceConvert(filteredNodes, nodeToSearchResp), - Total: total, + Total: int64(len(filteredNodes)), }) } diff --git a/server/handles/session.go b/server/handles/session.go new file mode 100644 index 00000000000..886be66ab01 --- /dev/null +++ b/server/handles/session.go @@ -0,0 +1,92 @@ +package handles + +import ( + "github.com/alist-org/alist/v3/internal/db" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +type SessionResp struct { + SessionID string `json:"session_id"` + UserID uint `json:"user_id,omitempty"` + LastActive int64 `json:"last_active"` + Status int `json:"status"` + UA string `json:"ua"` + IP string `json:"ip"` +} + +func ListMySessions(c *gin.Context) { + user := c.MustGet("user").(*model.User) + sessions, err := db.ListSessionsByUser(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + resp := make([]SessionResp, len(sessions)) + for i, s := range sessions { + resp[i] = SessionResp{ + SessionID: s.DeviceKey, + LastActive: s.LastActive, + Status: s.Status, + UA: s.UserAgent, + IP: s.IP, + } + } + common.SuccessResp(c, resp) +} + +type EvictSessionReq struct { + SessionID string `json:"session_id"` +} + +func EvictMySession(c *gin.Context) { + var req EvictSessionReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if _, err := db.GetSession(user.ID, req.SessionID); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := db.MarkInactive(req.SessionID); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} + +func ListSessions(c *gin.Context) { + sessions, err := db.ListSessions() + if err != nil { + common.ErrorResp(c, err, 500) + return + } + resp := make([]SessionResp, len(sessions)) + for i, s := range sessions { + resp[i] = SessionResp{ + SessionID: s.DeviceKey, + UserID: s.UserID, + LastActive: s.LastActive, + Status: s.Status, + UA: s.UserAgent, + IP: s.IP, + } + } + common.SuccessResp(c, resp) +} + +func EvictSession(c *gin.Context) { + var req EvictSessionReq + if err := c.ShouldBindJSON(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := db.MarkInactive(req.SessionID); err != nil { + common.ErrorResp(c, err, 500) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/setting.go b/server/handles/setting.go index f778b1803c5..81f7dc61c24 100644 --- a/server/handles/setting.go +++ b/server/handles/setting.go @@ -5,6 +5,7 @@ import ( "strings" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/frp" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/sign" @@ -14,9 +15,28 @@ import ( "github.com/gin-gonic/gin" ) +func getRoleOptions() string { + roles, _, err := op.GetRoles(1, model.MaxInt) + if err != nil { + return "" + } + names := make([]string, 0, len(roles)) + for _, r := range roles { + if r.Name == "admin" || r.Name == "guest" { + continue + } + names = append(names, r.Name) + } + return strings.Join(names, ",") +} + +type SetTokenReq struct { + Token string `json:"token" form:"token" binding:"required"` +} + func ResetToken(c *gin.Context) { token := random.Token() - item := model.SettingItem{Key: "token", Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} + item := model.SettingItem{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} if err := op.SaveSettingItem(&item); err != nil { common.ErrorResp(c, err, 500) return @@ -25,6 +45,21 @@ func ResetToken(c *gin.Context) { common.SuccessResp(c, token) } +func SetToken(c *gin.Context) { + var req SetTokenReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + item := model.SettingItem{Key: conf.Token, Value: req.Token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE} + if err := op.SaveSettingItem(&item); err != nil { + common.ErrorResp(c, err, 500) + return + } + sign.Instance() + common.SuccessResp(c, req.Token) +} + func GetSetting(c *gin.Context) { key := c.Query("key") keys := c.Query("keys") @@ -34,6 +69,17 @@ func GetSetting(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if item.Key == conf.DefaultRole { + copy := *item + copy.Options = getRoleOptions() + if id, err := strconv.Atoi(copy.Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + copy.Value = r.Name + } + } + common.SuccessResp(c, copy) + return + } common.SuccessResp(c, item) } else { items, err := op.GetSettingItemInKeys(strings.Split(keys, ",")) @@ -41,6 +87,17 @@ func GetSetting(c *gin.Context) { common.ErrorResp(c, err, 400) return } + for i := range items { + if items[i].Key == conf.DefaultRole { + if id, err := strconv.Atoi(items[i].Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + items[i].Value = r.Name + } + } + items[i].Options = getRoleOptions() + break + } + } common.SuccessResp(c, items) } } @@ -51,6 +108,22 @@ func SaveSettings(c *gin.Context) { common.ErrorResp(c, err, 400) return } + + for i := range req { + if req[i].Key == conf.DefaultRole { + role, err := op.GetRoleByName(req[i].Value) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + if role.Name == "admin" || role.Name == "guest" { + common.ErrorStrResp(c, "cannot set admin or guest as default role", 400) + return + } + req[i].Value = strconv.Itoa(int(role.ID)) + } + } + if err := op.SaveSettingItems(req); err != nil { common.ErrorResp(c, err, 500) } else { @@ -88,6 +161,17 @@ func ListSettings(c *gin.Context) { common.ErrorResp(c, err, 400) return } + for i := range settings { + if settings[i].Key == conf.DefaultRole { + if id, err := strconv.Atoi(settings[i].Value); err == nil { + if r, err := op.GetRole(uint(id)); err == nil { + settings[i].Value = r.Name + } + } + settings[i].Options = getRoleOptions() + break + } + } common.SuccessResp(c, settings) } @@ -100,6 +184,42 @@ func DeleteSetting(c *gin.Context) { common.SuccessResp(c) } +// SetFRP saves FRP settings and restarts the FRP client. +// Returns the current FRP connection status. +func SetFRP(c *gin.Context) { + var req []model.SettingItem + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + if err := op.SaveSettingItems(req); err != nil { + common.ErrorResp(c, err, 500) + return + } + if err := frp.Instance.Restart(); err != nil { + common.SuccessResp(c, frp.Instance.Status()) + return + } + common.SuccessResp(c, frp.Instance.Status()) +} + +// StopFRP stops the FRP client and returns current status. +func StopFRP(c *gin.Context) { + frp.Instance.Stop() + common.SuccessResp(c, frp.Instance.Status()) +} + +// GetFRPRuntime returns current FRP status and recent runtime logs. +func GetFRPRuntime(c *gin.Context) { + limit := 200 + if limitStr := c.Query("limit"); limitStr != "" { + if parsed, err := strconv.Atoi(limitStr); err == nil { + limit = parsed + } + } + common.SuccessResp(c, frp.Instance.Runtime(limit)) +} + func PublicSettings(c *gin.Context) { common.SuccessResp(c, op.GetPublicSettingsMap()) } diff --git a/server/handles/share.go b/server/handles/share.go new file mode 100644 index 00000000000..4de7da63672 --- /dev/null +++ b/server/handles/share.go @@ -0,0 +1,635 @@ +package handles + +import ( + "crypto/subtle" + "errors" + "fmt" + "net/http" + "net/url" + stdpath "path" + "regexp" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/db" + shareauth "github.com/alist-org/alist/v3/internal/share" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/pkg/utils/random" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +const shareAccessTokenLifetime = 24 * time.Hour + +var shareIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,32}$`) + +var ( + errShareIDInvalid = errors.New("share_id must be 1-32 characters of letters, numbers, underscore or hyphen") + errShareIDExists = errors.New("share link already exists") +) + +type CreateShareReq struct { + Path string `json:"path" binding:"required"` + ShareID string `json:"share_id"` + Name string `json:"name"` + Password string `json:"password"` + ExpireAt string `json:"expire_at"` + ExpireHours int64 `json:"expire_hours"` + AccessLimit int64 `json:"access_limit"` + BurnAfterRead *bool `json:"burn_after_read"` + AllowPreview *bool `json:"allow_preview"` + AllowDownload *bool `json:"allow_download"` +} + +type UpdateShareReq struct { + ShareID string `json:"share_id" binding:"required"` + NewShareID string `json:"new_share_id"` + Name string `json:"name"` + Password string `json:"password"` + ExpireAt *string `json:"expire_at"` + AccessLimit *int64 `json:"access_limit"` + AllowPreview *bool `json:"allow_preview"` + AllowDownload *bool `json:"allow_download"` +} + +type ShareDeleteReq struct { + ShareID string `json:"share_id" binding:"required"` +} + +type ShareAuthReq struct { + ShareID string `json:"share_id" binding:"required"` + Password string `json:"password"` +} + +type PublicShareReq struct { + ShareID string `json:"share_id" binding:"required"` + Path string `json:"path"` + Token string `json:"token"` +} + +type PublicShareListReq struct { + model.PageReq + ShareID string `json:"share_id" binding:"required"` + Path string `json:"path"` + Token string `json:"token"` +} + +type ShareResp struct { + ID uint `json:"id"` + ShareID string `json:"share_id"` + Name string `json:"name"` + RootPath string `json:"root_path"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + RemainingAccesses int64 `json:"remaining_accesses"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Enabled bool `json:"enabled"` + ViewCount int64 `json:"view_count"` + DownloadCount int64 `json:"download_count"` + LastAccessAt *time.Time `json:"last_access_at"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + URL string `json:"url"` +} + +type PublicShareInfoResp struct { + ShareID string `json:"share_id"` + Name string `json:"name"` + IsDir bool `json:"is_dir"` + HasPassword bool `json:"has_password"` + BurnAfterRead bool `json:"burn_after_read"` + AccessLimit int64 `json:"access_limit"` + AccessCount int64 `json:"access_count"` + RemainingAccesses int64 `json:"remaining_accesses"` + AllowPreview bool `json:"allow_preview"` + AllowDownload bool `json:"allow_download"` + Authed bool `json:"authed"` + ConsumedAt *time.Time `json:"consumed_at"` + ExpiresAt *time.Time `json:"expires_at"` + CreatedAt time.Time `json:"created_at"` +} + +type PublicShareObjResp struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + Thumb string `json:"thumb"` + Type int `json:"type"` + Path string `json:"path"` + StorageClass string `json:"storage_class,omitempty"` + DownloadURL string `json:"download_url,omitempty"` + PreviewURL string `json:"preview_url,omitempty"` +} + +type PublicShareListResp struct { + Content []PublicShareObjResp `json:"content"` + Total int64 `json:"total"` + Page int `json:"page"` + PerPage int `json:"per_page"` + HasMore bool `json:"has_more"` + PagesTotal int `json:"pages_total"` +} + +type PublicShareGetResp struct { + Item PublicShareObjResp `json:"item"` + Provider string `json:"provider"` +} + +func shareURL(c *gin.Context, shareID string) string { + return fmt.Sprintf("%s/s/%s", common.GetApiUrl(c.Request), shareID) +} + +func toShareResp(c *gin.Context, share *model.Share) ShareResp { + accessLimit := share.EffectiveAccessLimit() + return ShareResp{ + ID: share.ID, + ShareID: share.ShareID, + Name: share.Name, + RootPath: share.RootPath, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + BurnAfterRead: accessLimit == 1, + AccessLimit: accessLimit, + AccessCount: share.AccessCount, + RemainingAccesses: share.RemainingAccesses(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Enabled: share.Enabled, + ViewCount: share.ViewCount, + DownloadCount: share.DownloadCount, + LastAccessAt: share.LastAccessAt, + ConsumedAt: share.ConsumedAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + UpdatedAt: share.UpdatedAt, + URL: shareURL(c, share.ShareID), + } +} + +func normalizeShareName(obj model.Obj, name string) string { + return normalizeOptionalShareName(name, obj.GetName()) +} + +func normalizeOptionalShareName(name, fallback string) string { + if strings.TrimSpace(name) != "" { + return strings.TrimSpace(name) + } + return fallback +} + +func generateShareID() (string, error) { + for range 10 { + shareID := random.String(8) + exists, err := db.ShareIDExists(shareID) + if err != nil { + return "", err + } + if !exists { + return shareID, nil + } + } + return "", fmt.Errorf("failed to generate unique share id") +} + +func sharePasswordHash(password, salt string) string { + return model.HashPwd(model.StaticHash(password), salt) +} + +func validateCustomShareID(shareID string) error { + if shareID == "" { + return nil + } + if !shareIDPattern.MatchString(shareID) { + return errShareIDInvalid + } + return nil +} + +func resolveRequestedShareID(rawShareID, fallback string, excludeID uint) (string, error) { + shareID := strings.TrimSpace(rawShareID) + if shareID == "" { + if fallback != "" { + return fallback, nil + } + return generateShareID() + } + if err := validateCustomShareID(shareID); err != nil { + return "", err + } + if excludeID == 0 { + exists, err := db.ShareIDExists(shareID) + if err != nil { + return "", fmt.Errorf("check share id availability: %w", err) + } + if exists { + return "", errShareIDExists + } + return shareID, nil + } + exists, err := db.ShareIDExistsExceptID(shareID, excludeID) + if err != nil { + return "", fmt.Errorf("check share id availability: %w", err) + } + if exists { + return "", errShareIDExists + } + return shareID, nil +} + +func normalizeShareAccessLimit(accessLimit int64, burnAfterRead *bool) (int64, bool, error) { + if accessLimit < 0 { + return 0, false, fmt.Errorf("access_limit must be 0 or greater") + } + if accessLimit == 0 && burnAfterRead != nil && *burnAfterRead { + accessLimit = 1 + } + return accessLimit, accessLimit == 1, nil +} + +func parseShareExpireAt(raw string) (*time.Time, error) { + value := strings.TrimSpace(raw) + if value == "" { + return nil, nil + } + if parsed, err := time.Parse(time.RFC3339, value); err == nil { + return &parsed, nil + } + layouts := []string{ + "2006-01-02T15:04:05", + "2006-01-02T15:04", + "2006-01-02 15:04:05", + } + for _, layout := range layouts { + if parsed, err := time.ParseInLocation(layout, value, time.Local); err == nil { + return &parsed, nil + } + } + return nil, fmt.Errorf("invalid expire_at") +} + +func resolveShareExpireAt(expireAt string, expireHours int64) (*time.Time, error) { + if strings.TrimSpace(expireAt) != "" { + return parseShareExpireAt(expireAt) + } + if expireHours < 0 { + return nil, fmt.Errorf("expire_hours must be 0 or greater") + } + if expireHours == 0 { + return nil, nil + } + expires := time.Now().Add(time.Duration(expireHours) * time.Hour) + return &expires, nil +} + +func sharePasswordMatched(share *model.Share, password string) bool { + if !share.HasPassword() { + return true + } + hash := sharePasswordHash(password, share.PasswordSalt) + return subtle.ConstantTimeCompare([]byte(hash), []byte(share.PasswordHash)) == 1 +} + +func getShareAccessToken(c *gin.Context, fallback string) string { + if fallback != "" { + return fallback + } + if token := c.Query("auth"); token != "" { + return token + } + return c.GetHeader("X-Share-Token") +} + +func ensureShareAvailable(c *gin.Context, share *model.Share) bool { + now := time.Now() + if share.IsConsumed() { + common.ErrorStrResp(c, "share has been consumed", 410) + return false + } + if !share.Enabled { + common.ErrorStrResp(c, "share is disabled", 404) + return false + } + if share.IsExpired(now) { + common.ErrorStrResp(c, "share is expired", 410) + return false + } + return true +} + +func recordShareAccess(share *model.Share) error { + updated, err := db.RecordShareAccess(share.ShareID) + if err != nil { + return err + } + if updated != nil { + *share = *updated + } + return nil +} + +func ensureShareAccess(c *gin.Context, share *model.Share, token string) bool { + if !share.HasPassword() { + return true + } + if token == "" { + common.ErrorStrResp(c, "share password required", 401) + return false + } + if err := shareauth.VerifyAccess(share, token); err != nil { + common.ErrorResp(c, err, 401) + return false + } + return true +} + +func shouldTrackShareContentAccess(c *gin.Context) bool { + return c.Request.Method != http.MethodHead +} + +func resolveShareTarget(share *model.Share, rawRelPath string) (string, string, error) { + cleanRelPath := utils.FixAndCleanPath(rawRelPath) + if !share.IsDir && cleanRelPath != "/" { + return "", "", fmt.Errorf("file share does not support nested path") + } + if cleanRelPath == "/" { + return share.RootPath, "/", nil + } + target := utils.FixAndCleanPath(stdpath.Join(share.RootPath, cleanRelPath)) + if !utils.IsSubPath(share.RootPath, target) { + return "", "", fmt.Errorf("share path out of range") + } + return target, cleanRelPath, nil +} + +func resolveShareWildcardTarget(share *model.Share, rawPath string) (string, string, error) { + path, err := url.PathUnescape(rawPath) + if err != nil { + return "", "", err + } + return resolveShareTarget(share, strings.TrimPrefix(path, "/")) +} + +func buildPublicShareAssetURL(c *gin.Context, prefix, shareID, relPath, token string, preview bool) string { + base := common.GetApiUrl(c.Request) + prefix + shareID + cleanPath := utils.FixAndCleanPath(relPath) + if cleanPath != "/" { + base += utils.EncodePath(cleanPath, true) + } + query := url.Values{} + if token != "" { + query.Set("auth", token) + } + if preview { + query.Set("type", "preview") + } + if encoded := query.Encode(); encoded != "" { + base += "?" + encoded + } + return base +} + +func buildPublicSharePreviewURL(c *gin.Context, obj model.Obj, targetPath, shareID, relPath, token string) string { + prefix := "/sd/" + storage, err := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if err == nil && canProxy(storage, obj.GetName()) { + prefix = "/sp/" + } + return buildPublicShareAssetURL(c, prefix, shareID, relPath, token, true) +} + +func toPublicShareObjResp(c *gin.Context, share *model.Share, obj model.Obj, targetPath, relPath, token string) PublicShareObjResp { + thumb, _ := model.GetThumb(obj) + storageClass, _ := model.GetStorageClass(obj) + resp := PublicShareObjResp{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + Thumb: thumb, + Type: utils.GetObjType(obj.GetName(), obj.IsDir()), + Path: relPath, + StorageClass: storageClass, + } + if !obj.IsDir() && share.AllowDownload { + resp.DownloadURL = buildPublicShareAssetURL(c, "/sd/", share.ShareID, relPath, token, false) + } + if !obj.IsDir() && share.AllowPreview { + resp.PreviewURL = buildPublicSharePreviewURL(c, obj, targetPath, share.ShareID, relPath, token) + } + return resp +} + +func CreateShare(c *gin.Context) { + var req CreateShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + reqPath, err := user.JoinPath(req.Path) + if err != nil { + common.ErrorResp(c, err, 403) + return + } + if !common.CanReadPathByRole(user, reqPath) { + common.ErrorStrResp(c, "you have no permission", 403) + return + } + obj, err := fs.Get(c, reqPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + shareID, err := resolveRequestedShareID(req.ShareID, "", 0) + if err != nil { + if errors.Is(err, errShareIDInvalid) || errors.Is(err, errShareIDExists) { + common.ErrorResp(c, err, 400) + return + } + common.ErrorResp(c, err, 500, true) + return + } + allowPreview := true + if req.AllowPreview != nil { + allowPreview = *req.AllowPreview + } + allowDownload := true + if req.AllowDownload != nil { + allowDownload = *req.AllowDownload + } + accessLimit, burnAfterRead, err := normalizeShareAccessLimit(req.AccessLimit, req.BurnAfterRead) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + expiresAt, err := resolveShareExpireAt(req.ExpireAt, req.ExpireHours) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + share := &model.Share{ + ShareID: shareID, + CreatorID: user.ID, + Name: normalizeShareName(obj, req.Name), + RootPath: reqPath, + IsDir: obj.IsDir(), + BurnAfterRead: burnAfterRead, + AccessLimit: accessLimit, + AllowPreview: allowPreview, + AllowDownload: allowDownload, + Enabled: true, + ExpiresAt: expiresAt, + } + if req.Password != "" { + share.PasswordSalt = random.String(16) + share.PasswordHash = sharePasswordHash(req.Password, share.PasswordSalt) + } + if err := db.CreateShare(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, toShareResp(c, share)) +} + +func UpdateShare(c *gin.Context) { + var req UpdateShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + share, err := db.GetShareByCreatorAndShareID(user.ID, req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + + shareID, err := resolveRequestedShareID(req.NewShareID, share.ShareID, share.ID) + if err != nil { + if errors.Is(err, errShareIDInvalid) || errors.Is(err, errShareIDExists) { + common.ErrorResp(c, err, 400) + return + } + common.ErrorResp(c, err, 500, true) + return + } + + allowPreview := share.AllowPreview + if req.AllowPreview != nil { + allowPreview = *req.AllowPreview + } + allowDownload := share.AllowDownload + if req.AllowDownload != nil { + allowDownload = *req.AllowDownload + } + + accessLimit := share.EffectiveAccessLimit() + burnAfterRead := accessLimit == 1 + if req.AccessLimit != nil { + accessLimit, burnAfterRead, err = normalizeShareAccessLimit(*req.AccessLimit, nil) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + expiresAt := share.ExpiresAt + if req.ExpireAt != nil { + expiresAt, err = parseShareExpireAt(*req.ExpireAt) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + } + + share.ShareID = shareID + share.Name = normalizeOptionalShareName(req.Name, share.Name) + share.BurnAfterRead = burnAfterRead + share.AccessLimit = accessLimit + share.AllowPreview = allowPreview + share.AllowDownload = allowDownload + share.ExpiresAt = expiresAt + if req.Password != "" { + share.PasswordSalt = random.String(16) + share.PasswordHash = sharePasswordHash(req.Password, share.PasswordSalt) + } + if share.Enabled && accessLimit > 0 && share.AccessCount >= accessLimit { + now := time.Now() + share.Enabled = false + if share.ConsumedAt == nil { + share.ConsumedAt = &now + } + } + if err := db.UpdateShare(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, toShareResp(c, share)) +} + +func ListShares(c *gin.Context) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + user := c.MustGet("user").(*model.User) + shares, total, err := db.GetSharesByCreator(user.ID, req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + resp := make([]ShareResp, 0, len(shares)) + for i := range shares { + resp = append(resp, toShareResp(c, &shares[i])) + } + common.SuccessResp(c, common.PageResp{ + Content: resp, + Total: total, + }) +} + +func DisableShare(c *gin.Context) { + var req ShareDeleteReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if _, err := db.GetShareByCreatorAndShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 404) + return + } + if err := db.DisableShareByShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func DeleteShare(c *gin.Context) { + var req ShareDeleteReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + user := c.MustGet("user").(*model.User) + if err := db.DeleteShareByShareID(user.ID, req.ShareID); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} diff --git a/server/handles/share_page.go b/server/handles/share_page.go new file mode 100644 index 00000000000..98cc62d8daf --- /dev/null +++ b/server/handles/share_page.go @@ -0,0 +1,16 @@ +package handles + +import ( + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/gin-gonic/gin" +) + +func GetSharePage(c *gin.Context) { + c.Header("Content-Type", "text/html") + c.Status(200) + _, _ = c.Writer.WriteString(strings.Replace(conf.IndexHtml, "", "", 1)) + c.Writer.Flush() + c.Writer.WriteHeaderNow() +} diff --git a/server/handles/share_public.go b/server/handles/share_public.go new file mode 100644 index 00000000000..f88d5b338c5 --- /dev/null +++ b/server/handles/share_public.go @@ -0,0 +1,275 @@ +package handles + +import ( + stdpath "path" + "time" + + "github.com/alist-org/alist/v3/internal/db" + shareauth "github.com/alist-org/alist/v3/internal/share" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" +) + +func GetPublicShareInfo(c *gin.Context) { + shareID := c.Query("share_id") + if shareID == "" { + common.ErrorStrResp(c, "share_id is required", 400) + return + } + share, err := db.GetShareByShareID(shareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, "") + authed := !share.HasPassword() + if share.HasPassword() && token != "" { + authed = shareauth.VerifyAccess(share, token) == nil + } + if authed { + _ = db.TouchShareView(share.ShareID) + } + common.SuccessResp(c, PublicShareInfoResp{ + ShareID: share.ShareID, + Name: share.Name, + IsDir: share.IsDir, + HasPassword: share.HasPassword(), + BurnAfterRead: share.EffectiveAccessLimit() == 1, + AccessLimit: share.EffectiveAccessLimit(), + AccessCount: share.AccessCount, + RemainingAccesses: share.RemainingAccesses(), + AllowPreview: share.AllowPreview, + AllowDownload: share.AllowDownload, + Authed: authed, + ConsumedAt: share.ConsumedAt, + ExpiresAt: share.ExpiresAt, + CreatedAt: share.CreatedAt, + }) +} + +func AuthPublicShare(c *gin.Context) { + var req ShareAuthReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !sharePasswordMatched(share, req.Password) { + common.ErrorStrResp(c, "password is incorrect", 403) + return + } + token := "" + if share.HasPassword() { + ttl := shareAccessTokenLifetime + if share.ExpiresAt != nil { + remaining := time.Until(*share.ExpiresAt) + if remaining <= 0 { + common.ErrorStrResp(c, "share is expired", 410) + return + } + if remaining < ttl { + ttl = remaining + } + } + token = shareauth.SignAccess(share, ttl) + } + _ = db.TouchShareView(share.ShareID) + common.SuccessResp(c, gin.H{"token": token}) +} + +func ListPublicShare(c *gin.Context) { + var req PublicShareListReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Page, req.PerPage = normalizeListPage(req.Page, req.PerPage) + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, req.Token) + if !ensureShareAccess(c, share, token) { + return + } + targetPath, relPath, err := resolveShareTarget(share, req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !obj.IsDir() { + common.ErrorStrResp(c, "path is not a directory", 400) + return + } + objs, err := fs.List(c, targetPath, &fs.ListArgs{}) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + total, pageObjs := pagination(objs, &req.PageReq) + content := make([]PublicShareObjResp, 0, len(pageObjs)) + for _, item := range pageObjs { + itemRelPath := relPath + if itemRelPath == "/" { + itemRelPath = stdpath.Join("/", item.GetName()) + } else { + itemRelPath = stdpath.Join(relPath, item.GetName()) + } + itemTargetPath, _, err := resolveShareTarget(share, itemRelPath) + if err != nil { + continue + } + content = append(content, toPublicShareObjResp(c, share, item, itemTargetPath, itemRelPath, token)) + } + common.SuccessResp(c, PublicShareListResp{ + Content: content, + Total: int64(total), + Page: req.Page, + PerPage: req.PerPage, + HasMore: req.PerPage != AllPerPage && req.Page*req.PerPage < total, + PagesTotal: calcPagesTotal(total, req.PerPage), + }) +} + +func GetPublicShare(c *gin.Context) { + var req PublicShareReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + share, err := db.GetShareByShareID(req.ShareID) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + token := getShareAccessToken(c, req.Token) + if !ensureShareAccess(c, share, token) { + return + } + targetPath, relPath, err := resolveShareTarget(share, req.Path) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + provider := "unknown" + storage, storageErr := fs.GetStorage(targetPath, &fs.GetStoragesArgs{}) + if storageErr == nil { + provider = storage.GetStorage().Driver + } + common.SuccessResp(c, PublicShareGetResp{ + Item: toPublicShareObjResp(c, share, obj, targetPath, relPath, token), + Provider: provider, + }) +} + +func ShareDown(c *gin.Context) { + share, err := db.GetShareByShareID(c.Param("share_id")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !share.AllowDownload { + common.ErrorStrResp(c, "download is not allowed", 403) + return + } + token := getShareAccessToken(c, "") + if !ensureShareAccess(c, share, token) { + return + } + targetPath, _, err := resolveShareWildcardTarget(share, c.Param("path")) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "directory download is not supported", 400) + return + } + if shouldTrackShareContentAccess(c) { + _ = db.TouchShareDownload(share.ShareID) + if err := recordShareAccess(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("path", targetPath) + Down(c) +} + +func ShareProxy(c *gin.Context) { + share, err := db.GetShareByShareID(c.Param("share_id")) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if !ensureShareAvailable(c, share) { + return + } + if !share.AllowPreview { + common.ErrorStrResp(c, "preview is not allowed", 403) + return + } + token := getShareAccessToken(c, "") + if !ensureShareAccess(c, share, token) { + return + } + targetPath, _, err := resolveShareWildcardTarget(share, c.Param("path")) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + obj, err := fs.Get(c, targetPath, &fs.GetArgs{}) + if err != nil { + common.ErrorResp(c, err, 404) + return + } + if obj.IsDir() { + common.ErrorStrResp(c, "directory preview is not supported", 400) + return + } + if shouldTrackShareContentAccess(c) { + if err := recordShareAccess(share); err != nil { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("path", targetPath) + Proxy(c) +} diff --git a/server/handles/sshkey.go b/server/handles/sshkey.go new file mode 100644 index 00000000000..6f8d46b4969 --- /dev/null +++ b/server/handles/sshkey.go @@ -0,0 +1,125 @@ +package handles + +import ( + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" + "github.com/gin-gonic/gin" + "strconv" + "strings" +) + +type SSHKeyAddReq struct { + Title string `json:"title" binding:"required"` + Key string `json:"key" binding:"required"` +} + +func AddMyPublicKey(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok || userObj.IsGuest() { + common.ErrorStrResp(c, "user invalid", 401) + return + } + var req SSHKeyAddReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorStrResp(c, "request invalid", 400) + return + } + if req.Title == "" { + common.ErrorStrResp(c, "request invalid", 400) + return + } + key := &model.SSHPublicKey{ + Title: req.Title, + KeyStr: strings.TrimSpace(req.Key), + UserId: userObj.ID, + } + err, parsed := op.CreateSSHPublicKey(key) + if !parsed { + common.ErrorStrResp(c, "provided key invalid", 400) + return + } else if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func ListMyPublicKey(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok || userObj.IsGuest() { + common.ErrorStrResp(c, "user invalid", 401) + return + } + list(c, userObj) +} + +func DeleteMyPublicKey(c *gin.Context) { + userObj, ok := c.Value("user").(*model.User) + if !ok || userObj.IsGuest() { + common.ErrorStrResp(c, "user invalid", 401) + return + } + keyId, err := strconv.Atoi(c.Query("id")) + if err != nil { + common.ErrorStrResp(c, "id format invalid", 400) + return + } + key, err := op.GetSSHPublicKeyByIdAndUserId(uint(keyId), userObj.ID) + if err != nil { + common.ErrorStrResp(c, "failed to get public key", 404) + return + } + err = op.DeleteSSHPublicKeyById(key.ID) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func ListPublicKeys(c *gin.Context) { + userId, err := strconv.Atoi(c.Query("uid")) + if err != nil { + common.ErrorStrResp(c, "user id format invalid", 400) + return + } + userObj, err := op.GetUserById(uint(userId)) + if err != nil { + common.ErrorStrResp(c, "user invalid", 404) + return + } + list(c, userObj) +} + +func DeletePublicKey(c *gin.Context) { + keyId, err := strconv.Atoi(c.Query("id")) + if err != nil { + common.ErrorStrResp(c, "id format invalid", 400) + return + } + err = op.DeleteSSHPublicKeyById(uint(keyId)) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c) +} + +func list(c *gin.Context, userObj *model.User) { + var req model.PageReq + if err := c.ShouldBind(&req); err != nil { + common.ErrorResp(c, err, 400) + return + } + req.Validate() + keys, total, err := op.GetSSHPublicKeyByUserId(userObj.ID, req.Page, req.PerPage) + if err != nil { + common.ErrorResp(c, err, 500, true) + return + } + common.SuccessResp(c, common.PageResp{ + Content: keys, + Total: total, + }) +} diff --git a/server/handles/ssologin.go b/server/handles/ssologin.go index 89d8ecaf480..779cc13239b 100644 --- a/server/handles/ssologin.go +++ b/server/handles/ssologin.go @@ -1,16 +1,18 @@ package handles import ( - "encoding/base32" "encoding/base64" "errors" "fmt" + "github.com/alist-org/alist/v3/internal/op" "net/http" "net/url" "path" "strings" "time" + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/db" "github.com/alist-org/alist/v3/internal/model" @@ -21,29 +23,45 @@ import ( "github.com/coreos/go-oidc" "github.com/gin-gonic/gin" "github.com/go-resty/resty/v2" - "github.com/pquerna/otp" - "github.com/pquerna/otp/totp" "golang.org/x/oauth2" "gorm.io/gorm" ) -var opts = totp.ValidateOpts{ - // state verify won't expire in 30 secs, which is quite enough for the callback - Period: 30, - Skew: 1, - // in some OIDC providers(such as Authelia), state parameter must be at least 8 characters - Digits: otp.DigitsEight, - Algorithm: otp.AlgorithmSHA1, +const stateLength = 16 +const stateExpire = time.Minute * 5 + +var stateCache = cache.NewMemCache[string](cache.WithShards[string](stateLength)) + +func _keyState(clientID, state string) string { + return fmt.Sprintf("%s_%s", clientID, state) +} + +func generateState(clientID, ip string) string { + state := random.String(stateLength) + stateCache.Set(_keyState(clientID, state), ip, cache.WithEx[string](stateExpire)) + return state +} + +func verifyState(clientID, ip, state string) bool { + value, ok := stateCache.Get(_keyState(clientID, state)) + return ok && value == ip +} + +func ssoRedirectUri(c *gin.Context, useCompatibility bool, method string) string { + if useCompatibility { + return common.GetApiUrl(c.Request) + "/api/auth/" + method + } else { + return common.GetApiUrl(c.Request) + "/api/auth/sso_callback" + "?method=" + method + } } func SSOLoginRedirect(c *gin.Context) { method := c.Query("method") - usecompatibility := setting.GetBool(conf.SSOCompatibilityMode) + useCompatibility := setting.GetBool(conf.SSOCompatibilityMode) enabled := setting.GetBool(conf.SSOLoginEnabled) clientId := setting.GetStr(conf.SSOClientId) platform := setting.GetStr(conf.SSOLoginPlatform) - var r_url string - var redirect_uri string + var rUrl string if !enabled { common.ErrorStrResp(c, "Single sign-on is not enabled", 403) return @@ -53,69 +71,52 @@ func SSOLoginRedirect(c *gin.Context) { common.ErrorStrResp(c, "no method provided", 400) return } - if usecompatibility { - redirect_uri = common.GetApiUrl(c.Request) + "/api/auth/" + method - } else { - redirect_uri = common.GetApiUrl(c.Request) + "/api/auth/sso_callback" + "?method=" + method - } + redirectUri := ssoRedirectUri(c, useCompatibility, method) urlValues.Add("response_type", "code") - urlValues.Add("redirect_uri", redirect_uri) + urlValues.Add("redirect_uri", redirectUri) urlValues.Add("client_id", clientId) switch platform { case "Github": - r_url = "https://github.com/login/oauth/authorize?" + rUrl = "https://github.com/login/oauth/authorize?" urlValues.Add("scope", "read:user") case "Microsoft": - r_url = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?" + rUrl = "https://login.microsoftonline.com/common/oauth2/v2.0/authorize?" urlValues.Add("scope", "user.read") urlValues.Add("response_mode", "query") case "Google": - r_url = "https://accounts.google.com/o/oauth2/v2/auth?" + rUrl = "https://accounts.google.com/o/oauth2/v2/auth?" urlValues.Add("scope", "https://www.googleapis.com/auth/userinfo.profile") case "Dingtalk": - r_url = "https://login.dingtalk.com/oauth2/auth?" + rUrl = "https://login.dingtalk.com/oauth2/auth?" urlValues.Add("scope", "openid") urlValues.Add("prompt", "consent") urlValues.Add("response_type", "code") case "Casdoor": endpoint := strings.TrimSuffix(setting.GetStr(conf.SSOEndpointName), "/") - r_url = endpoint + "/login/oauth/authorize?" + rUrl = endpoint + "/login/oauth/authorize?" urlValues.Add("scope", "profile") urlValues.Add("state", endpoint) case "OIDC": - oauth2Config, err := GetOIDCClient(c) - if err != nil { - common.ErrorStrResp(c, err.Error(), 400) - return - } - // generate state parameter - state, err := totp.GenerateCodeCustom(base32.StdEncoding.EncodeToString([]byte(oauth2Config.ClientSecret)), time.Now(), opts) + oauth2Config, err := GetOIDCClient(c, useCompatibility, redirectUri, method) if err != nil { common.ErrorStrResp(c, err.Error(), 400) return } + state := generateState(clientId, c.ClientIP()) c.Redirect(http.StatusFound, oauth2Config.AuthCodeURL(state)) return default: common.ErrorStrResp(c, "invalid platform", 400) return } - c.Redirect(302, r_url+urlValues.Encode()) + c.Redirect(302, rUrl+urlValues.Encode()) } var ssoClient = resty.New().SetRetryCount(3) -func GetOIDCClient(c *gin.Context) (*oauth2.Config, error) { - var redirect_uri string - usecompatibility := setting.GetBool(conf.SSOCompatibilityMode) - argument := c.Query("method") - if usecompatibility { - argument = path.Base(c.Request.URL.Path) - } - if usecompatibility { - redirect_uri = common.GetApiUrl(c.Request) + "/api/auth/" + argument - } else { - redirect_uri = common.GetApiUrl(c.Request) + "/api/auth/sso_callback" + "?method=" + argument +func GetOIDCClient(c *gin.Context, useCompatibility bool, redirectUri, method string) (*oauth2.Config, error) { + if redirectUri == "" { + redirectUri = ssoRedirectUri(c, useCompatibility, method) } endpoint := setting.GetStr(conf.SSOEndpointName) provider, err := oidc.NewProvider(c, endpoint) @@ -124,16 +125,20 @@ func GetOIDCClient(c *gin.Context) (*oauth2.Config, error) { } clientId := setting.GetStr(conf.SSOClientId) clientSecret := setting.GetStr(conf.SSOClientSecret) + extraScopes := []string{} + if setting.GetStr(conf.SSOExtraScopes) != "" { + extraScopes = strings.Split(setting.GetStr(conf.SSOExtraScopes), " ") + } return &oauth2.Config{ ClientID: clientId, ClientSecret: clientSecret, - RedirectURL: redirect_uri, + RedirectURL: redirectUri, // Discovery returns the OAuth2 endpoints. Endpoint: provider.Endpoint(), // "openid" is a required scope for OpenID Connect flows. - Scopes: []string{oidc.ScopeOpenID, "profile"}, + Scopes: append([]string{oidc.ScopeOpenID, "profile"}, extraScopes...), }, nil } @@ -150,7 +155,7 @@ func autoRegister(username, userID string, err error) (*model.User, error) { Password: random.String(16), Permission: int32(setting.GetInt(conf.SSODefaultPermission, 0)), BasePath: setting.GetStr(conf.SSODefaultDir), - Role: 0, + Role: model.Roles{op.GetDefaultRoleID()}, Disabled: false, SsoID: userID, } @@ -181,9 +186,9 @@ func parseJWT(p string) ([]byte, error) { func OIDCLoginCallback(c *gin.Context) { useCompatibility := setting.GetBool(conf.SSOCompatibilityMode) - argument := c.Query("method") + method := c.Query("method") if useCompatibility { - argument = path.Base(c.Request.URL.Path) + method = path.Base(c.Request.URL.Path) } clientId := setting.GetStr(conf.SSOClientId) endpoint := setting.GetStr(conf.SSOEndpointName) @@ -192,18 +197,12 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorResp(c, err, 400) return } - oauth2Config, err := GetOIDCClient(c) + oauth2Config, err := GetOIDCClient(c, useCompatibility, "", method) if err != nil { common.ErrorResp(c, err, 400) return } - // add state verify process - stateVerification, err := totp.ValidateCustom(c.Query("state"), base32.StdEncoding.EncodeToString([]byte(oauth2Config.ClientSecret)), time.Now(), opts) - if err != nil { - common.ErrorResp(c, err, 400) - return - } - if !stateVerification { + if !verifyState(clientId, c.ClientIP(), c.Query("state")) { common.ErrorStrResp(c, "incorrect or expired state parameter", 400) return } @@ -231,12 +230,12 @@ func OIDCLoginCallback(c *gin.Context) { common.ErrorResp(c, err, 400) return } - userID := utils.Json.Get(payload, conf.SSOOIDCUsernameKey).ToString() + userID := utils.Json.Get(payload, setting.GetStr(conf.SSOOIDCUsernameKey, "name")).ToString() if userID == "" { common.ErrorStrResp(c, "cannot get username from OIDC provider", 400) return } - if argument == "get_sso_id" { + if method == "get_sso_id" { if useCompatibility { c.Redirect(302, common.GetApiUrl(c.Request)+"/@manage?sso_id="+userID) return @@ -252,15 +251,16 @@ func OIDCLoginCallback(c *gin.Context) { c.Data(200, "text/html; charset=utf-8", []byte(html)) return } - if argument == "sso_get_token" { + if method == "sso_get_token" { user, err := db.GetUserBySSOID(userID) if err != nil { user, err = autoRegister(userID, userID, err) if err != nil { common.ErrorResp(c, err, 400) + return } } - token, err := common.GenerateToken(user.Username) + token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) } @@ -398,7 +398,7 @@ func SSOLoginCallback(c *gin.Context) { } userID := utils.Json.Get(resp.Body(), idField).ToString() if utils.SliceContains([]string{"", "0"}, userID) { - common.ErrorResp(c, errors.New("error occured"), 400) + common.ErrorResp(c, errors.New("error occurred"), 400) return } if argument == "get_sso_id" { @@ -426,7 +426,7 @@ func SSOLoginCallback(c *gin.Context) { return } } - token, err := common.GenerateToken(user.Username) + token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400) } diff --git a/server/handles/task.go b/server/handles/task.go index d76bb586e27..a4dbce0f1fa 100644 --- a/server/handles/task.go +++ b/server/handles/task.go @@ -1,125 +1,226 @@ package handles import ( - "strconv" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/task" + "math" + "time" - "github.com/alist-org/alist/v3/internal/aria2" "github.com/alist-org/alist/v3/internal/fs" - "github.com/alist-org/alist/v3/internal/qbittorrent" - "github.com/alist-org/alist/v3/pkg/task" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "github.com/xhofe/tache" ) type TaskInfo struct { - ID string `json:"id"` - Name string `json:"name"` - State string `json:"state"` - Status string `json:"status"` - Progress int `json:"progress"` - Error string `json:"error"` + ID string `json:"id"` + Name string `json:"name"` + Creator string `json:"creator"` + CreatorRole model.Roles `json:"creator_role"` + State tache.State `json:"state"` + Status string `json:"status"` + Progress float64 `json:"progress"` + StartTime *time.Time `json:"start_time"` + EndTime *time.Time `json:"end_time"` + TotalBytes int64 `json:"total_bytes"` + Error string `json:"error"` } -type K2Str[K comparable] func(k K) string - -func uint64K2Str(k uint64) string { - return strconv.FormatUint(k, 10) +func getTaskInfo[T task.TaskExtensionInfo](task T) TaskInfo { + errMsg := "" + if task.GetErr() != nil { + errMsg = task.GetErr().Error() + } + progress := task.GetProgress() + // if progress is NaN, set it to 100 + if math.IsNaN(progress) { + progress = 100 + } + creatorName := "" + var creatorRole model.Roles + if task.GetCreator() != nil { + creatorName = task.GetCreator().Username + creatorRole = task.GetCreator().Role + } + return TaskInfo{ + ID: task.GetID(), + Name: task.GetName(), + Creator: creatorName, + CreatorRole: creatorRole, + State: task.GetState(), + Status: task.GetStatus(), + Progress: progress, + StartTime: task.GetStartTime(), + EndTime: task.GetEndTime(), + TotalBytes: task.GetTotalBytes(), + Error: errMsg, + } } -func strK2Str(str string) string { - return str +func getTaskInfos[T task.TaskExtensionInfo](tasks []T) []TaskInfo { + return utils.MustSliceConvert(tasks, getTaskInfo[T]) } -func getTaskInfo[K comparable](task *task.Task[K], k2Str K2Str[K]) TaskInfo { - return TaskInfo{ - ID: k2Str(task.ID), - Name: task.Name, - State: task.GetState(), - Status: task.GetStatus(), - Progress: task.GetProgress(), - Error: task.GetErrMsg(), - } +func argsContains[T comparable](v T, slice ...T) bool { + return utils.SliceContains(slice, v) } -func getTaskInfos[K comparable](tasks []*task.Task[K], k2Str K2Str[K]) []TaskInfo { - var infos []TaskInfo - for _, t := range tasks { - infos = append(infos, getTaskInfo(t, k2Str)) +func getUserInfo(c *gin.Context) (bool, uint, bool) { + if user, ok := c.Value("user").(*model.User); ok { + return user.IsAdmin(), user.ID, true + } else { + return false, 0, false } - return infos } -type Str2K[K comparable] func(str string) (K, error) - -func str2Uint64K(str string) (uint64, error) { - return strconv.ParseUint(str, 10, 64) +func getTargetedHandler[T task.TaskExtensionInfo](manager task.Manager[T], callback func(c *gin.Context, task T)) gin.HandlerFunc { + return func(c *gin.Context) { + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) + return + } + t, ok := manager.GetByID(c.Query("tid")) + if !ok { + common.ErrorStrResp(c, "task not found", 404) + return + } + if !isAdmin && uid != t.GetCreator().ID { + // to avoid an attacker using error messages to guess valid TID, return a 404 rather than a 403 + common.ErrorStrResp(c, "task not found", 404) + return + } + callback(c, t) + } } -func str2StrK(str string) (string, error) { - return str, nil +func getBatchHandler[T task.TaskExtensionInfo](manager task.Manager[T], callback func(task T)) gin.HandlerFunc { + return func(c *gin.Context) { + isAdmin, uid, ok := getUserInfo(c) + if !ok { + common.ErrorStrResp(c, "user invalid", 401) + return + } + var tids []string + if err := c.ShouldBind(&tids); err != nil { + common.ErrorStrResp(c, "invalid request format", 400) + return + } + retErrs := make(map[string]string) + for _, tid := range tids { + t, ok := manager.GetByID(tid) + if !ok || (!isAdmin && uid != t.GetCreator().ID) { + retErrs[tid] = "task not found" + continue + } + callback(t) + } + common.SuccessResp(c, retErrs) + } } -func taskRoute[K comparable](g *gin.RouterGroup, manager *task.Manager[K], k2Str K2Str[K], str2K Str2K[K]) { +func taskRoute[T task.TaskExtensionInfo](g *gin.RouterGroup, manager task.Manager[T]) { g.GET("/undone", func(c *gin.Context) { - common.SuccessResp(c, getTaskInfos(manager.ListUndone(), k2Str)) + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) + return + } + common.SuccessResp(c, getTaskInfos(manager.GetByCondition(func(task T) bool { + // avoid directly passing the user object into the function to reduce closure size + return (isAdmin || uid == task.GetCreator().ID) && + argsContains(task.GetState(), tache.StatePending, tache.StateRunning, tache.StateCanceling, + tache.StateErrored, tache.StateFailing, tache.StateWaitingRetry, tache.StateBeforeRetry) + }))) }) g.GET("/done", func(c *gin.Context) { - common.SuccessResp(c, getTaskInfos(manager.ListDone(), k2Str)) - }) - g.POST("/cancel", func(c *gin.Context) { - tid := c.Query("tid") - id, err := str2K(tid) - if err != nil { - common.ErrorResp(c, err, 400) + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) return } - if err := manager.Cancel(id); err != nil { - common.ErrorResp(c, err, 500) - } else { - common.SuccessResp(c) - } + common.SuccessResp(c, getTaskInfos(manager.GetByCondition(func(task T) bool { + return (isAdmin || uid == task.GetCreator().ID) && + argsContains(task.GetState(), tache.StateCanceled, tache.StateFailed, tache.StateSucceeded) + }))) }) - g.POST("/delete", func(c *gin.Context) { - tid := c.Query("tid") - id, err := str2K(tid) - if err != nil { - common.ErrorResp(c, err, 400) + g.POST("/info", getTargetedHandler(manager, func(c *gin.Context, task T) { + common.SuccessResp(c, getTaskInfo(task)) + })) + g.POST("/cancel", getTargetedHandler(manager, func(c *gin.Context, task T) { + manager.Cancel(task.GetID()) + common.SuccessResp(c) + })) + g.POST("/delete", getTargetedHandler(manager, func(c *gin.Context, task T) { + manager.Remove(task.GetID()) + common.SuccessResp(c) + })) + g.POST("/retry", getTargetedHandler(manager, func(c *gin.Context, task T) { + manager.Retry(task.GetID()) + common.SuccessResp(c) + })) + g.POST("/cancel_some", getBatchHandler(manager, func(task T) { + manager.Cancel(task.GetID()) + })) + g.POST("/delete_some", getBatchHandler(manager, func(task T) { + manager.Remove(task.GetID()) + })) + g.POST("/retry_some", getBatchHandler(manager, func(task T) { + manager.Retry(task.GetID()) + })) + g.POST("/clear_done", func(c *gin.Context) { + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) return } - if err := manager.Remove(id); err != nil { - common.ErrorResp(c, err, 500) - } else { - common.SuccessResp(c) - } + manager.RemoveByCondition(func(task T) bool { + return (isAdmin || uid == task.GetCreator().ID) && + argsContains(task.GetState(), tache.StateCanceled, tache.StateFailed, tache.StateSucceeded) + }) + common.SuccessResp(c) }) - g.POST("/retry", func(c *gin.Context) { - tid := c.Query("tid") - id, err := str2K(tid) - if err != nil { - common.ErrorResp(c, err, 400) + g.POST("/clear_succeeded", func(c *gin.Context) { + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) return } - if err := manager.Retry(id); err != nil { - common.ErrorResp(c, err, 500) - } else { - common.SuccessResp(c) - } - }) - g.POST("/clear_done", func(c *gin.Context) { - manager.ClearDone() + manager.RemoveByCondition(func(task T) bool { + return (isAdmin || uid == task.GetCreator().ID) && task.GetState() == tache.StateSucceeded + }) common.SuccessResp(c) }) - g.POST("/clear_succeeded", func(c *gin.Context) { - manager.ClearSucceeded() + g.POST("/retry_failed", func(c *gin.Context) { + isAdmin, uid, ok := getUserInfo(c) + if !ok { + // if there is no bug, here is unreachable + common.ErrorStrResp(c, "user invalid", 401) + return + } + tasks := manager.GetByCondition(func(task T) bool { + return (isAdmin || uid == task.GetCreator().ID) && task.GetState() == tache.StateFailed + }) + for _, t := range tasks { + manager.Retry(t.GetID()) + } common.SuccessResp(c) }) } func SetupTaskRoute(g *gin.RouterGroup) { - taskRoute(g.Group("/aria2_down"), aria2.DownTaskManager, strK2Str, str2StrK) - taskRoute(g.Group("/aria2_transfer"), aria2.TransferTaskManager, uint64K2Str, str2Uint64K) - taskRoute(g.Group("/upload"), fs.UploadTaskManager, uint64K2Str, str2Uint64K) - taskRoute(g.Group("/copy"), fs.CopyTaskManager, uint64K2Str, str2Uint64K) - taskRoute(g.Group("/qbit_down"), qbittorrent.DownTaskManager, strK2Str, str2StrK) - taskRoute(g.Group("/qbit_transfer"), qbittorrent.TransferTaskManager, uint64K2Str, str2Uint64K) + taskRoute(g.Group("/upload"), fs.UploadTaskManager) + taskRoute(g.Group("/copy"), fs.CopyTaskManager) + taskRoute(g.Group("/offline_download"), tool.DownloadTaskManager) + taskRoute(g.Group("/offline_download_transfer"), tool.TransferTaskManager) + taskRoute(g.Group("/s3_transition"), fs.S3TransitionTaskManager) + taskRoute(g.Group("/decompress"), fs.ArchiveDownloadTaskManager) + taskRoute(g.Group("/decompress_upload"), fs.ArchiveContentUploadTaskManager) } diff --git a/server/handles/user.go b/server/handles/user.go index 2220648f750..ac3a06e8180 100644 --- a/server/handles/user.go +++ b/server/handles/user.go @@ -3,6 +3,8 @@ package handles import ( "strconv" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/server/common" @@ -35,12 +37,16 @@ func CreateUser(c *gin.Context) { common.ErrorResp(c, err, 400) return } + if len(req.Role) == 0 { + req.Role = model.Roles{op.GetDefaultRoleID()} + } if req.IsAdmin() || req.IsGuest() { common.ErrorStrResp(c, "admin or guest user can not be created", 400, true) return } req.SetPassword(req.Password) req.Password = "" + req.Authn = "[]" if err := op.CreateUser(&req); err != nil { common.ErrorResp(c, err, 500, true) } else { @@ -59,10 +65,18 @@ func UpdateUser(c *gin.Context) { common.ErrorResp(c, err, 500) return } - if user.Role != req.Role { - common.ErrorStrResp(c, "role can not be changed", 400) - return + + if user.Username == "admin" { + if !utils.SliceEqual(user.Role, req.Role) { + common.ErrorStrResp(c, "cannot change role of admin user", 403) + return + } + //if user.Username != req.Username { + // common.ErrorStrResp(c, "cannot change username of admin user", 403) + // return + //} } + if req.Password == "" { req.PwdHash = user.PwdHash req.Salt = user.Salt @@ -73,10 +87,25 @@ func UpdateUser(c *gin.Context) { if req.OtpSecret == "" { req.OtpSecret = user.OtpSecret } - if req.Disabled && req.IsAdmin() { - common.ErrorStrResp(c, "admin user can not be disabled", 400) - return + if req.Disabled && user.IsAdmin() { + count, err := op.CountEnabledAdminsExcluding(user.ID) + if err != nil { + common.ErrorResp(c, err, 500) + return + } + if count == 0 { + common.ErrorStrResp(c, "at least one enabled admin must be kept", 400) + return + } + } + + if !utils.SliceEqual(user.Role, req.Role) { + if req.IsAdmin() || req.IsGuest() { + common.ErrorStrResp(c, "cannot assign admin or guest role to user", 400, true) + return + } } + if err := op.UpdateUser(&req); err != nil { common.ErrorResp(c, err, 500) } else { diff --git a/server/handles/webauthn.go b/server/handles/webauthn.go index 952cf480b46..c6a7650c991 100644 --- a/server/handles/webauthn.go +++ b/server/handles/webauthn.go @@ -2,6 +2,7 @@ package handles import ( "encoding/base64" + "encoding/binary" "encoding/json" "fmt" @@ -13,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" + "github.com/go-webauthn/webauthn/protocol" "github.com/go-webauthn/webauthn/webauthn" ) @@ -22,28 +24,30 @@ func BeginAuthnLogin(c *gin.Context) { common.ErrorStrResp(c, "WebAuthn is not enabled", 403) return } - username := c.Query("username") - if username == "" { - common.ErrorStrResp(c, "empty or no username provided", 400) - return - } - user, err := db.GetUserByName(username) - if err != nil { - common.ErrorResp(c, err, 400) - return - } authnInstance, err := authn.NewAuthnInstance(c.Request) if err != nil { common.ErrorResp(c, err, 400) return } - options, sessionData, err := authnInstance.BeginLogin(user) - + var ( + options *protocol.CredentialAssertion + sessionData *webauthn.SessionData + ) + if username := c.Query("username"); username != "" { + var user *model.User + user, err = db.GetUserByName(username) + if err == nil { + options, sessionData, err = authnInstance.BeginLogin(user) + } + } else { // client-side discoverable login + options, sessionData, err = authnInstance.BeginDiscoverableLogin() + } if err != nil { common.ErrorResp(c, err, 400) return } + val, err := json.Marshal(sessionData) if err != nil { common.ErrorResp(c, err, 400) @@ -61,20 +65,13 @@ func FinishAuthnLogin(c *gin.Context) { common.ErrorStrResp(c, "WebAuthn is not enabled", 403) return } - username := c.Query("username") - user, err := db.GetUserByName(username) + authnInstance, err := authn.NewAuthnInstance(c.Request) if err != nil { common.ErrorResp(c, err, 400) return } sessionDataString := c.GetHeader("session") - - authnInstance, err := authn.NewAuthnInstance(c.Request) - if err != nil { - common.ErrorResp(c, err, 400) - return - } sessionDataBytes, err := base64.StdEncoding.DecodeString(sessionDataString) if err != nil { common.ErrorResp(c, err, 400) @@ -87,14 +84,34 @@ func FinishAuthnLogin(c *gin.Context) { return } - _, err = authnInstance.FinishLogin(user, sessionData, c.Request) + var user *model.User + if username := c.Query("username"); username != "" { + user, err = db.GetUserByName(username) + if err != nil { + common.ErrorResp(c, err, 400) + return + } + _, err = authnInstance.FinishLogin(user, sessionData, c.Request) + } else { // client-side discoverable login + _, err = authnInstance.FinishDiscoverableLogin(func(_, userHandle []byte) (webauthn.User, error) { + // first param `rawID` in this callback function is equal to ID in webauthn.Credential, + // but it's unnnecessary to check it. + // userHandle param is equal to (User).WebAuthnID(). + userID := uint(binary.LittleEndian.Uint64(userHandle)) + user, err = db.GetUserById(userID) + if err != nil { + return nil, err + } + return user, nil + }, sessionData, c.Request) + } if err != nil { common.ErrorResp(c, err, 400) return } - token, err := common.GenerateToken(user.Username) + token, err := common.GenerateToken(user) if err != nil { common.ErrorResp(c, err, 400, true) return @@ -190,6 +207,10 @@ func DeleteAuthnLogin(c *gin.Context) { return } err = db.RemoveAuthn(user, req.ID) + if err != nil { + common.ErrorResp(c, err, 400) + return + } err = op.DelUserCache(user.Username) if err != nil { common.ErrorResp(c, err, 400) diff --git a/server/mcp/auth.go b/server/mcp/auth.go new file mode 100644 index 00000000000..6d1e670d14a --- /dev/null +++ b/server/mcp/auth.go @@ -0,0 +1,182 @@ +package mcp + +import ( + "context" + "crypto/subtle" + "fmt" + "net/http" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/server/common" + log "github.com/sirupsen/logrus" +) + +type ctxKey string + +const userKey ctxKey = "user" + +// HTTPContextFunc extracts JWT/admin token from HTTP request and injects user into context. +// Used as WithHTTPContextFunc callback for Streamable HTTP transport. +func HTTPContextFunc(ctx context.Context, r *http.Request) context.Context { + token := r.Header.Get("Authorization") + if token == "" { + token = r.URL.Query().Get("token") + } + + user, err := authenticateToken(token) + if err != nil { + log.Debugf("MCP auth failed: %v", err) + return ctx + } + + return context.WithValue(ctx, userKey, user) +} + +func authenticateToken(token string) (*model.User, error) { + // Check admin static token + if token != "" && subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { + admin, err := op.GetAdmin() + if err != nil { + return nil, fmt.Errorf("failed to get admin: %w", err) + } + if err := loadRoles(admin); err != nil { + return nil, err + } + return admin, nil + } + + // No token: guest + if token == "" { + guest, err := op.GetGuest() + if err != nil { + return nil, fmt.Errorf("failed to get guest: %w", err) + } + if guest.Disabled { + return nil, fmt.Errorf("guest user is disabled") + } + if err := loadRoles(guest); err != nil { + return nil, err + } + return guest, nil + } + + // JWT token + claims, err := common.ParseToken(token) + if err != nil { + return nil, fmt.Errorf("invalid token: %w", err) + } + + user, err := op.GetUserByName(claims.Username) + if err != nil { + return nil, fmt.Errorf("user not found: %w", err) + } + + if claims.PwdTS != user.PwdTS { + return nil, fmt.Errorf("password has been changed") + } + if user.Disabled { + return nil, fmt.Errorf("user is disabled") + } + + if err := loadRoles(user); err != nil { + return nil, err + } + return user, nil +} + +func loadRoles(user *model.User) error { + if len(user.Role) > 0 { + roles, err := op.GetRolesByUserID(user.ID) + if err != nil { + return fmt.Errorf("failed to load roles: %w", err) + } + user.RolesDetail = roles + } + return nil +} + +// resolveUser extracts the authenticated user from context. +func resolveUser(ctx context.Context) (*model.User, error) { + user, ok := ctx.Value(userKey).(*model.User) + if !ok || user == nil { + return nil, fmt.Errorf("authentication required") + } + return user, nil +} + +// buildFsContext resolves path and sets meta in context for fs operations. +func buildFsContext(ctx context.Context, user *model.User, path string) (context.Context, string, error) { + reqPath, err := user.JoinPath(path) + if err != nil { + return ctx, "", err + } + meta, _ := op.GetNearestMeta(reqPath) + ctx = context.WithValue(ctx, "meta", meta) + ctx = context.WithValue(ctx, "user", user) + return ctx, reqPath, nil +} + +// checkAccess checks if user can access the path (read). +func checkAccess(user *model.User, reqPath string) error { + meta, _ := op.GetNearestMeta(reqPath) + if !common.CanAccessWithRoles(user, meta, reqPath, "") { + return fmt.Errorf("permission denied") + } + perm := common.MergeRolePermissions(user, reqPath) + if !user.IsAdmin() && !common.HasPermission(perm, common.PermMCPAccess) { + return fmt.Errorf("MCP access not permitted") + } + return nil +} + +// checkManage checks if user can perform write operations via MCP. +func checkManage(user *model.User, reqPath string, permBit uint) error { + if err := checkAccess(user, reqPath); err != nil { + return err + } + perm := common.MergeRolePermissions(user, reqPath) + if !user.IsAdmin() && !common.HasPermission(perm, common.PermMCPManage) { + return fmt.Errorf("MCP manage not permitted") + } + if !user.IsAdmin() && !common.HasPermission(perm, permBit) { + return fmt.Errorf("permission denied for this operation") + } + return nil +} + +// UserContextFunc returns an HTTPContextFunc that injects a specific user (for STDIO mode). +func userContextMiddleware(user *model.User) func(ctx context.Context) context.Context { + return func(ctx context.Context) context.Context { + return context.WithValue(ctx, userKey, user) + } +} + +// resolveUserForStdio resolves a user by username for STDIO mode. +func resolveUserForStdio(username string) (*model.User, error) { + username = strings.TrimSpace(username) + if username == "" || username == "admin" { + admin, err := op.GetAdmin() + if err != nil { + return nil, fmt.Errorf("failed to get admin user: %w", err) + } + if err := loadRoles(admin); err != nil { + return nil, err + } + return admin, nil + } + user, err := op.GetUserByName(username) + if err != nil { + return nil, fmt.Errorf("user %q not found: %w", username, err) + } + if user.Disabled { + return nil, fmt.Errorf("user %q is disabled", username) + } + if err := loadRoles(user); err != nil { + return nil, err + } + return user, nil +} diff --git a/server/mcp/convert.go b/server/mcp/convert.go new file mode 100644 index 00000000000..7eede8cbc87 --- /dev/null +++ b/server/mcp/convert.go @@ -0,0 +1,56 @@ +package mcp + +import ( + "encoding/json" + "time" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/mark3labs/mcp-go/mcp" +) + +type objJSON struct { + Name string `json:"name"` + Size int64 `json:"size"` + IsDir bool `json:"is_dir"` + Modified time.Time `json:"modified"` + Created time.Time `json:"created"` + HashInfo map[string]string `json:"hash_info,omitempty"` +} + +func objToJSON(obj model.Obj) objJSON { + j := objJSON{ + Name: obj.GetName(), + Size: obj.GetSize(), + IsDir: obj.IsDir(), + Modified: obj.ModTime(), + Created: obj.CreateTime(), + } + hi := obj.GetHash() + if hm := hashInfoToMap(hi); len(hm) > 0 { + j.HashInfo = hm + } + return j +} + +func hashInfoToMap(hi utils.HashInfo) map[string]string { + m := make(map[string]string) + for ht, v := range hi.All() { + if v != "" { + m[ht.Name] = v + } + } + return m +} + +func jsonResult(v interface{}) (*mcp.CallToolResult, error) { + data, err := json.Marshal(v) + if err != nil { + return nil, err + } + return mcp.NewToolResultText(string(data)), nil +} + +func textResult(msg string) (*mcp.CallToolResult, error) { + return mcp.NewToolResultText(msg), nil +} diff --git a/server/mcp/errors.go b/server/mcp/errors.go new file mode 100644 index 00000000000..c107633739f --- /dev/null +++ b/server/mcp/errors.go @@ -0,0 +1,40 @@ +package mcp + +import ( + "fmt" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/mark3labs/mcp-go/mcp" + pkgerr "github.com/pkg/errors" +) + +func toolError(msg string) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(msg), nil +} + +func toolErrorf(format string, args ...interface{}) (*mcp.CallToolResult, error) { + return mcp.NewToolResultError(fmt.Sprintf(format, args...)), nil +} + +func wrapError(err error) (*mcp.CallToolResult, error) { + if err == nil { + return nil, nil + } + cause := pkgerr.Cause(err) + switch { + case errs.IsObjectNotFound(err) || errs.IsNotFoundError(err): + return toolErrorf("not found: %s", err.Error()) + case cause == errs.PermissionDenied: + return toolError("permission denied") + case cause == errs.NotImplement: + return toolError("not supported by storage driver") + case cause == errs.NotSupport: + return toolError("operation not supported") + case cause == errs.UploadNotSupported: + return toolError("upload not supported by storage") + case cause == errs.MoveBetweenTwoStorages: + return toolError("can't move between two storages, use copy instead") + default: + return toolErrorf("error: %s", err.Error()) + } +} diff --git a/server/mcp/server.go b/server/mcp/server.go new file mode 100644 index 00000000000..b45a0113eb3 --- /dev/null +++ b/server/mcp/server.go @@ -0,0 +1,64 @@ +package mcp + +import ( + "context" + "net/http" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +// NewServer creates an MCP server with all alist tools registered. +func NewServer() *mcpserver.MCPServer { + s := mcpserver.NewMCPServer( + "alist", + conf.Version, + mcpserver.WithToolCapabilities(false), + mcpserver.WithRecovery(), + ) + registerReadTools(s) + registerManageTools(s) + registerUploadTools(s) + return s +} + +// NewHTTPHandler creates a Streamable HTTP handler for the MCP server. +func NewHTTPHandler() http.Handler { + s := NewServer() + return mcpserver.NewStreamableHTTPServer(s, + mcpserver.WithHTTPContextFunc(HTTPContextFunc), + ) +} + +// NewStdioServer creates an MCP server configured for STDIO mode with a fixed user. +func NewStdioServer(username string) (*mcpserver.MCPServer, *model.User, error) { + user, err := resolveUserForStdio(username) + if err != nil { + return nil, nil, err + } + s := NewServer() + return s, user, nil +} + +// ServeStdio starts the MCP server in STDIO mode. +func ServeStdio(username string) error { + s, user, err := NewStdioServer(username) + if err != nil { + return err + } + ctxFunc := userContextMiddleware(user) + return mcpserver.ServeStdio(s, mcpserver.WithStdioContextFunc(ctxFunc)) +} + +// toolHandlerWithAuth wraps a tool handler to require authentication. +func toolHandlerWithAuth(fn func(ctx context.Context, user *model.User, request mcp.CallToolRequest) (*mcp.CallToolResult, error)) mcpserver.ToolHandlerFunc { + return func(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + user, err := resolveUser(ctx) + if err != nil { + return toolError("authentication required") + } + return fn(ctx, user, request) + } +} diff --git a/server/mcp/tools_manage.go b/server/mcp/tools_manage.go new file mode 100644 index 00000000000..d287bb6bed0 --- /dev/null +++ b/server/mcp/tools_manage.go @@ -0,0 +1,228 @@ +package mcp + +import ( + "context" + + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +func registerManageTools(s *mcpserver.MCPServer) { + // fs_mkdir + s.AddTool(mcp.NewTool("fs_mkdir", + mcp.WithDescription("Create a new directory"), + mcp.WithString("path", mcp.Required(), mcp.Description("Full path of directory to create")), + ), toolHandlerWithAuth(handleFsMkdir)) + + // fs_rename + s.AddTool(mcp.NewTool("fs_rename", + mcp.WithDescription("Rename a file or directory"), + mcp.WithString("path", mcp.Required(), mcp.Description("Current path of the file/directory")), + mcp.WithString("name", mcp.Required(), mcp.Description("New name (filename only, not a path)")), + ), toolHandlerWithAuth(handleFsRename)) + + // fs_move + s.AddTool(mcp.NewTool("fs_move", + mcp.WithDescription("Move files/directories to another location"), + mcp.WithString("src_dir", mcp.Required(), mcp.Description("Source directory")), + mcp.WithString("dst_dir", mcp.Required(), mcp.Description("Destination directory")), + mcp.WithArray("names", mcp.Description("Names of files/directories to move")), + ), toolHandlerWithAuth(handleFsMove)) + + // fs_copy + s.AddTool(mcp.NewTool("fs_copy", + mcp.WithDescription("Copy files/directories to another location"), + mcp.WithString("src_dir", mcp.Required(), mcp.Description("Source directory")), + mcp.WithString("dst_dir", mcp.Required(), mcp.Description("Destination directory")), + mcp.WithArray("names", mcp.Description("Names of files/directories to copy")), + ), toolHandlerWithAuth(handleFsCopy)) + + // fs_remove + s.AddTool(mcp.NewTool("fs_remove", + mcp.WithDescription("Delete files/directories"), + mcp.WithString("dir", mcp.Required(), mcp.Description("Directory containing items to remove")), + mcp.WithArray("names", mcp.Description("Names of files/directories to remove")), + ), toolHandlerWithAuth(handleFsRemove)) +} + +func handleFsMkdir(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqPath, common.PermWrite); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.MakeDir(ctx, reqPath); err != nil { + return wrapError(err) + } + return textResult("directory created successfully") +} + +func handleFsRename(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + name, err := req.RequireString("name") + if err != nil { + return toolError("name is required") + } + if err := utils.ValidateNameComponent(name); err != nil { + return toolErrorf("invalid name: %s", err.Error()) + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqPath, common.PermRename); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.Rename(ctx, reqPath, name); err != nil { + return wrapError(err) + } + return textResult("renamed successfully") +} + +func handleFsMove(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + srcDirStr, err := req.RequireString("src_dir") + if err != nil { + return toolError("src_dir is required") + } + dstDirStr, err := req.RequireString("dst_dir") + if err != nil { + return toolError("dst_dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + srcDir, err := user.JoinPath(srcDirStr) + if err != nil { + return wrapError(err) + } + dstDir, err := user.JoinPath(dstDirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, srcDir, common.PermMove); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for i, name := range names { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if err := fs.Move(ctx, srcPath, dstDir, len(names) > i+1); err != nil { + return wrapError(err) + } + } + return textResult("moved successfully") +} + +func handleFsCopy(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + srcDirStr, err := req.RequireString("src_dir") + if err != nil { + return toolError("src_dir is required") + } + dstDirStr, err := req.RequireString("dst_dir") + if err != nil { + return toolError("dst_dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + srcDir, err := user.JoinPath(srcDirStr) + if err != nil { + return wrapError(err) + } + dstDir, err := user.JoinPath(dstDirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, srcDir, common.PermCopy); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for i, name := range names { + srcPath, err := utils.JoinUnderBase(srcDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if _, err := fs.Copy(ctx, srcPath, dstDir, len(names) > i+1); err != nil { + return wrapError(err) + } + } + return textResult("copied successfully") +} + +func handleFsRemove(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + dirStr, err := req.RequireString("dir") + if err != nil { + return toolError("dir is required") + } + names := getStringArray(req, "names") + if len(names) == 0 { + return toolError("names is required and must not be empty") + } + + reqDir, err := user.JoinPath(dirStr) + if err != nil { + return wrapError(err) + } + if err := checkManage(user, reqDir, common.PermRemove); err != nil { + return toolError(err.Error()) + } + + ctx = context.WithValue(ctx, "user", user) + for _, name := range names { + removePath, err := utils.JoinUnderBase(reqDir, name) + if err != nil { + return toolErrorf("invalid name %q: %s", name, err.Error()) + } + if err := fs.Remove(ctx, removePath); err != nil { + return wrapError(err) + } + } + return textResult("removed successfully") +} + +// getStringArray extracts a string array from tool request arguments. +func getStringArray(req mcp.CallToolRequest, name string) []string { + args := req.GetArguments() + val, ok := args[name] + if !ok { + return nil + } + arr, ok := val.([]interface{}) + if !ok { + return nil + } + result := make([]string, 0, len(arr)) + for _, v := range arr { + if s, ok := v.(string); ok { + result = append(result, s) + } + } + return result +} diff --git a/server/mcp/tools_read.go b/server/mcp/tools_read.go new file mode 100644 index 00000000000..d8af570b233 --- /dev/null +++ b/server/mcp/tools_read.go @@ -0,0 +1,207 @@ +package mcp + +import ( + "context" + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/search" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" + "github.com/pkg/errors" +) + +func registerReadTools(s *mcpserver.MCPServer) { + // fs_list + s.AddTool(mcp.NewTool("fs_list", + mcp.WithDescription("List files and directories at the given path"), + mcp.WithString("path", mcp.Required(), mcp.Description("Directory path to list")), + mcp.WithNumber("page", mcp.Description("Page number (default: 1)")), + mcp.WithNumber("per_page", mcp.Description("Items per page (default: 30, max: 500)")), + mcp.WithBoolean("refresh", mcp.Description("Force refresh from storage (default: false)")), + ), toolHandlerWithAuth(handleFsList)) + + // fs_get + s.AddTool(mcp.NewTool("fs_get", + mcp.WithDescription("Get file or directory metadata"), + mcp.WithString("path", mcp.Required(), mcp.Description("Path to file or directory")), + ), toolHandlerWithAuth(handleFsGet)) + + // fs_search + s.AddTool(mcp.NewTool("fs_search", + mcp.WithDescription("Search for files by keywords"), + mcp.WithString("path", mcp.Required(), mcp.Description("Parent directory to search within")), + mcp.WithString("keywords", mcp.Required(), mcp.Description("Search keywords")), + mcp.WithNumber("scope", mcp.Description("0=all, 1=dir only, 2=file only (default: 0)")), + mcp.WithNumber("page", mcp.Description("Page number (default: 1)")), + mcp.WithNumber("per_page", mcp.Description("Items per page (default: 20)")), + ), toolHandlerWithAuth(handleFsSearch)) + + // fs_download_url + s.AddTool(mcp.NewTool("fs_download_url", + mcp.WithDescription("Get download URL for a file"), + mcp.WithString("path", mcp.Required(), mcp.Description("Path to the file")), + ), toolHandlerWithAuth(handleFsDownloadURL)) +} + +func handleFsList(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + page := intParam(req, "page", 1) + perPage := intParam(req, "per_page", 30) + if perPage > 500 { + perPage = 500 + } + refresh := req.GetBool("refresh", false) + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + objs, err := fs.List(ctx, reqPath, &fs.ListArgs{Refresh: refresh}) + if err != nil { + return wrapError(err) + } + + // Paginate + total := len(objs) + start := (page - 1) * perPage + if start > total { + start = total + } + end := start + perPage + if end > total { + end = total + } + pageObjs := objs[start:end] + + items := make([]objJSON, 0, len(pageObjs)) + for _, obj := range pageObjs { + items = append(items, objToJSON(obj)) + } + + return jsonResult(map[string]interface{}{ + "content": items, + "total": total, + "page": page, + "per_page": perPage, + }) +} + +func handleFsGet(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + obj, err := fs.Get(ctx, reqPath, &fs.GetArgs{}) + if err != nil { + return wrapError(err) + } + + return jsonResult(objToJSON(obj)) +} + +func handleFsSearch(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + keywords, err := req.RequireString("keywords") + if err != nil { + return toolError("keywords is required") + } + scope := intParam(req, "scope", 0) + page := intParam(req, "page", 1) + perPage := intParam(req, "per_page", 20) + + parent, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + + searchReq := model.SearchReq{ + Parent: parent, + Keywords: keywords, + Scope: scope, + PageReq: model.PageReq{Page: page, PerPage: perPage}, + } + if err := searchReq.Validate(); err != nil { + return toolErrorf("invalid search request: %s", err.Error()) + } + + nodes, total, err := search.Search(ctx, searchReq) + if err != nil { + return wrapError(err) + } + + // Filter by permission + filtered := make([]model.SearchNode, 0, len(nodes)) + for _, node := range nodes { + if !strings.HasPrefix(node.Parent, user.BasePath) { + continue + } + meta, err := op.GetNearestMeta(node.Parent) + if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) { + continue + } + if !common.CanAccessWithRoles(user, meta, path.Join(node.Parent, node.Name), "") { + continue + } + filtered = append(filtered, node) + } + + return jsonResult(map[string]interface{}{ + "content": filtered, + "total": total, + }) +} + +func handleFsDownloadURL(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + + ctx, reqPath, err := buildFsContext(ctx, user, pathStr) + if err != nil { + return wrapError(err) + } + if err := checkAccess(user, reqPath); err != nil { + return toolError(err.Error()) + } + + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{}) + if err != nil { + return wrapError(err) + } + + return jsonResult(map[string]interface{}{ + "raw_url": link.URL, + }) +} + +// intParam extracts an integer parameter with a default value. +func intParam(req mcp.CallToolRequest, name string, defaultVal int) int { + v := req.GetFloat(name, float64(defaultVal)) + return int(v) +} diff --git a/server/mcp/tools_upload.go b/server/mcp/tools_upload.go new file mode 100644 index 00000000000..c84eb9623a9 --- /dev/null +++ b/server/mcp/tools_upload.go @@ -0,0 +1,131 @@ +package mcp + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + stdpath "path" + "strconv" + "strings" + "time" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/mark3labs/mcp-go/mcp" + mcpserver "github.com/mark3labs/mcp-go/server" +) + +func registerUploadTools(s *mcpserver.MCPServer) { + s.AddTool(mcp.NewTool("fs_upload", + mcp.WithDescription("Upload a local file to alist. Automatically uses direct internal upload (local deployment) or HTTP API upload (remote deployment)."), + mcp.WithString("path", mcp.Required(), mcp.Description("Destination path in alist including filename")), + mcp.WithString("local_path", mcp.Required(), mcp.Description("Absolute local file path to upload")), + ), toolHandlerWithAuth(handleFsUpload)) +} + +func handleFsUpload(ctx context.Context, user *model.User, req mcp.CallToolRequest) (*mcp.CallToolResult, error) { + pathStr, err := req.RequireString("path") + if err != nil { + return toolError("path is required") + } + localPath, err := req.RequireString("local_path") + if err != nil { + return toolError("local_path is required") + } + + reqPath, err := user.JoinPath(pathStr) + if err != nil { + return wrapError(err) + } + dir := stdpath.Dir(reqPath) + + if err := checkManage(user, dir, common.PermWrite); err != nil { + return toolError(err.Error()) + } + + if strings.Contains(conf.Conf.SiteURL, "://") { + return uploadViaHTTP(reqPath, localPath) + } + return uploadDirectly(ctx, user, reqPath, localPath) +} + +func uploadDirectly(ctx context.Context, user *model.User, reqPath, localPath string) (*mcp.CallToolResult, error) { + file, err := os.Open(localPath) + if err != nil { + return toolErrorf("failed to open local file: %s", err.Error()) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return toolErrorf("failed to stat local file: %s", err.Error()) + } + + dir := stdpath.Dir(reqPath) + name := stdpath.Base(reqPath) + + fileStream := &stream.FileStream{ + Ctx: ctx, + Obj: &model.Object{ + Name: name, + Size: info.Size(), + Modified: time.Now(), + IsFolder: false, + }, + Reader: io.NopCloser(file), + Closers: utils.EmptyClosers(), + } + + ctx = context.WithValue(ctx, "user", user) + if err := fs.PutDirectly(ctx, dir, fileStream); err != nil { + return wrapError(err) + } + return textResult("uploaded successfully") +} + +func uploadViaHTTP(reqPath, localPath string) (*mcp.CallToolResult, error) { + file, err := os.Open(localPath) + if err != nil { + return toolErrorf("failed to open local file: %s", err.Error()) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return toolErrorf("failed to stat local file: %s", err.Error()) + } + + name := stdpath.Base(reqPath) + apiURL := fmt.Sprintf("%s/api/fs/put", conf.Conf.SiteURL) + + httpReq, err := http.NewRequest(http.MethodPut, apiURL, file) + if err != nil { + return toolErrorf("failed to create request: %s", err.Error()) + } + + httpReq.Header.Set("File-Path", url.PathEscape(reqPath)) + httpReq.Header.Set("Content-Length", strconv.FormatInt(info.Size(), 10)) + httpReq.Header.Set("Content-Type", utils.GetMimeType(name)) + httpReq.Header.Set("Authorization", setting.GetStr(conf.Token)) + httpReq.ContentLength = info.Size() + + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + return toolErrorf("upload request failed: %s", err.Error()) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return toolErrorf("upload failed (HTTP %d): %s", resp.StatusCode, string(body)) + } + return textResult("uploaded successfully") +} diff --git a/server/middlewares/auth.go b/server/middlewares/auth.go index 14bba5678a1..204b4b7205e 100644 --- a/server/middlewares/auth.go +++ b/server/middlewares/auth.go @@ -2,11 +2,16 @@ package middlewares import ( "crypto/subtle" + "errors" + "fmt" "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -23,7 +28,9 @@ func Auth(c *gin.Context) { c.Abort() return } - c.Set("user", admin) + if !HandleSession(c, admin) { + return + } log.Debugf("use admin token: %+v", admin) c.Next() return @@ -40,7 +47,18 @@ func Auth(c *gin.Context) { c.Abort() return } - c.Set("user", guest) + if len(guest.Role) > 0 { + roles, err := op.GetRolesByUserID(guest.ID) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("Fail to load guest roles: %v", err), 500) + c.Abort() + return + } + guest.RolesDetail = roles + } + if !HandleSession(c, guest) { + return + } log.Debugf("use empty token: %+v", guest) c.Next() return @@ -57,16 +75,56 @@ func Auth(c *gin.Context) { c.Abort() return } + // validate password timestamp + if userClaims.PwdTS != user.PwdTS { + common.ErrorStrResp(c, "Password has been changed, login please", 401) + c.Abort() + return + } if user.Disabled { common.ErrorStrResp(c, "Current user is disabled, replace please", 401) c.Abort() return } - c.Set("user", user) + if len(user.Role) > 0 { + roles, err := op.GetRolesByUserID(user.ID) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("Fail to load roles: %v", err), 500) + c.Abort() + return + } + user.RolesDetail = roles + } + if !HandleSession(c, user) { + return + } log.Debugf("use login token: %+v", user) c.Next() } +// HandleSession verifies device sessions and stores context values. +func HandleSession(c *gin.Context, user *model.User) bool { + clientID := c.GetHeader("Client-Id") + if clientID == "" { + clientID = c.Query("client_id") + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", user.ID, clientID)) + if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + token := c.GetHeader("Authorization") + if errors.Is(err, errs.SessionInactive) { + _ = common.InvalidateToken(token) + common.ErrorResp(c, err, 401) + } else { + common.ErrorResp(c, err, 403) + } + c.Abort() + return false + } + c.Set("device_key", key) + c.Set("user", user) + return true +} + func Authn(c *gin.Context) { token := c.GetHeader("Authorization") if subtle.ConstantTimeCompare([]byte(token), []byte(setting.GetStr(conf.Token))) == 1 { @@ -105,16 +163,45 @@ func Authn(c *gin.Context) { c.Abort() return } + // validate password timestamp + if userClaims.PwdTS != user.PwdTS { + common.ErrorStrResp(c, "Password has been changed, login please", 401) + c.Abort() + return + } if user.Disabled { common.ErrorStrResp(c, "Current user is disabled, replace please", 401) c.Abort() return } + if len(user.Role) > 0 { + var roles []model.Role + for _, roleID := range user.Role { + role, err := op.GetRole(uint(roleID)) + if err != nil { + common.ErrorStrResp(c, fmt.Sprintf("load role %d failed", roleID), 500) + c.Abort() + return + } + roles = append(roles, *role) + } + user.RolesDetail = roles + } c.Set("user", user) log.Debugf("use login token: %+v", user) c.Next() } +func AuthNotGuest(c *gin.Context) { + user := c.MustGet("user").(*model.User) + if user.IsGuest() { + common.ErrorStrResp(c, "You are a guest", 403) + c.Abort() + } else { + c.Next() + } +} + func AuthAdmin(c *gin.Context) { user := c.MustGet("user").(*model.User) if !user.IsAdmin() { diff --git a/server/middlewares/down.go b/server/middlewares/down.go index 05e9dc856d8..7e67663b74b 100644 --- a/server/middlewares/down.go +++ b/server/middlewares/down.go @@ -1,6 +1,7 @@ package middlewares import ( + "net/url" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -9,40 +10,40 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/gin-gonic/gin" "github.com/pkg/errors" ) -func Down(c *gin.Context) { - rawPath := parsePath(c.Param("path")) - c.Set("path", rawPath) - meta, err := op.GetNearestMeta(rawPath) - if err != nil { - if !errors.Is(errors.Cause(err), errs.MetaNotFound) { - common.ErrorResp(c, err, 500, true) - return - } - } - c.Set("meta", meta) - // verify sign - if needSign(meta, rawPath) { - s := c.Query("sign") - err = sign.Verify(rawPath, strings.TrimSuffix(s, "/")) +func Down(verifyFunc func(string, string) error) func(c *gin.Context) { + return func(c *gin.Context) { + rawPath := parsePath(c.Param("path")) + c.Set("path", rawPath) + meta, err := op.GetNearestMeta(rawPath) if err != nil { - common.ErrorResp(c, err, 401) - c.Abort() - return + if !errors.Is(errors.Cause(err), errs.MetaNotFound) { + common.ErrorResp(c, err, 500, true) + return + } + } + c.Set("meta", meta) + // verify sign + if needSign(meta, rawPath) { + s := c.Query("sign") + err = verifyFunc(rawPath, strings.TrimSuffix(s, "/")) + if err != nil { + common.ErrorResp(c, err, 401) + c.Abort() + return + } } + c.Next() } - c.Next() } -// TODO: implement -// path maybe contains # ? etc. func parsePath(path string) string { + path, _ = url.PathUnescape(path) return utils.FixAndCleanPath(path) } diff --git a/server/middlewares/fsup.go b/server/middlewares/fsup.go index 2aa7fca6d03..243c22e4131 100644 --- a/server/middlewares/fsup.go +++ b/server/middlewares/fsup.go @@ -35,7 +35,9 @@ func FsUp(c *gin.Context) { return } } - if !(common.CanAccess(user, meta, path, password) && (user.CanWrite() || common.CanWrite(meta, stdpath.Dir(path)))) { + perm := common.MergeRolePermissions(user, path) + if !(common.CanAccessWithRoles(user, meta, path, password) && + (common.HasPermission(perm, common.PermWrite) || common.CanWrite(meta, stdpath.Dir(path)))) { common.ErrorResp(c, errs.PermissionDenied, 403) c.Abort() return diff --git a/server/middlewares/limit.go b/server/middlewares/limit.go index 44c079b37e0..2ccee950c80 100644 --- a/server/middlewares/limit.go +++ b/server/middlewares/limit.go @@ -1,7 +1,9 @@ package middlewares import ( + "github.com/alist-org/alist/v3/internal/stream" "github.com/gin-gonic/gin" + "io" ) func MaxAllowed(n int) gin.HandlerFunc { @@ -14,3 +16,37 @@ func MaxAllowed(n int) gin.HandlerFunc { c.Next() } } + +func UploadRateLimiter(limiter stream.Limiter) gin.HandlerFunc { + return func(c *gin.Context) { + c.Request.Body = &stream.RateLimitReader{ + Reader: c.Request.Body, + Limiter: limiter, + Ctx: c, + } + c.Next() + } +} + +type ResponseWriterWrapper struct { + gin.ResponseWriter + WrapWriter io.Writer +} + +func (w *ResponseWriterWrapper) Write(p []byte) (n int, err error) { + return w.WrapWriter.Write(p) +} + +func DownloadRateLimiter(limiter stream.Limiter) gin.HandlerFunc { + return func(c *gin.Context) { + c.Writer = &ResponseWriterWrapper{ + ResponseWriter: c.Writer, + WrapWriter: &stream.RateLimitWriter{ + Writer: c.Writer, + Limiter: limiter, + Ctx: c, + }, + } + c.Next() + } +} diff --git a/server/middlewares/session_refresh.go b/server/middlewares/session_refresh.go new file mode 100644 index 00000000000..2073020ce79 --- /dev/null +++ b/server/middlewares/session_refresh.go @@ -0,0 +1,26 @@ +package middlewares + +import ( + "github.com/alist-org/alist/v3/internal/device" + "github.com/alist-org/alist/v3/internal/model" + "github.com/gin-gonic/gin" +) + +// SessionRefresh updates session's last_active after successful requests. +func SessionRefresh(c *gin.Context) { + c.Next() + if c.Writer.Status() >= 400 { + return + } + if inactive, ok := c.Get("session_inactive"); ok { + if b, ok := inactive.(bool); ok && b { + return + } + } + userVal, uok := c.Get("user") + keyVal, kok := c.Get("device_key") + if uok && kok { + user := userVal.(*model.User) + device.Refresh(user.ID, keyVal.(string)) + } +} diff --git a/server/router.go b/server/router.go index 92ede88bfde..e42d09472d1 100644 --- a/server/router.go +++ b/server/router.go @@ -4,6 +4,8 @@ import ( "github.com/alist-org/alist/v3/cmd/flags" "github.com/alist-org/alist/v3/internal/conf" "github.com/alist-org/alist/v3/internal/message" + "github.com/alist-org/alist/v3/internal/sign" + "github.com/alist-org/alist/v3/internal/stream" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/handles" @@ -20,6 +22,7 @@ func Init(e *gin.Engine) { }) } Cors(e) + e.Use(middlewares.SessionRefresh) g := e.Group(conf.URL.Path) if conf.Conf.Scheme.HttpPort != -1 && conf.Conf.Scheme.HttpsPort != -1 && conf.Conf.Scheme.ForceHttps { e.Use(middlewares.ForceHttps) @@ -36,11 +39,31 @@ func Init(e *gin.Engine) { g.Use(middlewares.MaxAllowed(conf.Conf.MaxConnections)) } WebDav(g.Group("/dav")) + S3(g.Group("/s3")) - g.GET("/d/*path", middlewares.Down, handles.Down) - g.GET("/p/*path", middlewares.Down, handles.Proxy) - g.HEAD("/d/*path", middlewares.Down, handles.Down) - g.HEAD("/p/*path", middlewares.Down, handles.Proxy) + downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) + signCheck := middlewares.Down(sign.Verify) + g.GET("/d/*path", signCheck, downloadLimiter, handles.Down) + g.GET("/p/*path", signCheck, downloadLimiter, handles.Proxy) + g.HEAD("/d/*path", signCheck, handles.Down) + g.HEAD("/p/*path", signCheck, handles.Proxy) + g.GET("/s/:share_id", handles.GetSharePage) + g.GET("/s/:share_id/*path", handles.GetSharePage) + g.GET("/sd/:share_id", downloadLimiter, handles.ShareDown) + g.GET("/sd/:share_id/*path", downloadLimiter, handles.ShareDown) + g.HEAD("/sd/:share_id", handles.ShareDown) + g.HEAD("/sd/:share_id/*path", handles.ShareDown) + g.GET("/sp/:share_id", downloadLimiter, handles.ShareProxy) + g.GET("/sp/:share_id/*path", downloadLimiter, handles.ShareProxy) + g.HEAD("/sp/:share_id", handles.ShareProxy) + g.HEAD("/sp/:share_id/*path", handles.ShareProxy) + archiveSignCheck := middlewares.Down(sign.VerifyArchive) + g.GET("/ad/*path", archiveSignCheck, downloadLimiter, handles.ArchiveDown) + g.GET("/ap/*path", archiveSignCheck, downloadLimiter, handles.ArchiveProxy) + g.GET("/ae/*path", archiveSignCheck, downloadLimiter, handles.ArchiveInternalExtract) + g.HEAD("/ad/*path", archiveSignCheck, handles.ArchiveDown) + g.HEAD("/ap/*path", archiveSignCheck, handles.ArchiveProxy) + g.HEAD("/ae/*path", archiveSignCheck, handles.ArchiveInternalExtract) api := g.Group("/api") auth := api.Group("", middlewares.Auth) @@ -48,10 +71,18 @@ func Init(e *gin.Engine) { api.POST("/auth/login", handles.Login) api.POST("/auth/login/hash", handles.LoginHash) + api.POST("/auth/login/ldap", handles.LoginLdap) + api.POST("/auth/register", handles.Register) auth.GET("/me", handles.CurrentUser) auth.POST("/me/update", handles.UpdateCurrent) + auth.GET("/me/sshkey/list", handles.ListMyPublicKey) + auth.POST("/me/sshkey/add", handles.AddMyPublicKey) + auth.POST("/me/sshkey/delete", handles.DeleteMyPublicKey) auth.POST("/auth/2fa/generate", handles.Generate2FA) auth.POST("/auth/2fa/verify", handles.Verify2FA) + auth.GET("/auth/logout", handles.LogOut) + auth.GET("/me/sessions", handles.ListMySessions) + auth.POST("/me/sessions/evict", handles.EvictMySession) // auth api.GET("/auth/sso", handles.SSOLoginRedirect) @@ -59,19 +90,34 @@ func Init(e *gin.Engine) { api.GET("/auth/get_sso_id", handles.SSOLoginCallback) api.GET("/auth/sso_get_token", handles.SSOLoginCallback) - //webauthn + // webauthn + api.GET("/authn/webauthn_begin_login", handles.BeginAuthnLogin) + api.POST("/authn/webauthn_finish_login", handles.FinishAuthnLogin) webauthn.GET("/webauthn_begin_registration", handles.BeginAuthnRegistration) webauthn.POST("/webauthn_finish_registration", handles.FinishAuthnRegistration) - webauthn.GET("/webauthn_begin_login", handles.BeginAuthnLogin) - webauthn.POST("/webauthn_finish_login", handles.FinishAuthnLogin) webauthn.POST("/delete_authn", handles.DeleteAuthnLogin) webauthn.GET("/getcredentials", handles.GetAuthnCredentials) // no need auth public := api.Group("/public") public.Any("/settings", handles.PublicSettings) + public.Any("/offline_download_tools", handles.OfflineDownloadTools) + public.Any("/archive_extensions", handles.ArchiveExtensions) + public.GET("/share/info", handles.GetPublicShareInfo) + public.POST("/share/auth", handles.AuthPublicShare) + public.POST("/share/list", handles.ListPublicShare) + public.POST("/share/get", handles.GetPublicShare) _fs(auth.Group("/fs")) + share := auth.Group("/share", middlewares.AuthNotGuest) + share.POST("/create", handles.CreateShare) + share.POST("/update", handles.UpdateShare) + share.POST("/disable", handles.DisableShare) + share.GET("/list", handles.ListShares) + share.POST("/delete", handles.DeleteShare) + _task(auth.Group("/task", middlewares.AuthNotGuest)) + _label(auth.Group("/label")) + _labelFileBinding(auth.Group("/label_file_binding")) admin(auth.Group("/admin", middlewares.AuthAdmin)) if flags.Debug || flags.Dev { debug(g.Group("/debug")) @@ -97,6 +143,15 @@ func admin(g *gin.RouterGroup) { user.POST("/cancel_2fa", handles.Cancel2FAById) user.POST("/delete", handles.DeleteUser) user.POST("/del_cache", handles.DelUserCache) + user.GET("/sshkey/list", handles.ListPublicKeys) + user.POST("/sshkey/delete", handles.DeletePublicKey) + + role := g.Group("/role") + role.GET("/list", handles.ListRoles) + role.GET("/get", handles.GetRole) + role.POST("/create", handles.CreateRole) + role.POST("/update", handles.UpdateRole) + role.POST("/delete", handles.DeleteRole) storage := g.Group("/storage") storage.GET("/list", handles.ListStorages) @@ -119,11 +174,20 @@ func admin(g *gin.RouterGroup) { setting.POST("/save", handles.SaveSettings) setting.POST("/delete", handles.DeleteSetting) setting.POST("/reset_token", handles.ResetToken) + setting.POST("/set_token", handles.SetToken) setting.POST("/set_aria2", handles.SetAria2) setting.POST("/set_qbit", handles.SetQbittorrent) + setting.POST("/set_transmission", handles.SetTransmission) + setting.POST("/set_115", handles.Set115) + setting.POST("/set_pikpak", handles.SetPikPak) + setting.POST("/set_thunder", handles.SetThunder) + setting.POST("/set_guangyapan", handles.SetGuangYaPan) + setting.POST("/set_frp", handles.SetFRP) + setting.POST("/stop_frp", handles.StopFRP) + setting.GET("/frp_runtime", handles.GetFRPRuntime) - task := g.Group("/task") - handles.SetupTaskRoute(task) + // retain /admin/task API to ensure compatibility with legacy automation scripts + _task(g.Group("/task")) ms := g.Group("/message") ms.POST("/get", message.HttpInstance.GetHandle) @@ -135,6 +199,23 @@ func admin(g *gin.RouterGroup) { index.POST("/stop", middlewares.SearchIndex, handles.StopIndex) index.POST("/clear", middlewares.SearchIndex, handles.ClearIndex) index.GET("/progress", middlewares.SearchIndex, handles.GetProgress) + + label := g.Group("/label") + label.POST("/create", handles.CreateLabel) + label.POST("/update", handles.UpdateLabel) + label.POST("/delete", handles.DeleteLabel) + + labelFileBinding := g.Group("/label_file_binding") + labelFileBinding.GET("/list", handles.ListLabelFileBinding) + labelFileBinding.POST("/create", handles.CreateLabelFileBinDing) + labelFileBinding.POST("/create_batch", handles.CreateLabelFileBinDingBatch) + labelFileBinding.POST("/delete", handles.DelLabelByFileName) + labelFileBinding.POST("/restore", handles.RestoreLabelFileBinding) + + session := g.Group("/session") + session.GET("/list", handles.ListSessions) + session.POST("/evict", handles.EvictSession) + } func _fs(g *gin.RouterGroup) { @@ -142,6 +223,7 @@ func _fs(g *gin.RouterGroup) { g.Any("/search", middlewares.SearchIndex, handles.Search) g.Any("/get", handles.FsGet) g.Any("/other", handles.FsOther) + g.GET("/lark/export/download", handles.LarkExportDownload) g.Any("/dirs", handles.FsDirs) g.POST("/mkdir", handles.FsMkdir) g.POST("/rename", handles.FsRename) @@ -152,17 +234,44 @@ func _fs(g *gin.RouterGroup) { g.POST("/copy", handles.FsCopy) g.POST("/remove", handles.FsRemove) g.POST("/remove_empty_directory", handles.FsRemoveEmptyDirectory) - g.PUT("/put", middlewares.FsUp, handles.FsStream) - g.PUT("/form", middlewares.FsUp, handles.FsForm) + uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit) + g.PUT("/put", middlewares.FsUp, uploadLimiter, handles.FsStream) + g.PUT("/form", middlewares.FsUp, uploadLimiter, handles.FsForm) g.POST("/link", middlewares.AuthAdmin, handles.Link) - g.POST("/add_aria2", handles.AddAria2) - g.POST("/add_qbit", handles.AddQbittorrent) + // g.POST("/add_aria2", handles.AddOfflineDownload) + // g.POST("/add_qbit", handles.AddQbittorrent) + // g.POST("/add_transmission", handles.SetTransmission) + g.POST("/add_offline_download", handles.AddOfflineDownload) + a := g.Group("/archive") + a.Any("/meta", handles.FsArchiveMeta) + a.Any("/list", handles.FsArchiveList) + a.POST("/decompress", handles.FsArchiveDecompress) +} + +func _task(g *gin.RouterGroup) { + handles.SetupTaskRoute(g) +} + +func _label(g *gin.RouterGroup) { + g.GET("/list", handles.ListLabel) + g.GET("/get", handles.GetLabel) +} + +func _labelFileBinding(g *gin.RouterGroup) { + g.GET("/get", handles.GetLabelByFileName) + g.GET("/get_file_by_label", handles.GetFileByLabel) } func Cors(r *gin.Engine) { config := cors.DefaultConfig() - config.AllowAllOrigins = true - config.AllowHeaders = []string{"*"} - config.AllowMethods = []string{"*"} + // config.AllowAllOrigins = true + config.AllowOrigins = conf.Conf.Cors.AllowOrigins + config.AllowHeaders = conf.Conf.Cors.AllowHeaders + config.AllowMethods = conf.Conf.Cors.AllowMethods r.Use(cors.New(config)) } + +func InitS3(e *gin.Engine) { + Cors(e) + S3Server(e.Group("/")) +} diff --git a/server/s3.go b/server/s3.go new file mode 100644 index 00000000000..21b95527ded --- /dev/null +++ b/server/s3.go @@ -0,0 +1,39 @@ +package server + +import ( + "context" + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/server/common" + "github.com/alist-org/alist/v3/server/s3" + "github.com/gin-gonic/gin" +) + +func S3(g *gin.RouterGroup) { + if !conf.Conf.S3.Enable { + g.Any("/*path", func(c *gin.Context) { + common.ErrorStrResp(c, "S3 server is not enabled", 403) + }) + return + } + if conf.Conf.S3.Port != -1 { + g.Any("/*path", func(c *gin.Context) { + common.ErrorStrResp(c, "S3 server bound to single port", 403) + }) + return + } + h, _ := s3.NewServer(context.Background()) + + g.Any("/*path", func(c *gin.Context) { + adjustedPath := strings.TrimPrefix(c.Request.URL.Path, path.Join(conf.URL.Path, "/s3")) + c.Request.URL.Path = adjustedPath + gin.WrapH(h)(c) + }) +} + +func S3Server(g *gin.RouterGroup) { + h, _ := s3.NewServer(context.Background()) + g.Any("/*path", gin.WrapH(h)) +} diff --git a/server/s3/backend.go b/server/s3/backend.go new file mode 100644 index 00000000000..a1e990441be --- /dev/null +++ b/server/s3/backend.go @@ -0,0 +1,439 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "context" + "encoding/hex" + "fmt" + "io" + "path" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/gofakes3" + "github.com/ncw/swift/v2" + log "github.com/sirupsen/logrus" +) + +var ( + emptyPrefix = &gofakes3.Prefix{} + timeFormat = "Mon, 2 Jan 2006 15:04:05 GMT" +) + +// s3Backend implements the gofacess3.Backend interface to make an S3 +// backend for gofakes3 +type s3Backend struct { + meta *sync.Map +} + +// newBackend creates a new SimpleBucketBackend. +func newBackend() gofakes3.Backend { + return &s3Backend{ + meta: new(sync.Map), + } +} + +// ListBuckets always returns the default bucket. +func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, error) { + buckets, err := getAndParseBuckets() + if err != nil { + return nil, err + } + var response []gofakes3.BucketInfo + for _, b := range buckets { + node, _ := fs.Get(ctx, b.Path, &fs.GetArgs{}) + response = append(response, gofakes3.BucketInfo{ + // Name: gofakes3.URLEncode(b.Name), + Name: b.Name, + CreationDate: gofakes3.NewContentTime(node.ModTime()), + }) + } + return response, nil +} + +// ListBucket lists the objects in the given bucket. +func (b *s3Backend) ListBucket(ctx context.Context, bucketName string, prefix *gofakes3.Prefix, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { + bucket, err := getBucketByName(bucketName) + if err != nil { + return nil, err + } + bucketPath := bucket.Path + + if prefix == nil { + prefix = emptyPrefix + } + + // workaround + if strings.TrimSpace(prefix.Prefix) == "" { + prefix.HasPrefix = false + } + if strings.TrimSpace(prefix.Delimiter) == "" { + prefix.HasDelimiter = false + } + + response := gofakes3.NewObjectList() + path, remaining := prefixParser(prefix) + + err = b.entryListR(bucketPath, path, remaining, prefix.HasDelimiter, response) + if err == gofakes3.ErrNoSuchKey { + // AWS just returns an empty list + response = gofakes3.NewObjectList() + } else if err != nil { + return nil, err + } + + return b.pager(response, page) +} + +// HeadObject returns the fileinfo for the given object name. +// +// Note that the metadata is not supported yet. +func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName string) (*gofakes3.Object, error) { + bucket, err := getBucketByName(bucketName) + if err != nil { + return nil, err + } + bucketPath := bucket.Path + + fp := path.Join(bucketPath, objectName) + fmeta, _ := op.GetNearestMeta(fp) + node, err := fs.Get(context.WithValue(ctx, "meta", fmeta), fp, &fs.GetArgs{}) + if err != nil { + return nil, gofakes3.KeyNotFound(objectName) + } + + if node.IsDir() { + return nil, gofakes3.KeyNotFound(objectName) + } + + size := node.GetSize() + // hash := getFileHashByte(fobj) + + meta := map[string]string{ + "Last-Modified": node.ModTime().Format(timeFormat), + "Content-Type": utils.GetMimeType(fp), + } + + if val, ok := b.meta.Load(fp); ok { + metaMap := val.(map[string]string) + for k, v := range metaMap { + meta[k] = v + } + } + + return &gofakes3.Object{ + Name: objectName, + // Hash: hash, + Metadata: meta, + Size: size, + Contents: noOpReadCloser{}, + }, nil +} + +// GetObject fetchs the object from the filesystem. +func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string, rangeRequest *gofakes3.ObjectRangeRequest) (obj *gofakes3.Object, err error) { + bucket, err := getBucketByName(bucketName) + if err != nil { + return nil, err + } + bucketPath := bucket.Path + + fp := path.Join(bucketPath, objectName) + fmeta, _ := op.GetNearestMeta(fp) + node, err := fs.Get(context.WithValue(ctx, "meta", fmeta), fp, &fs.GetArgs{}) + if err != nil { + return nil, gofakes3.KeyNotFound(objectName) + } + + if node.IsDir() { + return nil, gofakes3.KeyNotFound(objectName) + } + + link, file, err := fs.Link(ctx, fp, model.LinkArgs{}) + if err != nil { + return nil, err + } + + size := file.GetSize() + rnge, err := rangeRequest.Range(size) + if err != nil { + return nil, err + } + + if link.RangeReadCloser == nil && link.MFile == nil && len(link.URL) == 0 { + return nil, fmt.Errorf("the remote storage driver need to be enhanced to support s3") + } + + var rdr io.ReadCloser + length := int64(-1) + start := int64(0) + if rnge != nil { + start, length = rnge.Start, rnge.Length + } + // 参考 server/common/proxy.go + if link.MFile != nil { + _, err := link.MFile.Seek(start, io.SeekStart) + if err != nil { + return nil, err + } + rdr = link.MFile + } else { + remoteFileSize := file.GetSize() + if length >= 0 && start+length >= remoteFileSize { + length = -1 + } + rrc := link.RangeReadCloser + if len(link.URL) > 0 { + var converted, err = stream.GetRangeReadCloserFromLink(remoteFileSize, link) + if err != nil { + return nil, err + } + rrc = converted + } + if rrc != nil { + remoteReader, err := rrc.RangeRead(ctx, http_range.Range{Start: start, Length: length}) + if err != nil { + return nil, err + } + rdr = utils.ReadCloser{Reader: remoteReader, Closer: rrc} + } else { + return nil, errs.NotSupport + } + } + + meta := map[string]string{ + "Last-Modified": node.ModTime().Format(timeFormat), + "Content-Type": utils.GetMimeType(fp), + } + + if val, ok := b.meta.Load(fp); ok { + metaMap := val.(map[string]string) + for k, v := range metaMap { + meta[k] = v + } + } + + return &gofakes3.Object{ + // Name: gofakes3.URLEncode(objectName), + Name: objectName, + // Hash: "", + Metadata: meta, + Size: size, + Range: rnge, + Contents: rdr, + }, nil +} + +// TouchObject creates or updates meta on specified object. +func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) { + //TODO: implement + return result, gofakes3.ErrNotImplemented +} + +// PutObject creates or overwrites the object with the given name. +func (b *s3Backend) PutObject( + ctx context.Context, bucketName, objectName string, + meta map[string]string, + input io.Reader, size int64, +) (result gofakes3.PutObjectResult, err error) { + bucket, err := getBucketByName(bucketName) + if err != nil { + return result, err + } + bucketPath := bucket.Path + + isDir := strings.HasSuffix(objectName, "/") + log.Debugf("isDir: %v", isDir) + + fp := path.Join(bucketPath, objectName) + log.Debugf("fp: %s, bucketPath: %s, objectName: %s", fp, bucketPath, objectName) + + var reqPath string + if isDir { + reqPath = fp + "/" + } else { + reqPath = path.Dir(fp) + } + log.Debugf("reqPath: %s", reqPath) + fmeta, _ := op.GetNearestMeta(fp) + ctx = context.WithValue(ctx, "meta", fmeta) + + _, err = fs.Get(ctx, reqPath, &fs.GetArgs{}) + if err != nil { + if errs.IsObjectNotFound(err) && strings.Contains(objectName, "/") { + log.Debugf("reqPath: %s not found and objectName contains /, need to makeDir", reqPath) + err = fs.MakeDir(ctx, reqPath, true) + if err != nil { + return result, errors.WithMessagef(err, "failed to makeDir, reqPath: %s", reqPath) + } + } else { + return result, gofakes3.KeyNotFound(objectName) + } + } + + if isDir { + return result, nil + } + + var ti time.Time + + if val, ok := meta["X-Amz-Meta-Mtime"]; ok { + ti, _ = swift.FloatStringToTime(val) + } + + if val, ok := meta["mtime"]; ok { + ti, _ = swift.FloatStringToTime(val) + } + + obj := model.Object{ + Name: path.Base(fp), + Size: size, + Modified: ti, + Ctime: time.Now(), + } + stream := &stream.FileStream{ + Obj: &obj, + Reader: input, + Mimetype: meta["Content-Type"], + } + + err = fs.PutDirectly(ctx, reqPath, stream) + if err != nil { + return result, err + } + + if err := stream.Close(); err != nil { + // remove file when close error occurred (FsPutErr) + _ = fs.Remove(ctx, fp) + return result, err + } + + b.meta.Store(fp, meta) + + return result, nil +} + +// DeleteMulti deletes multiple objects in a single request. +func (b *s3Backend) DeleteMulti(ctx context.Context, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) { + for _, object := range objects { + if err := b.deleteObject(ctx, bucketName, object); err != nil { + utils.Log.Errorf("serve s3", "delete object failed: %v", err) + result.Error = append(result.Error, gofakes3.ErrorResult{ + Code: gofakes3.ErrInternal, + Message: gofakes3.ErrInternal.Message(), + Key: object, + }) + } else { + result.Deleted = append(result.Deleted, gofakes3.ObjectID{ + Key: object, + }) + } + } + + return result, nil +} + +// DeleteObject deletes the object with the given name. +func (b *s3Backend) DeleteObject(ctx context.Context, bucketName, objectName string) (result gofakes3.ObjectDeleteResult, rerr error) { + return result, b.deleteObject(ctx, bucketName, objectName) +} + +// deleteObject deletes the object from the filesystem. +func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName string) error { + bucket, err := getBucketByName(bucketName) + if err != nil { + return err + } + bucketPath := bucket.Path + + fp := path.Join(bucketPath, objectName) + fmeta, _ := op.GetNearestMeta(fp) + // S3 does not report an error when attemping to delete a key that does not exist, so + // we need to skip IsNotExist errors. + if _, err := fs.Get(context.WithValue(ctx, "meta", fmeta), fp, &fs.GetArgs{}); err != nil && !errs.IsObjectNotFound(err) { + return err + } + + fs.Remove(ctx, fp) + return nil +} + +// CreateBucket creates a new bucket. +func (b *s3Backend) CreateBucket(ctx context.Context, name string) error { + return gofakes3.ErrNotImplemented +} + +// DeleteBucket deletes the bucket with the given name. +func (b *s3Backend) DeleteBucket(ctx context.Context, name string) error { + return gofakes3.ErrNotImplemented +} + +// BucketExists checks if the bucket exists. +func (b *s3Backend) BucketExists(ctx context.Context, name string) (exists bool, err error) { + buckets, err := getAndParseBuckets() + if err != nil { + return false, err + } + for _, b := range buckets { + if b.Name == name { + return true, nil + } + } + return false, nil +} + +// CopyObject copy specified object from srcKey to dstKey. +func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket, dstKey string, meta map[string]string) (result gofakes3.CopyObjectResult, err error) { + if srcBucket == dstBucket && srcKey == dstKey { + //TODO: update meta + return result, nil + } + + srcB, err := getBucketByName(srcBucket) + if err != nil { + return result, err + } + srcBucketPath := srcB.Path + + srcFp := path.Join(srcBucketPath, srcKey) + fmeta, _ := op.GetNearestMeta(srcFp) + srcNode, err := fs.Get(context.WithValue(ctx, "meta", fmeta), srcFp, &fs.GetArgs{}) + + c, err := b.GetObject(ctx, srcBucket, srcKey, nil) + if err != nil { + return + } + defer func() { + _ = c.Contents.Close() + }() + + for k, v := range c.Metadata { + if _, found := meta[k]; !found && k != "X-Amz-Acl" { + meta[k] = v + } + } + if _, ok := meta["mtime"]; !ok { + meta["mtime"] = swift.TimeToFloatString(srcNode.ModTime()) + } + + _, err = b.PutObject(ctx, dstBucket, dstKey, meta, c.Contents, c.Size) + if err != nil { + return + } + + return gofakes3.CopyObjectResult{ + ETag: `"` + hex.EncodeToString(c.Hash) + `"`, + LastModified: gofakes3.NewContentTime(srcNode.ModTime()), + }, nil +} diff --git a/server/s3/ioutils.go b/server/s3/ioutils.go new file mode 100644 index 00000000000..6b49cacc708 --- /dev/null +++ b/server/s3/ioutils.go @@ -0,0 +1,36 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import "io" + +type noOpReadCloser struct{} + +type readerWithCloser struct { + io.Reader + closer func() error +} + +var _ io.ReadCloser = &readerWithCloser{} + +func (d noOpReadCloser) Read(b []byte) (n int, err error) { + return 0, io.EOF +} + +func (d noOpReadCloser) Close() error { + return nil +} + +func limitReadCloser(rdr io.Reader, closer func() error, sz int64) io.ReadCloser { + return &readerWithCloser{ + Reader: io.LimitReader(rdr, sz), + closer: closer, + } +} + +func (rwc *readerWithCloser) Close() error { + if rwc.closer != nil { + return rwc.closer() + } + return nil +} diff --git a/server/s3/list.go b/server/s3/list.go new file mode 100644 index 00000000000..40a9e8ab356 --- /dev/null +++ b/server/s3/list.go @@ -0,0 +1,53 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "path" + "strings" + + "github.com/alist-org/gofakes3" +) + +func (b *s3Backend) entryListR(bucket, fdPath, name string, addPrefix bool, response *gofakes3.ObjectList) error { + fp := path.Join(bucket, fdPath) + + dirEntries, err := getDirEntries(fp) + if err != nil { + return err + } + + for _, entry := range dirEntries { + object := entry.GetName() + + // workround for control-chars detect + objectPath := path.Join(fdPath, object) + + if !strings.HasPrefix(object, name) { + continue + } + + if entry.IsDir() { + if addPrefix { + // response.AddPrefix(gofakes3.URLEncode(objectPath)) + response.AddPrefix(objectPath) + continue + } + err := b.entryListR(bucket, path.Join(fdPath, object), "", false, response) + if err != nil { + return err + } + } else { + item := &gofakes3.Content{ + // Key: gofakes3.URLEncode(objectPath), + Key: objectPath, + LastModified: gofakes3.NewContentTime(entry.ModTime()), + ETag: getFileHash(entry), + Size: entry.GetSize(), + StorageClass: gofakes3.StorageStandard, + } + response.Add(item) + } + } + return nil +} diff --git a/server/s3/logger.go b/server/s3/logger.go new file mode 100644 index 00000000000..798734c341d --- /dev/null +++ b/server/s3/logger.go @@ -0,0 +1,27 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "fmt" + + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/gofakes3" +) + +// logger output formatted message +type logger struct{} + +// print log message +func (l logger) Print(level gofakes3.LogLevel, v ...interface{}) { + switch level { + default: + fallthrough + case gofakes3.LogErr: + utils.Log.Errorf("serve s3: %s", fmt.Sprintln(v...)) + case gofakes3.LogWarn: + utils.Log.Infof("serve s3: %s", fmt.Sprintln(v...)) + case gofakes3.LogInfo: + utils.Log.Debugf("serve s3: %s", fmt.Sprintln(v...)) + } +} diff --git a/server/s3/pager.go b/server/s3/pager.go new file mode 100644 index 00000000000..27dd5c9a5cd --- /dev/null +++ b/server/s3/pager.go @@ -0,0 +1,67 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "sort" + + "github.com/alist-org/gofakes3" +) + +// pager splits the object list into smulitply pages. +func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) { + // sort by alphabet + sort.Slice(list.CommonPrefixes, func(i, j int) bool { + return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix + }) + // sort by modtime + sort.Slice(list.Contents, func(i, j int) bool { + return list.Contents[i].LastModified.Before(list.Contents[j].LastModified.Time) + }) + tokens := page.MaxKeys + if tokens == 0 { + tokens = 1000 + } + if page.HasMarker { + for i, obj := range list.Contents { + if obj.Key == page.Marker { + list.Contents = list.Contents[i+1:] + break + } + } + for i, obj := range list.CommonPrefixes { + if obj.Prefix == page.Marker { + list.CommonPrefixes = list.CommonPrefixes[i+1:] + break + } + } + } + + response := gofakes3.NewObjectList() + for _, obj := range list.CommonPrefixes { + if tokens <= 0 { + break + } + response.AddPrefix(obj.Prefix) + tokens-- + } + + for _, obj := range list.Contents { + if tokens <= 0 { + break + } + response.Add(obj) + tokens-- + } + + if len(list.CommonPrefixes)+len(list.Contents) > int(page.MaxKeys) { + response.IsTruncated = true + if len(response.Contents) > 0 { + response.NextMarker = response.Contents[len(response.Contents)-1].Key + } else { + response.NextMarker = response.CommonPrefixes[len(response.CommonPrefixes)-1].Prefix + } + } + + return response, nil +} diff --git a/server/s3/server.go b/server/s3/server.go new file mode 100644 index 00000000000..2f7d15c0804 --- /dev/null +++ b/server/s3/server.go @@ -0,0 +1,27 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "context" + "math/rand" + "net/http" + + "github.com/alist-org/gofakes3" +) + +// Make a new S3 Server to serve the remote +func NewServer(ctx context.Context) (h http.Handler, err error) { + var newLogger logger + faker := gofakes3.New( + newBackend(), + // gofakes3.WithHostBucket(!opt.pathBucketMode), + gofakes3.WithLogger(newLogger), + gofakes3.WithRequestID(rand.Uint64()), + gofakes3.WithoutVersioning(), + gofakes3.WithV4Auth(authlistResolver()), + gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied + ) + + return faker.Server(), nil +} diff --git a/server/s3/utils.go b/server/s3/utils.go new file mode 100644 index 00000000000..6636835a62b --- /dev/null +++ b/server/s3/utils.go @@ -0,0 +1,160 @@ +// Credits: https://pkg.go.dev/github.com/rclone/rclone@v1.65.2/cmd/serve/s3 +// Package s3 implements a fake s3 server for alist +package s3 + +import ( + "context" + "encoding/json" + "strings" + + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/fs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/gofakes3" +) + +type Bucket struct { + Name string `json:"name"` + Path string `json:"path"` +} + +func getAndParseBuckets() ([]Bucket, error) { + var res []Bucket + err := json.Unmarshal([]byte(setting.GetStr(conf.S3Buckets)), &res) + return res, err +} + +func getBucketByName(name string) (Bucket, error) { + buckets, err := getAndParseBuckets() + if err != nil { + return Bucket{}, err + } + for _, b := range buckets { + if b.Name == name { + return b, nil + } + } + return Bucket{}, gofakes3.BucketNotFound(name) +} + +func getDirEntries(path string) ([]model.Obj, error) { + ctx := context.Background() + meta, _ := op.GetNearestMeta(path) + fi, err := fs.Get(context.WithValue(ctx, "meta", meta), path, &fs.GetArgs{}) + if errs.IsNotFoundError(err) { + return nil, gofakes3.ErrNoSuchKey + } else if err != nil { + return nil, gofakes3.ErrNoSuchKey + } + + if !fi.IsDir() { + return nil, gofakes3.ErrNoSuchKey + } + + dirEntries, err := fs.List(context.WithValue(ctx, "meta", meta), path, &fs.ListArgs{}) + if err != nil { + return nil, err + } + + return dirEntries, nil +} + +// func getFileHashByte(node interface{}) []byte { +// b, err := hex.DecodeString(getFileHash(node)) +// if err != nil { +// return nil +// } +// return b +// } + +func getFileHash(node interface{}) string { + // var o fs.Object + + // switch b := node.(type) { + // case vfs.Node: + // fsObj, ok := b.DirEntry().(fs.Object) + // if !ok { + // fs.Debugf("serve s3", "File uploading - reading hash from VFS cache") + // in, err := b.Open(os.O_RDONLY) + // if err != nil { + // return "" + // } + // defer func() { + // _ = in.Close() + // }() + // h, err := hash.NewMultiHasherTypes(hash.NewHashSet(hash.MD5)) + // if err != nil { + // return "" + // } + // _, err = io.Copy(h, in) + // if err != nil { + // return "" + // } + // return h.Sums()[hash.MD5] + // } + // o = fsObj + // case fs.Object: + // o = b + // } + + // hash, err := o.Hash(context.Background(), hash.MD5) + // if err != nil { + // return "" + // } + // return hash + return "" +} + +func prefixParser(p *gofakes3.Prefix) (path, remaining string) { + idx := strings.LastIndexByte(p.Prefix, '/') + if idx < 0 { + return "", p.Prefix + } + return p.Prefix[:idx], p.Prefix[idx+1:] +} + +// // FIXME this could be implemented by VFS.MkdirAll() +// func mkdirRecursive(path string, VFS *vfs.VFS) error { +// path = strings.Trim(path, "/") +// dirs := strings.Split(path, "/") +// dir := "" +// for _, d := range dirs { +// dir += "/" + d +// if _, err := VFS.Stat(dir); err != nil { +// err := VFS.Mkdir(dir, 0777) +// if err != nil { +// return err +// } +// } +// } +// return nil +// } + +// func rmdirRecursive(p string, VFS *vfs.VFS) { +// dir := path.Dir(p) +// if !strings.ContainsAny(dir, "/\\") { +// // might be bucket(root) +// return +// } +// if _, err := VFS.Stat(dir); err == nil { +// err := VFS.Remove(dir) +// if err != nil { +// return +// } +// rmdirRecursive(dir, VFS) +// } +// } + +func authlistResolver() map[string]string { + s3accesskeyid := setting.GetStr(conf.S3AccessKeyId) + s3secretaccesskey := setting.GetStr(conf.S3SecretAccessKey) + if s3accesskeyid == "" && s3secretaccesskey == "" { + return nil + } + authList := make(map[string]string) + authList[s3accesskeyid] = s3secretaccesskey + return authList +} diff --git a/server/sftp.go b/server/sftp.go new file mode 100644 index 00000000000..7d8c7212e9a --- /dev/null +++ b/server/sftp.go @@ -0,0 +1,144 @@ +package server + +import ( + "context" + "github.com/KirCute/sftpd-alist" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/internal/setting" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" + "github.com/alist-org/alist/v3/server/ftp" + "github.com/alist-org/alist/v3/server/sftp" + "github.com/pkg/errors" + "golang.org/x/crypto/ssh" + "net/http" + "time" +) + +type SftpDriver struct { + proxyHeader *http.Header + config *sftpd.Config +} + +func NewSftpDriver() (*SftpDriver, error) { + sftp.InitHostKey() + header := &http.Header{} + header.Add("User-Agent", setting.GetStr(conf.FTPProxyUserAgent)) + return &SftpDriver{ + proxyHeader: header, + }, nil +} + +func (d *SftpDriver) GetConfig() *sftpd.Config { + if d.config != nil { + return d.config + } + serverConfig := ssh.ServerConfig{ + NoClientAuth: true, + NoClientAuthCallback: d.NoClientAuth, + PasswordCallback: d.PasswordAuth, + PublicKeyCallback: d.PublicKeyAuth, + AuthLogCallback: d.AuthLogCallback, + BannerCallback: d.GetBanner, + } + for _, k := range sftp.SSHSigners { + serverConfig.AddHostKey(k) + } + d.config = &sftpd.Config{ + ServerConfig: serverConfig, + HostPort: conf.Conf.SFTP.Listen, + ErrorLogFunc: utils.Log.Error, + //DebugLogFunc: utils.Log.Debugf, + } + return d.config +} + +func (d *SftpDriver) GetFileSystem(sc *ssh.ServerConn) (sftpd.FileSystem, error) { + userObj, err := op.GetUserByName(sc.User()) + if err != nil { + return nil, err + } + ctx := context.Background() + ctx = context.WithValue(ctx, "user", userObj) + ctx = context.WithValue(ctx, "meta_pass", "") + ctx = context.WithValue(ctx, "client_ip", sc.RemoteAddr().String()) + ctx = context.WithValue(ctx, "proxy_header", d.proxyHeader) + return &sftp.DriverAdapter{FtpDriver: ftp.NewAferoAdapter(ctx)}, nil +} + +func (d *SftpDriver) Close() { +} + +func (d *SftpDriver) NoClientAuth(conn ssh.ConnMetadata) (*ssh.Permissions, error) { + if conn.User() != "guest" { + return nil, errors.New("only guest is allowed to login without authorization") + } + guest, err := op.GetGuest() + if err != nil { + return nil, err + } + permGuest := common.MergeRolePermissions(guest, guest.BasePath) + if guest.Disabled || !common.HasPermission(permGuest, common.PermFTPAccess) { + return nil, errors.New("user is not allowed to access via SFTP") + } + return nil, nil +} + +func (d *SftpDriver) PasswordAuth(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) { + userObj, err := op.GetUserByName(conn.User()) + if err != nil { + return nil, err + } + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { + return nil, errors.New("user is not allowed to access via SFTP") + } + passHash := model.StaticHash(string(password)) + if err = userObj.ValidatePwdStaticHash(passHash); err != nil { + return nil, err + } + return nil, nil +} + +func (d *SftpDriver) PublicKeyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { + userObj, err := op.GetUserByName(conn.User()) + if err != nil { + return nil, err + } + perm := common.MergeRolePermissions(userObj, userObj.BasePath) + if userObj.Disabled || !common.HasPermission(perm, common.PermFTPAccess) { + return nil, errors.New("user is not allowed to access via SFTP") + } + keys, _, err := op.GetSSHPublicKeyByUserId(userObj.ID, 1, -1) + if err != nil { + return nil, err + } + marshal := string(key.Marshal()) + for _, sk := range keys { + if marshal != sk.KeyStr { + pubKey, _, _, _, e := ssh.ParseAuthorizedKey([]byte(sk.KeyStr)) + if e != nil || marshal != string(pubKey.Marshal()) { + continue + } + } + sk.LastUsedTime = time.Now() + _ = op.UpdateSSHPublicKey(&sk) + return nil, nil + } + return nil, errors.New("public key refused") +} + +func (d *SftpDriver) AuthLogCallback(conn ssh.ConnMetadata, method string, err error) { + ip := conn.RemoteAddr().String() + if err == nil { + utils.Log.Infof("[SFTP] %s(%s) logged in via %s", conn.User(), ip, method) + } else if method != "none" { + utils.Log.Infof("[SFTP] %s(%s) tries logging in via %s but with error: %s", conn.User(), ip, method, err) + } +} + +func (d *SftpDriver) GetBanner(_ ssh.ConnMetadata) string { + return setting.GetStr(conf.Announcement) +} diff --git a/server/sftp/const.go b/server/sftp/const.go new file mode 100644 index 00000000000..58bfe3824ca --- /dev/null +++ b/server/sftp/const.go @@ -0,0 +1,11 @@ +package sftp + +// From leffss/sftpd +const ( + SSH_FXF_READ = 0x00000001 + SSH_FXF_WRITE = 0x00000002 + SSH_FXF_APPEND = 0x00000004 + SSH_FXF_CREAT = 0x00000008 + SSH_FXF_TRUNC = 0x00000010 + SSH_FXF_EXCL = 0x00000020 +) diff --git a/server/sftp/hostkey.go b/server/sftp/hostkey.go new file mode 100644 index 00000000000..0db103dd6cf --- /dev/null +++ b/server/sftp/hostkey.go @@ -0,0 +1,105 @@ +package sftp + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "github.com/alist-org/alist/v3/cmd/flags" + "github.com/alist-org/alist/v3/pkg/utils" + "golang.org/x/crypto/ssh" + "os" + "path/filepath" +) + +var SSHSigners []ssh.Signer + +func InitHostKey() { + if SSHSigners != nil { + return + } + sshPath := filepath.Join(flags.DataDir, "ssh") + if !utils.Exists(sshPath) { + err := utils.CreateNestedDirectory(sshPath) + if err != nil { + utils.Log.Fatalf("failed to create ssh directory: %+v", err) + return + } + } + SSHSigners = make([]ssh.Signer, 0, 4) + if rsaKey, ok := LoadOrGenerateRSAHostKey(sshPath); ok { + SSHSigners = append(SSHSigners, rsaKey) + } + // TODO Add keys for other encryption algorithms +} + +func LoadOrGenerateRSAHostKey(parentDir string) (ssh.Signer, bool) { + privateKeyPath := filepath.Join(parentDir, "ssh_host_rsa_key") + publicKeyPath := filepath.Join(parentDir, "ssh_host_rsa_key.pub") + privateKeyBytes, err := os.ReadFile(privateKeyPath) + if err == nil { + var privateKey *rsa.PrivateKey + privateKey, err = rsaDecodePrivateKey(privateKeyBytes) + if err == nil { + var ret ssh.Signer + ret, err = ssh.NewSignerFromKey(privateKey) + if err == nil { + return ret, true + } + } + } + _ = os.Remove(privateKeyPath) + _ = os.Remove(publicKeyPath) + privateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + utils.Log.Fatalf("failed to generate RSA private key: %+v", err) + return nil, false + } + publicKey, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + utils.Log.Fatalf("failed to generate RSA public key: %+v", err) + return nil, false + } + ret, err := ssh.NewSignerFromKey(privateKey) + if err != nil { + utils.Log.Fatalf("failed to generate RSA signer: %+v", err) + return nil, false + } + privateBytes := rsaEncodePrivateKey(privateKey) + publicBytes := ssh.MarshalAuthorizedKey(publicKey) + err = os.WriteFile(privateKeyPath, privateBytes, 0600) + if err != nil { + utils.Log.Fatalf("failed to write RSA private key to file: %+v", err) + return nil, false + } + err = os.WriteFile(publicKeyPath, publicBytes, 0644) + if err != nil { + _ = os.Remove(privateKeyPath) + utils.Log.Fatalf("failed to write RSA public key to file: %+v", err) + return nil, false + } + return ret, true +} + +func rsaEncodePrivateKey(privateKey *rsa.PrivateKey) []byte { + privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey) + privateBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: privateKeyBytes, + } + return pem.EncodeToMemory(privateBlock) +} + +func rsaDecodePrivateKey(bytes []byte) (*rsa.PrivateKey, error) { + block, _ := pem.Decode(bytes) + if block == nil { + return nil, fmt.Errorf("failed to parse PEM block containing the key") + } + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return nil, err + } + return privateKey, nil +} diff --git a/server/sftp/sftp.go b/server/sftp/sftp.go new file mode 100644 index 00000000000..1ceb3f59295 --- /dev/null +++ b/server/sftp/sftp.go @@ -0,0 +1,123 @@ +package sftp + +import ( + "github.com/KirCute/sftpd-alist" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/ftp" + "os" +) + +type DriverAdapter struct { + FtpDriver *ftp.AferoAdapter +} + +func (s *DriverAdapter) OpenFile(_ string, _ uint32, _ *sftpd.Attr) (sftpd.File, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (s *DriverAdapter) OpenDir(_ string) (sftpd.Dir, error) { + // See also GetHandle + return nil, errs.NotImplement +} + +func (s *DriverAdapter) Remove(name string) error { + return s.FtpDriver.Remove(name) +} + +func (s *DriverAdapter) Rename(old, new string, _ uint32) error { + return s.FtpDriver.Rename(old, new) +} + +func (s *DriverAdapter) Mkdir(name string, attr *sftpd.Attr) error { + return s.FtpDriver.Mkdir(name, attr.Mode) +} + +func (s *DriverAdapter) Rmdir(name string) error { + return s.Remove(name) +} + +func (s *DriverAdapter) Stat(name string, _ bool) (*sftpd.Attr, error) { + stat, err := s.FtpDriver.Stat(name) + if err != nil { + return nil, err + } + return fileInfoToSftpAttr(stat), nil +} + +func (s *DriverAdapter) SetStat(_ string, _ *sftpd.Attr) error { + return errs.NotSupport +} + +func (s *DriverAdapter) ReadLink(_ string) (string, error) { + return "", errs.NotSupport +} + +func (s *DriverAdapter) CreateLink(_, _ string, _ uint32) error { + return errs.NotSupport +} + +func (s *DriverAdapter) RealPath(path string) (string, error) { + return utils.FixAndCleanPath(path), nil +} + +func (s *DriverAdapter) GetHandle(name string, flags uint32, _ *sftpd.Attr, offset uint64) (sftpd.FileTransfer, error) { + return s.FtpDriver.GetHandle(name, sftpFlagToOpenMode(flags), int64(offset)) +} + +func (s *DriverAdapter) ReadDir(name string) ([]sftpd.NamedAttr, error) { + dir, err := s.FtpDriver.ReadDir(name) + if err != nil { + return nil, err + } + ret := make([]sftpd.NamedAttr, len(dir)) + for i, d := range dir { + ret[i] = *fileInfoToSftpNamedAttr(d) + } + return ret, nil +} + +// From leffss/sftpd +func sftpFlagToOpenMode(flags uint32) int { + mode := 0 + if (flags & SSH_FXF_READ) != 0 { + mode |= os.O_RDONLY + } + if (flags & SSH_FXF_WRITE) != 0 { + mode |= os.O_WRONLY + } + if (flags & SSH_FXF_APPEND) != 0 { + mode |= os.O_APPEND + } + if (flags & SSH_FXF_CREAT) != 0 { + mode |= os.O_CREATE + } + if (flags & SSH_FXF_TRUNC) != 0 { + mode |= os.O_TRUNC + } + if (flags & SSH_FXF_EXCL) != 0 { + mode |= os.O_EXCL + } + return mode +} + +func fileInfoToSftpAttr(stat os.FileInfo) *sftpd.Attr { + ret := &sftpd.Attr{} + ret.Flags |= sftpd.ATTR_SIZE + ret.Size = uint64(stat.Size()) + ret.Flags |= sftpd.ATTR_MODE + ret.Mode = stat.Mode() + ret.Flags |= sftpd.ATTR_TIME + ret.ATime = stat.Sys().(model.Obj).CreateTime() + ret.MTime = stat.ModTime() + return ret +} + +func fileInfoToSftpNamedAttr(stat os.FileInfo) *sftpd.NamedAttr { + return &sftpd.NamedAttr{ + Name: stat.Name(), + Attr: *fileInfoToSftpAttr(stat), + } +} diff --git a/server/static/static.go b/server/static/static.go index 624637936cd..d5d6ff685cd 100644 --- a/server/static/static.go +++ b/server/static/static.go @@ -3,8 +3,10 @@ package static import ( "errors" "fmt" + "io" "io/fs" "net/http" + "os" "strings" "github.com/alist-org/alist/v3/internal/conf" @@ -14,14 +16,35 @@ import ( "github.com/gin-gonic/gin" ) -func InitIndex() { - index, err := public.Public.ReadFile("dist/index.html") +var static fs.FS + +func initStatic() { + if conf.Conf.DistDir == "" { + dist, err := fs.Sub(public.Public, "dist") + if err != nil { + utils.Log.Fatalf("failed to read dist dir") + } + static = dist + return + } + static = os.DirFS(conf.Conf.DistDir) +} + +func initIndex() { + indexFile, err := static.Open("index.html") if err != nil { if errors.Is(err, fs.ErrNotExist) { utils.Log.Fatalf("index.html not exist, you may forget to put dist of frontend to public/dist") } utils.Log.Fatalf("failed to read index.html: %v", err) } + defer func() { + _ = indexFile.Close() + }() + index, err := io.ReadAll(indexFile) + if err != nil { + utils.Log.Fatalf("failed to read dist/index.html") + } conf.RawIndexHtml = string(index) siteConfig := getSiteConfig() replaceMap := map[string]string{ @@ -60,7 +83,8 @@ func UpdateIndex() { } func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { - InitIndex() + initStatic() + initIndex() folders := []string{"assets", "images", "streamer", "static"} r.Use(func(c *gin.Context) { for i := range folders { @@ -70,8 +94,7 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { } }) for i, folder := range folders { - folder = "dist/" + folder - sub, err := fs.Sub(public.Public, folder) + sub, err := fs.Sub(static, folder) if err != nil { utils.Log.Fatalf("can't find folder: %s", folder) } @@ -79,6 +102,10 @@ func Static(r *gin.RouterGroup, noRoute func(handlers ...gin.HandlerFunc)) { } noRoute(func(c *gin.Context) { + if c.Request.Method != "GET" && c.Request.Method != "POST" { + c.Status(405) + return + } c.Header("Content-Type", "text/html") c.Status(200) if strings.HasPrefix(c.Request.URL.Path, "/@manage") { diff --git a/server/webdav.go b/server/webdav.go index 2b5c9618b86..d188cb8010d 100644 --- a/server/webdav.go +++ b/server/webdav.go @@ -3,15 +3,23 @@ package server import ( "context" "crypto/subtle" + "fmt" "net/http" + "net/url" "path" "strings" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/server/middlewares" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/device" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" "github.com/alist-org/alist/v3/internal/setting" "github.com/alist-org/alist/v3/pkg/utils" + "github.com/alist-org/alist/v3/server/common" "github.com/alist-org/alist/v3/server/webdav" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -24,12 +32,20 @@ func WebDav(dav *gin.RouterGroup) { Prefix: path.Join(conf.URL.Path, "/dav"), LockSystem: webdav.NewMemLS(), Logger: func(request *http.Request, err error) { + // Skip logging for NotFoundError as it's not a program error + // but a normal case when a file doesn't exist + if errs.IsNotFoundError(err) { + log.Debugf("%s %s %v", request.Method, request.URL.Path, err) + return + } log.Errorf("%s %s %+v", request.Method, request.URL.Path, err) }, } dav.Use(WebDAVAuth) - dav.Any("/*path", ServeWebDAV) - dav.Any("", ServeWebDAV) + uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit) + downloadLimiter := middlewares.DownloadRateLimiter(stream.ClientDownloadLimit) + dav.Any("/*path", uploadLimiter, downloadLimiter, ServeWebDAV) + dav.Any("", uploadLimiter, downloadLimiter, ServeWebDAV) dav.Handle("PROPFIND", "/*path", ServeWebDAV) dav.Handle("PROPFIND", "", ServeWebDAV) dav.Handle("MKCOL", "/*path", ServeWebDAV) @@ -63,6 +79,13 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", admin.ID, c.ClientIP())) + if err := device.Handle(admin.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + c.Set("device_key", key) c.Set("user", admin) c.Next() return @@ -89,17 +112,23 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } - if user.Disabled || !user.CanWebdavRead() { - if c.Request.Method == "OPTIONS" { - c.Set("user", guest) - c.Next() - return - } + if roles, err := op.GetRolesByUserID(user.ID); err == nil { + user.RolesDetail = roles + } + reqPath := c.Param("path") + if reqPath == "" { + reqPath = "/" + } + reqPath, _ = url.PathUnescape(reqPath) + reqPath, err = webdav.ResolvePath(user, reqPath) + if err != nil { c.Status(http.StatusForbidden) c.Abort() return } - if !user.CanWebdavManage() && utils.SliceContains([]string{"PUT", "DELETE", "PROPPATCH", "MKCOL", "COPY", "MOVE"}, c.Request.Method) { + perm := common.MergeRolePermissions(user, reqPath) + webdavRead := common.HasPermission(perm, common.PermWebdavRead) + if user.Disabled || (!webdavRead && (c.Request.Method != "PROPFIND" || !common.HasChildPermission(user, reqPath, common.PermWebdavRead))) { if c.Request.Method == "OPTIONS" { c.Set("user", guest) c.Next() @@ -109,6 +138,38 @@ func WebDAVAuth(c *gin.Context) { c.Abort() return } + if (c.Request.Method == "PUT" || c.Request.Method == "MKCOL") && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermWrite)) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "MOVE" && (!common.HasPermission(perm, common.PermWebdavManage) || (!common.HasPermission(perm, common.PermMove) && !common.HasPermission(perm, common.PermRename))) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "COPY" && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermCopy)) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "DELETE" && (!common.HasPermission(perm, common.PermWebdavManage) || !common.HasPermission(perm, common.PermRemove)) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + if c.Request.Method == "PROPPATCH" && !common.HasPermission(perm, common.PermWebdavManage) { + c.Status(http.StatusForbidden) + c.Abort() + return + } + key := utils.GetMD5EncodeStr(fmt.Sprintf("%d-%s", user.ID, c.ClientIP())) + if err := device.Handle(user.ID, key, c.Request.UserAgent(), c.ClientIP()); err != nil { + c.Status(http.StatusForbidden) + c.Abort() + return + } + c.Set("device_key", key) c.Set("user", user) c.Next() } diff --git a/server/webdav/file.go b/server/webdav/file.go index 01e96f7d223..419c7b07207 100644 --- a/server/webdav/file.go +++ b/server/webdav/file.go @@ -14,6 +14,7 @@ import ( "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/server/common" ) // slashClean is equivalent to but slightly more efficient than @@ -33,6 +34,14 @@ func moveFiles(ctx context.Context, src, dst string, overwrite bool) (status int dstDir := path.Dir(dst) srcName := path.Base(src) dstName := path.Base(dst) + user := ctx.Value("user").(*model.User) + perm := common.MergeRolePermissions(user, src) + if srcDir != dstDir && !common.HasPermission(perm, common.PermMove) { + return http.StatusForbidden, nil + } + if srcName != dstName && !common.HasPermission(perm, common.PermRename) { + return http.StatusForbidden, nil + } if srcDir == dstDir { err = fs.Rename(ctx, src, dstName) } else { @@ -85,6 +94,7 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn depth = 0 } meta, _ := op.GetNearestMeta(name) + user := ctx.Value("user").(*model.User) // Read directory names. objs, err := fs.List(context.WithValue(ctx, "meta", meta), name, &fs.ListArgs{}) //f, err := fs.OpenFile(ctx, name, os.O_RDONLY, 0) @@ -99,6 +109,9 @@ func walkFS(ctx context.Context, depth int, name string, info model.Obj, walkFn for _, fileInfo := range objs { filename := path.Join(name, fileInfo.GetName()) + if !common.CanReadPathByRole(user, filename) { + continue + } if err != nil { if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { return err diff --git a/server/webdav/path.go b/server/webdav/path.go new file mode 100644 index 00000000000..9a18da9551f --- /dev/null +++ b/server/webdav/path.go @@ -0,0 +1,22 @@ +package webdav + +import ( + "path" + "strings" + + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" +) + +// ResolvePath normalizes the provided raw path and resolves it against the user's base path +// before delegating to the user-aware JoinPath permission checks. +func ResolvePath(user *model.User, raw string) (string, error) { + cleaned := utils.FixAndCleanPath(raw) + basePath := utils.FixAndCleanPath(user.BasePath) + + if cleaned != "/" && basePath != "/" && !utils.IsSubPath(basePath, cleaned) { + cleaned = path.Join(basePath, strings.TrimPrefix(cleaned, "/")) + } + + return user.JoinPath(cleaned) +} diff --git a/server/webdav/prop.go b/server/webdav/prop.go index 3c3b10d8227..a81f31b05c7 100644 --- a/server/webdav/prop.go +++ b/server/webdav/prop.go @@ -14,8 +14,11 @@ import ( "net/http" "path" "strconv" + "strings" + "time" "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/server/common" ) // Proppatch describes a property update instruction as defined in RFC 4918. @@ -99,7 +102,7 @@ type DeadPropsHolder interface { Patch([]Proppatch) ([]Propstat, error) } -// liveProps contains all supported, protected DAV: properties. +// liveProps contains all supported properties. var liveProps = map[xml.Name]struct { // findFn implements the propfind function of this property. If nil, // it indicates a hidden property. @@ -158,6 +161,10 @@ var liveProps = map[xml.Name]struct { findFn: findSupportedLock, dir: true, }, + {Space: "http://owncloud.org/ns", Local: "checksums"}: { + findFn: findChecksums, + dir: false, + }, } // TODO(nigeltao) merge props and allprop? @@ -384,7 +391,11 @@ func findLastModified(ctx context.Context, ls LockSystem, name string, fi model. return fi.ModTime().UTC().Format(http.TimeFormat), nil } func findCreationDate(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { - return fi.CreateTime().UTC().Format(http.TimeFormat), nil + userAgent := ctx.Value("userAgent").(string) + if strings.Contains(strings.ToLower(userAgent), "microsoft-webdav") { + return fi.CreateTime().UTC().Format(http.TimeFormat), nil + } + return fi.CreateTime().UTC().Format(time.RFC3339), nil } // ErrNotImplemented should be returned by optional interfaces if they @@ -467,7 +478,7 @@ func findETag(ctx context.Context, ls LockSystem, name string, fi model.Obj) (st // The Apache http 2.4 web server by default concatenates the // modification time and size of a file. We replicate the heuristic // with nanosecond granularity. - return fmt.Sprintf(`"%x%x"`, fi.ModTime().UnixNano(), fi.GetSize()), nil + return common.GetEtag(fi), nil } func findSupportedLock(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { @@ -477,3 +488,11 @@ func findSupportedLock(ctx context.Context, ls LockSystem, name string, fi model `` + ``, nil } + +func findChecksums(ctx context.Context, ls LockSystem, name string, fi model.Obj) (string, error) { + checksums := "" + for hashType, hashValue := range fi.GetHash().All() { + checksums += fmt.Sprintf("%s:%s", hashType.Name, hashValue) + } + return checksums, nil +} diff --git a/server/webdav/util.go b/server/webdav/util.go index 15d9e07cc56..a2a1641ce62 100644 --- a/server/webdav/util.go +++ b/server/webdav/util.go @@ -8,16 +8,21 @@ import ( ) func (h *Handler) getModTime(r *http.Request) time.Time { - return h.getHeaderTime(r, "X-OC-Mtime") + return h.getHeaderTime(r, "X-OC-Mtime", "") } -// owncloud/ nextcloud haven't impl this, but we can add the support since rclone may support this soon +// owncloud/ nextcloud haven't impl this, but we can add the support since rclone may support this soon. +// try ModTime if CreateTime not found in header func (h *Handler) getCreateTime(r *http.Request) time.Time { - return h.getHeaderTime(r, "X-OC-Ctime") + return h.getHeaderTime(r, "X-OC-Ctime", "X-OC-Mtime") } -func (h *Handler) getHeaderTime(r *http.Request, header string) time.Time { +func (h *Handler) getHeaderTime(r *http.Request, header, alternative string) time.Time { hVal := r.Header.Get(header) + // try alternative + if hVal == "" && alternative != "" { + hVal = r.Header.Get(alternative) + } if hVal != "" { modTimeUnix, err := strconv.ParseInt(hVal, 10, 64) if err == nil { diff --git a/server/webdav/webdav.go b/server/webdav/webdav.go index f2e3fd8a409..df1d2045140 100644 --- a/server/webdav/webdav.go +++ b/server/webdav/webdav.go @@ -6,6 +6,7 @@ package webdav // import "golang.org/x/net/webdav" import ( + "context" "errors" "fmt" "net/http" @@ -20,10 +21,8 @@ import ( "github.com/alist-org/alist/v3/internal/errs" "github.com/alist-org/alist/v3/internal/fs" "github.com/alist-org/alist/v3/internal/model" - "github.com/alist-org/alist/v3/internal/sign" "github.com/alist-org/alist/v3/pkg/utils" "github.com/alist-org/alist/v3/server/common" - log "github.com/sirupsen/logrus" ) type Handler struct { @@ -58,7 +57,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { status, err = h.handleOptions(brw, r) case "GET", "HEAD", "POST": useBufferedWriter = false - status, err = h.handleGetHeadPost(w, r) + Writer := &common.WrittenResponseWriter{ResponseWriter: w} + status, err = h.handleGetHeadPost(Writer, r) + if status != 0 && Writer.IsWritten() { + status = 0 + } case "DELETE": status, err = h.handleDelete(brw, r) case "PUT": @@ -190,7 +193,7 @@ func (h *Handler) handleOptions(w http.ResponseWriter, r *http.Request) (status } ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -218,7 +221,7 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta // TODO: check locks for read-only access?? ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return http.StatusForbidden, err } @@ -226,11 +229,6 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta if err != nil { return http.StatusNotFound, err } - etag, err := findETag(ctx, h.LockSystem, reqPath, fi) - if err != nil { - return http.StatusInternalServerError, err - } - w.Header().Set("ETag", etag) if r.Method == http.MethodHead { w.Header().Set("Content-Length", fmt.Sprintf("%d", fi.GetSize())) return http.StatusOK, nil @@ -246,20 +244,19 @@ func (h *Handler) handleGetHeadPost(w http.ResponseWriter, r *http.Request) (sta if err != nil { return http.StatusInternalServerError, err } + if storage.GetStorage().ProxyRange { + common.ProxyRange(link, fi.GetSize()) + } err = common.Proxy(w, r, link, fi) if err != nil { - log.Errorf("webdav proxy error: %+v", err) - return http.StatusInternalServerError, err + return http.StatusInternalServerError, fmt.Errorf("webdav proxy error: %+v", err) } } else if storage.GetStorage().WebdavProxy() && downProxyUrl != "" { - u := fmt.Sprintf("%s%s?sign=%s", - strings.Split(downProxyUrl, "\n")[0], - utils.EncodePath(reqPath, true), - sign.Sign(reqPath)) + u := common.BuildDownProxyURL(downProxyUrl, reqPath, storage.GetStorage().DownProxySign) w.Header().Set("Cache-Control", "max-age=0, no-cache, no-store, must-revalidate") http.Redirect(w, r, u, http.StatusFound) } else { - link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{IP: utils.ClientIP(r), Header: r.Header, HttpReq: r}) + link, _, err := fs.Link(ctx, reqPath, model.LinkArgs{IP: utils.ClientIP(r), Header: r.Header, HttpReq: r, Redirect: true}) if err != nil { return http.StatusInternalServerError, err } @@ -281,7 +278,7 @@ func (h *Handler) handleDelete(w http.ResponseWriter, r *http.Request) (status i ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -320,7 +317,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, // comments in http.checkEtag. ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return http.StatusForbidden, err } @@ -330,21 +327,21 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, Modified: h.getModTime(r), Ctime: h.getCreateTime(r), } - stream := &stream.FileStream{ + fsStream := &stream.FileStream{ Obj: &obj, Reader: r.Body, Mimetype: r.Header.Get("Content-Type"), } - if stream.Mimetype == "" { - stream.Mimetype = utils.GetMimeType(reqPath) + if fsStream.Mimetype == "" { + fsStream.Mimetype = utils.GetMimeType(reqPath) } - err = fs.PutDirectly(ctx, path.Dir(reqPath), stream) + err = fs.PutDirectly(ctx, path.Dir(reqPath), fsStream) if errs.IsNotFoundError(err) { return http.StatusNotFound, err } _ = r.Body.Close() - _ = stream.Close() + _ = fsStream.Close() // TODO(rost): Returning 405 Method Not Allowed might not be appropriate. if err != nil { return http.StatusMethodNotAllowed, err @@ -357,7 +354,7 @@ func (h *Handler) handlePut(w http.ResponseWriter, r *http.Request) (status int, if err != nil { return http.StatusInternalServerError, err } - w.Header().Set("ETag", etag) + w.Header().Set("Etag", etag) return http.StatusCreated, nil } @@ -374,7 +371,7 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -382,6 +379,21 @@ func (h *Handler) handleMkcol(w http.ResponseWriter, r *http.Request) (status in if r.ContentLength > 0 { return http.StatusUnsupportedMediaType, nil } + + // RFC 4918 9.3.1 + //405 (Method Not Allowed) - MKCOL can only be executed on an unmapped URL + if _, err := fs.Get(ctx, reqPath, &fs.GetArgs{}); err == nil { + return http.StatusMethodNotAllowed, err + } + // RFC 4918 9.3.1 + // 409 (Conflict) The server MUST NOT create those intermediate collections automatically. + reqDir := path.Dir(reqPath) + if _, err := fs.Get(ctx, reqDir, &fs.GetArgs{}); err != nil { + if errs.IsObjectNotFound(err) { + return http.StatusConflict, err + } + return http.StatusMethodNotAllowed, err + } if err := fs.MakeDir(ctx, reqPath); err != nil { if os.IsNotExist(err) { return http.StatusConflict, err @@ -423,11 +435,11 @@ func (h *Handler) handleCopyMove(w http.ResponseWriter, r *http.Request) (status ctx := r.Context() user := ctx.Value("user").(*model.User) - src, err = user.JoinPath(src) + src, err = ResolvePath(user, src) if err != nil { return 403, err } - dst, err = user.JoinPath(dst) + dst, err = ResolvePath(user, dst) if err != nil { return 403, err } @@ -521,12 +533,12 @@ func (h *Handler) handleLock(w http.ResponseWriter, r *http.Request) (retStatus } } reqPath, status, err := h.stripPrefix(r.URL.Path) - reqPath, err = user.JoinPath(reqPath) if err != nil { - return 403, err + return status, err } + reqPath, err = ResolvePath(user, reqPath) if err != nil { - return status, err + return 403, err } ld = LockDetails{ Root: reqPath, @@ -604,8 +616,10 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status return status, err } ctx := r.Context() + userAgent := r.Header.Get("User-Agent") + ctx = context.WithValue(ctx, "userAgent", userAgent) user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -630,6 +644,98 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status mw := multistatusWriter{w: w} + if utils.PathEqual(reqPath, user.BasePath) { + hasRootPerm := false + for _, role := range user.RolesDetail { + for _, entry := range role.PermissionScopes { + if utils.PathEqual(entry.Path, user.BasePath) { + hasRootPerm = true + break + } + } + if hasRootPerm { + break + } + } + if !hasRootPerm { + basePaths := model.GetAllBasePathsFromRoles(user) + type infoItem struct { + path string + info model.Obj + } + infos := []infoItem{{reqPath, fi}} + seen := make(map[string]struct{}) + for _, p := range basePaths { + if !utils.IsSubPath(user.BasePath, p) { + continue + } + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(p), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + dir := strings.Split(rel, "/")[0] + if dir == "" { + continue + } + if _, ok := seen[dir]; ok { + continue + } + seen[dir] = struct{}{} + sp := utils.FixAndCleanPath(path.Join(user.BasePath, dir)) + info, err := fs.Get(ctx, sp, &fs.GetArgs{}) + if err != nil { + continue + } + infos = append(infos, infoItem{sp, info}) + } + for _, item := range infos { + var pstats []Propstat + if pf.Propname != nil { + pnames, err := propnames(ctx, h.LockSystem, item.info) + if err != nil { + return http.StatusInternalServerError, err + } + pstat := Propstat{Status: http.StatusOK} + for _, xmlname := range pnames { + pstat.Props = append(pstat.Props, Property{XMLName: xmlname}) + } + pstats = append(pstats, pstat) + } else if pf.Allprop != nil { + pstats, err = allprop(ctx, h.LockSystem, item.info, pf.Prop) + if err != nil { + return http.StatusInternalServerError, err + } + } else { + pstats, err = props(ctx, h.LockSystem, item.info, pf.Prop) + if err != nil { + return http.StatusInternalServerError, err + } + } + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(item.path), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + href := utils.EncodePath(path.Join("/", h.Prefix, rel), true) + if href != "/" && item.info.IsDir() { + href += "/" + } + if err := mw.write(makePropstatResponse(href, pstats)); err != nil { + return http.StatusInternalServerError, err + } + } + if err := mw.close(); err != nil { + return http.StatusInternalServerError, err + } + return 0, nil + } + } + walkFn := func(reqPath string, info model.Obj, err error) error { if err != nil { return err @@ -653,7 +759,14 @@ func (h *Handler) handlePropfind(w http.ResponseWriter, r *http.Request) (status if err != nil { return err } - href := path.Join(h.Prefix, strings.TrimPrefix(reqPath, user.BasePath)) + rel := strings.TrimPrefix( + strings.TrimPrefix( + utils.FixAndCleanPath(reqPath), + utils.FixAndCleanPath(user.BasePath), + ), + "/", + ) + href := utils.EncodePath(path.Join("/", h.Prefix, rel), true) if href != "/" && info.IsDir() { href += "/" } @@ -684,7 +797,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu ctx := r.Context() user := ctx.Value("user").(*model.User) - reqPath, err = user.JoinPath(reqPath) + reqPath, err = ResolvePath(user, reqPath) if err != nil { return 403, err } @@ -716,7 +829,7 @@ func (h *Handler) handleProppatch(w http.ResponseWriter, r *http.Request) (statu func makePropstatResponse(href string, pstats []Propstat) *response { resp := response{ - Href: []string{(&url.URL{Path: href}).EscapedPath()}, + Href: []string{href}, Propstat: make([]propstat, 0, len(pstats)), } for _, p := range pstats {