3.66

* chore(deps): bump github.com/viant/ptrie from 0.3.1 to 1.0.1

Bumps [github.com/viant/ptrie](https://github.com/viant/ptrie) from 0.3.1 to 1.0.1.
- [Release notes](https://github.com/viant/ptrie/releases)
- [Changelog](https://github.com/viant/ptrie/blob/master/CHANGELOG.md)
- [Commits](https://github.com/viant/ptrie/compare/v0.3.1...v1.0.1)

---
updated-dependencies:
- dependency-name: github.com/viant/ptrie
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>

* fix compilation

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: chrislu <chris.lu@gmail.com>

.github/workflows/binaries_dev.yml

@@ -38,13 +38,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -60,7 +60,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -87,13 +87,13 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
 
       - name: Go Release Binaries Large Disk
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -109,7 +109,7 @@ jobs:
           asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
 
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release0.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release1.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release2.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release3.yml

@@ -28,9 +28,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
@@ -44,7 +44,7 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}

.github/workflows/binaries_release4.yml

@@ -28,15 +28,15 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       - name: Go Release Binaries Normal Volume Size
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
           overwrite: true
-          build_flags: -tags elastic,ydb,gocdk,tikv
+          build_flags: -tags elastic,gocdk,sqlite,ydb,tikv,rclone
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
           # build_flags: -tags 5BytesOffset # optional, default is
           ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
@@ -45,14 +45,14 @@ jobs:
           binary_name: weed
           asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
       - name: Go Release Large Disk Binaries
-        uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
+        uses: wangyoucao577/go-release-action@8fa1e8368c8465264d64e0198208e10f71474c87 # v1.22
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           goos: ${{ matrix.goos }}
           goarch: ${{ matrix.goarch }}
           overwrite: true
           pre_command: export CGO_ENABLED=0 && export GODEBUG=http2client=0
-          build_flags: -tags 5BytesOffset,elastic,ydb,gocdk,tikv
+          build_flags: -tags 5BytesOffset,elastic,gocdk,sqlite,ydb,tikv,rclone
           ldflags: -s -w -extldflags -static -X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=${{github.sha}}
           # Where to run `go build .`
           project_path: weed

.github/workflows/codeql.yml

@@ -18,11 +18,11 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         # Override language selection by uncommenting this and choosing your languages
         with:
           languages: go
@@ -30,7 +30,7 @@ jobs:
       # Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below).
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -44,4 +44,4 @@ jobs:
       #     make release
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3

.github/workflows/container_dev.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -36,27 +36,27 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_latest.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -37,27 +37,27 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
         with:
           buildkitd-flags: "--debug"
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Login to GHCR
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           registry: ghcr.io
           username: ${{ secrets.GHCR_USERNAME }}
           password: ${{ secrets.GHCR_TOKEN }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release1.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -37,17 +37,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release2.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -38,17 +38,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release3.yml

@@ -17,11 +17,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -38,17 +38,17 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/container_release4.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -37,22 +37,22 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          build-args: TAGS=elastic,ydb,gocdk,tikv
+          build-args: TAGS=elastic,ydb,gocdk,tikv,rclone
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release5.yml

@@ -16,11 +16,11 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
       -
         name: Docker meta
         id: docker_meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v3
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v3
         with:
           images: |
             chrislusf/seaweedfs
@@ -37,22 +37,22 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v1
+        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v1
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v1
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       -
         name: Build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v2
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v2
         with:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv
+          build-args: TAGS=5BytesOffset,elastic,ydb,gocdk,tikv,rclone
           platforms: linux/amd64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034
+        uses: actions/dependency-review-action@0659a74c94536054bfa5aeb92241f70d680cc78e

.github/workflows/e2e.yml

@@ -24,13 +24,13 @@ jobs:
     timeout-minutes: 30
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v2
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
     - name: Install dependencies
       run: |
@@ -94,7 +94,7 @@ jobs:
 
     - name: Archive logs
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: output-logs
         path: docker/output.log

.github/workflows/go.yml

@@ -21,20 +21,20 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v2
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
+      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v2
 
     - name: Get dependencies
       run: |
         cd weed; go get -v -t -d ./...
 
     - name: Build
-      run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv" -v .
+      run: cd weed; go build -tags "elastic gocdk sqlite ydb tikv rclone" -v .
 
     - name: Test
-      run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
+      run: cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...

.github/workflows/helm_chart_release.yml

@@ -1,7 +1,7 @@
 name: "helm: publish charts"
 on:
   push:
-    tags: 
+    tags:
       - '*'
 
 permissions:
@@ -12,7 +12,7 @@ jobs:
   release:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
       - name: Publish Helm charts
         uses: stefanprodan/helm-gh-pages@master
         with:          

.github/workflows/helm_ci.yml

@@ -16,36 +16,36 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
+        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
         with:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v4
         with:
           version: v3.10.0
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           check-latest: true
 
       - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.4.0
+        uses: helm/chart-testing-action@v2.6.1
 
       - name: Run chart-testing (list-changed)
         id: list-changed
         run: |
-          changed=$(ct list-changed --chart-dirs k8s/charts)
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs k8s/charts)
           if [[ -n "$changed" ]]; then
             echo "::set-output name=changed::true"
           fi
 
       - name: Run chart-testing (lint)
-        run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
+        run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
 
       - name: Create kind cluster
-        uses: helm/kind-action@v1.8.0
+        uses: helm/kind-action@v1.10.0
 
       - name: Run chart-testing (install)
-        run: ct install --all --chart-dirs k8s/charts
+        run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts

.github/workflows/s3tests.yml

@@ -0,0 +1,200 @@
+name: "Ceph S3 tests"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+concurrency:
+  group: ${{ github.head_ref }}/s3tests
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    working-directory: docker
+
+jobs:
+  s3tests:
+    name: Ceph S3 tests
+    runs-on: ubuntu-22.04
+    container:
+      image: docker.io/kmlebedev/ceph-s3-tests:0.0.2
+    timeout-minutes: 30
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go 1.x
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: 'go.mod'
+        id: go
+
+      - name: Run Ceph S3 tests
+        timeout-minutes: 15
+        env:
+          S3TEST_CONF: /__w/seaweedfs/seaweedfs/docker/compose/s3tests.conf
+        shell: bash
+        run: |
+          cd /__w/seaweedfs/seaweedfs/weed
+          go install -buildvcs=false
+          set -x
+          nohup weed -v 0 server -filer -filer.maxMB=64 -s3 -ip.bind 0.0.0.0 \
+            -master.raftHashicorp -master.electionTimeout 1s -master.volumeSizeLimitMB=1024 \
+            -volume.max=100 -volume.preStopSeconds=1 -s3.port=8000 -metricsPort=9324 \
+            -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=../docker/compose/s3.json &
+          sleep 10          
+          cd /s3-tests
+          tox -- \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_distinct \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_many \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_ends_with_delimiter \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_prefix_underscore \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_percentage \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_whitespace \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_dot \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_delimiter_prefix_delimiter_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_fetchowner_notempty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_basic \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_alt \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_prefix_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_prefix_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_one \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_one \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_maxkeys_zero \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_maxkeys_zero \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_none \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_empty \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_continuationtoken \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_both_continuationtoken_startafter \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_unreadable \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_not_in_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_not_in_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_marker_after_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_startafter_after_list \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_objects_anonymous_fail \
+          s3tests_boto3/functional/test_s3.py::test_bucket_listv2_objects_anonymous_fail \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_long_name \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests_boto3/functional/test_s3.py::test_bucket_delete_notexist \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_read_not_exist \
+          s3tests_boto3/functional/test_s3.py::test_multi_object_delete \
+          s3tests_boto3/functional/test_s3.py::test_multi_objectv2_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_head_zero_bytes \
+          s3tests_boto3/functional/test_s3.py::test_object_write_check_etag \
+          s3tests_boto3/functional/test_s3.py::test_object_write_cache_control \
+          s3tests_boto3/functional/test_s3.py::test_object_write_expires \
+          s3tests_boto3/functional/test_s3.py::test_object_write_read_update_read_delete \
+          s3tests_boto3/functional/test_s3.py::test_object_metadata_replaced_on_put \
+          s3tests_boto3/functional/test_s3.py::test_object_write_file \
+          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_date_format \
+          s3tests_boto3/functional/test_s3.py::test_post_object_no_key_specified \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_signature \
+          s3tests_boto3/functional/test_s3.py::test_post_object_condition_is_case_sensitive \
+          s3tests_boto3/functional/test_s3.py::test_post_object_expires_is_case_sensitive \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_expires_condition \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_conditions_list \
+          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_limit_exceeded \
+          s3tests_boto3/functional/test_s3.py::test_post_object_missing_content_length_argument \
+          s3tests_boto3/functional/test_s3.py::test_post_object_invalid_content_length_argument \
+          s3tests_boto3/functional/test_s3.py::test_post_object_upload_size_below_minimum \
+          s3tests_boto3/functional/test_s3.py::test_post_object_empty_conditions \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmatch_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifnonematch_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_good \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifmodifiedsince_failed \
+          s3tests_boto3/functional/test_s3.py::test_get_object_ifunmodifiedsince_failed \
+          s3tests_boto3/functional/test_s3.py::test_bucket_head \
+          s3tests_boto3/functional/test_s3.py::test_bucket_head_notexist \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_bucket_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_authenticated_object_gone \
+          s3tests_boto3/functional/test_s3.py::test_object_raw_get_x_amz_expires_out_range_zero \
+          s3tests_boto3/functional/test_s3.py::test_object_anon_put \
+          s3tests_boto3/functional/test_s3.py::test_object_put_authenticated \
+          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_overwrite_acl \
+          s3tests_boto3/functional/test_s3.py::test_bucket_recreate_new_acl \
+          s3tests_boto3/functional/test_s3.py::test_buckets_create_then_list \
+          s3tests_boto3/functional/test_s3.py::test_buckets_list_ctime \
+          s3tests_boto3/functional/test_s3.py::test_list_buckets_invalid_auth \
+          s3tests_boto3/functional/test_s3.py::test_list_buckets_bad_auth \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_period \
+          s3tests_boto3/functional/test_s3.py::test_bucket_create_naming_good_contains_hyphen \
+          s3tests_boto3/functional/test_s3.py::test_bucket_list_special_prefix \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_zero_size \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_same_bucket \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_to_itself \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_diff_bucket \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_canned_acl \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_bucket_not_found \
+          s3tests_boto3/functional/test_s3.py::test_object_copy_key_not_found \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_small \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_without_range \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_special_names \
+          s3tests_boto3/functional/test_s3.py::test_multipart_copy_multiple_sizes \
+          s3tests_boto3/functional/test_s3.py::test_multipart_get_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_empty \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_multiple_sizes \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_contents \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_overwrite_existing_object \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_size_too_small \
+          s3tests_boto3/functional/test_s3.py::test_multipart_resend_first_finishes_last \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_resend_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_missing_part \
+          s3tests_boto3/functional/test_s3.py::test_multipart_upload_incorrect_etag \
+          s3tests_boto3/functional/test_s3.py::test_abort_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_list_multipart_upload \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_read_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_write_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_1mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_4mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_dual_write_8mb \
+          s3tests_boto3/functional/test_s3.py::test_atomic_multipart_upload_write \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_big_request_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_skip_leading_bytes_response_code \
+          s3tests_boto3/functional/test_s3.py::test_ranged_request_return_trailing_bytes_response_code \
+          s3tests_boto3/functional/test_s3.py::test_copy_object_ifmatch_good \
+          s3tests_boto3/functional/test_s3.py::test_copy_object_ifnonematch_failed \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_set \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_get \
+          s3tests_boto3/functional/test_s3.py::test_lifecycle_set_filter

.gitignore

@@ -87,4 +87,15 @@ other/java/hdfs/dependency-reduced-pom.xml
 
 # binary file
 weed/weed
-weed/mq/client/cmd/weed_pub/weed_pub
+weed/mq/client/cmd/weed_pub_kv/weed_pub
+docker/weed
+
+# test generated files
+weed/*/*.jpg
+docker/weed_sub
+docker/weed_pub
+weed/mq/schema/example.parquet
+docker/weed_pub_kv
+docker/weed_pub_record
+docker/weed_sub_kv
+docker/weed_sub_record

Makefile

@@ -1,14 +1,35 @@
 BINARY = weed
 
 SOURCE_DIR = .
+debug ?= 0
 
 all: install
 
 install:
 	cd weed; go install
 
-full_install:
-	cd weed; go install -tags "elastic gocdk sqlite ydb tikv"
+warp_install:
+	go install github.com/minio/warp@v0.7.6
 
-tests:
-	cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
+full_install:
+	cd weed; go install -tags "elastic gocdk sqlite ydb tikv rclone"
+
+server: install
+	weed -v 0 server -s3 -filer -filer.maxMB=64 -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=true -s3.config=./docker/compose/s3.json -metricsPort=9324
+
+benchmark: install warp_install
+	pkill weed || true
+	pkill warp || true
+	weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
+	warp client &
+	while ! nc -z localhost 8000 ; do sleep 1 ; done
+	warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
+	pkill warp
+	pkill weed
+
+# curl -o profile "http://127.0.0.1:6060/debug/pprof/profile?debug=1"
+benchmark_with_pprof: debug = 1
+benchmark_with_pprof: benchmark
+
+test:
+	cd weed; go test -tags "elastic gocdk sqlite ydb tikv rclone" -v ./...

README.md

@@ -374,7 +374,7 @@ Each individual file size is limited to the volume size.
 
 ### Saving memory ###
 
-All file meta information stored on an volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
+All file meta information stored on a volume server is readable from memory without disk access. Each file takes just a 16-byte map entry of <64bit key, 32bit offset, 32bit size>. Of course, each map entry has its own space cost for the map. But usually the disk space runs out before the memory does.
 
 ### Tiered Storage to the cloud ###
 
@@ -582,6 +582,71 @@ Percentage of the requests served within a certain time (ms)
   100%     54.1 ms
 ```
 
+### Run WARP and launch a mixed benchmark. ###
+
+```
+make benchmark
+warp: Benchmark data written to "warp-mixed-2023-10-16[102354]-l70a.csv.zst"                                                                                                                                                                                               
+Mixed operations.
+Operation: DELETE, 10%, Concurrency: 20, Ran 4m59s.
+ * Throughput: 6.19 obj/s
+
+Operation: GET, 45%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 279.85 MiB/s, 27.99 obj/s
+
+Operation: PUT, 15%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 89.86 MiB/s, 8.99 obj/s
+
+Operation: STAT, 30%, Concurrency: 20, Ran 5m0s.
+ * Throughput: 18.63 obj/s
+
+Cluster Total: 369.74 MiB/s, 61.79 obj/s, 0 errors over 5m0s.
+```
+
+To see segmented request statistics, use the --analyze.v parameter.
+```
+warp analyze --analyze.v warp-mixed-2023-10-16[102354]-l70a.csv.zst
+18642 operations loaded... Done!
+Mixed operations.
+----------------------------------------
+Operation: DELETE - total: 1854, 10.0%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
+ * Throughput: 6.19 obj/s
+
+Requests considered: 1855:
+ * Avg: 104ms, 50%: 30ms, 90%: 207ms, 99%: 1.355s, Fastest: 1ms, Slowest: 4.613s, StdDev: 320ms
+
+----------------------------------------
+Operation: GET - total: 8388, 45.3%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.12 +0500 +05
+ * Throughput: 279.77 MiB/s, 27.98 obj/s
+
+Requests considered: 8389:
+ * Avg: 221ms, 50%: 106ms, 90%: 492ms, 99%: 1.739s, Fastest: 8ms, Slowest: 8.633s, StdDev: 383ms
+ * TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 171ms, 99th: 669ms, Worst: 4.783s StdDev: 163ms
+ * First Access: Avg: 240ms, 50%: 105ms, 90%: 511ms, 99%: 2.08s, Fastest: 12ms, Slowest: 8.633s, StdDev: 480ms
+ * First Access TTFB: Avg: 88ms, Best: 2ms, 25th: 24ms, Median: 38ms, 75th: 64ms, 90th: 179ms, 99th: 919ms, Worst: 4.783s StdDev: 199ms
+ * Last Access: Avg: 219ms, 50%: 106ms, 90%: 463ms, 99%: 1.782s, Fastest: 9ms, Slowest: 8.633s, StdDev: 416ms
+ * Last Access TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 161ms, 99th: 657ms, Worst: 4.783s StdDev: 176ms
+
+----------------------------------------
+Operation: PUT - total: 2688, 14.5%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
+ * Throughput: 89.83 MiB/s, 8.98 obj/s
+
+Requests considered: 2689:
+ * Avg: 1.165s, 50%: 878ms, 90%: 2.015s, 99%: 5.74s, Fastest: 99ms, Slowest: 8.264s, StdDev: 968ms
+
+----------------------------------------
+Operation: STAT - total: 5586, 30.2%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.113 +0500 +05
+ * Throughput: 18.63 obj/s
+
+Requests considered: 5587:
+ * Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 80ms, Fastest: 0s, Slowest: 245ms, StdDev: 17ms
+ * First Access: Avg: 14ms, 50%: 10ms, 90%: 33ms, 99%: 69ms, Fastest: 0s, Slowest: 203ms, StdDev: 16ms
+ * Last Access: Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 74ms, Fastest: 0s, Slowest: 203ms, StdDev: 17ms
+
+Cluster Total: 369.64 MiB/s, 61.77 obj/s, 0 errors over 5m0s.
+Total Errors:0.
+```
+
 [Back to TOC](#table-of-contents)
 
 ## License ##

docker/Dockerfile.gccgo_build

@@ -1,44 +0,0 @@
-FROM gcc:11 as builder
-RUN mkdir -p /go/src/github.com/seaweedfs/
-RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
-ARG BRANCH=${BRANCH:-master}
-RUN cd /go/src/github.com/seaweedfs/seaweedfs && git checkout $BRANCH
-RUN cd /go/src/github.com/seaweedfs/seaweedfs/weed \
-  && apt-get update \
-  && apt-get install -y golang-src \
-  && export LDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
-  && CGO_ENABLED=0 go install -ldflags "-extldflags -static ${LDFLAGS}" -compiler=gccgo -tags gccgo,noasm
-
-FROM alpine AS final
-LABEL author="Chris Lu"
-COPY --from=builder /go/bin/weed /usr/bin/
-RUN mkdir -p /etc/seaweedfs
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
-COPY --from=builder /go/src/github.com/seaweedfs/seaweedfs/docker/entrypoint.sh /entrypoint.sh
-RUN apk add fuse # for weed mount
-
-# volume server gprc port
-EXPOSE 18080
-# volume server http port
-EXPOSE 8080
-# filer server gprc port
-EXPOSE 18888
-# filer server http port
-EXPOSE 8888
-# master server shared gprc port
-EXPOSE 19333
-# master server shared http port
-EXPOSE 9333
-# s3 server http port
-EXPOSE 8333
-# webdav server http port
-EXPOSE 7333
-
-RUN mkdir -p /data/filerldb2
-
-VOLUME /data
-WORKDIR /data
-
-RUN chmod +x /entrypoint.sh
-
-ENTRYPOINT ["/entrypoint.sh"]

docker/Dockerfile.go_build

@@ -1,4 +1,4 @@
-FROM golang:1.20-alpine as builder
+FROM golang:1.22-alpine as builder
 RUN apk add git g++ fuse
 RUN mkdir -p /go/src/github.com/seaweedfs/
 RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs

docker/Dockerfile.local

@@ -1,6 +1,8 @@
 FROM alpine AS final
 LABEL author="Chris Lu"
 COPY  ./weed /usr/bin/
+COPY  ./weed_pub* /usr/bin/
+COPY  ./weed_sub* /usr/bin/
 RUN mkdir -p /etc/seaweedfs
 COPY ./filer.toml /etc/seaweedfs/filer.toml
 COPY ./entrypoint.sh /entrypoint.sh

docker/Dockerfile.rocksdb_dev_env

@@ -1,9 +1,9 @@
-FROM golang:1.20-buster as builder
+FROM golang:1.22 as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.1.1
+ENV ROCKSDB_VERSION v8.10.0
 
 # build RocksDB
 RUN cd /tmp && \

docker/Dockerfile.rocksdb_large

@@ -1,9 +1,9 @@
-FROM golang:1.20-buster as builder
+FROM golang:1.22 as builder
 
 RUN apt-get update
 RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
 
-ENV ROCKSDB_VERSION v8.1.1
+ENV ROCKSDB_VERSION v8.10.0
 
 # build RocksDB
 RUN cd /tmp && \

docker/Makefile

@@ -9,6 +9,10 @@ binary:
 	export SWCOMMIT=$(shell git rev-parse --short HEAD)
 	export SWLDFLAGS="-X github.com/seaweedfs/seaweedfs/weed/util.COMMIT=$(SWCOMMIT)"
 	cd ../weed && CGO_ENABLED=$(cgo) GOOS=linux go build $(options) -tags "$(tags)" -ldflags "-s -w -extldflags -static $(SWLDFLAGS)" && mv weed ../docker/
+	cd ../weed/mq/client/cmd/weed_pub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_kv ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_pub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_pub_record ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_sub_kv && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_kv ../../../../../docker/
+	cd ../weed/mq/client/cmd/weed_sub_record && CGO_ENABLED=$(cgo) GOOS=linux go build && mv weed_sub_record ../../../../../docker/
 
 binary_race: options = -race
 binary_race: cgo = 1
@@ -81,12 +85,18 @@ cluster: build
 2mount: build
 	docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
 
+filer_backup: build
+	docker compose -f compose/local-filer-backup-compose.yml -p seaweedfs up
+
 hashicorp_raft: build
 	docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
 
 s3tests: build s3tests_build
 	docker compose -f compose/local-s3tests-compose.yml -p seaweedfs up
 
+brokers: build
+	docker compose -f compose/local-brokers-compose.yml -p seaweedfs up
+
 filer_etcd: build
 	docker stack deploy -c compose/swarm-etcd.yml fs
 
@@ -102,12 +112,12 @@ clean:
 	rm ./weed
 
 certstrap:
-	go get github.com/square/certstrap
-	certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
-	certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
+	go install -v github.com/square/certstrap@latest
+	certstrap --depot-path compose/tls init --curve P-256 --passphrase "" --common-name "SeaweedFS CA" || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --domain localhost --common-name volume01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name master01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name filer01.dev || true
+	certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name client01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
 	certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true

docker/compose/local-brokers-compose.yml

@@ -0,0 +1,127 @@
+version: '3.9'
+
+services:
+  master0:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master0 -port=9333 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  master1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9334:9334
+      - 19334:19334
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master1 -port=9334 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  master2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9335:9335
+      - 19335:19335
+    command: "-v=0 master -volumeSizeLimitMB 100 -resumeState=false -ip=master2 -port=9335 -peers=master0:9333,master1:9334,master2:9335 -mdir=/tmp"
+    environment:
+      WEED_MASTER_VOLUME_GROWTH_COPY_1: 1
+      WEED_MASTER_VOLUME_GROWTH_COPY_2: 2
+      WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+  volume1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8080:8080
+      - 18080:18080
+    command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  volume2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8082:8082
+      - 18082:18082
+    command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  volume3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8083:8083
+      - 18083:18083
+    command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+  filer1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8888:8888
+      - 18888:18888
+    command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335" -port=8888 -ip=filer1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+  filer2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 8889:8889
+      - 18889:18889
+    command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335" -port=8889 -ip=filer2'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - filer1
+  broker1:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17777:17777
+    command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17777 -ip=broker1'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - filer1
+      - filer2
+  broker2:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17778:17778
+    command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17778 -ip=broker2'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - filer1
+      - filer2
+  broker3:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17779:17779
+    command: 'mq.broker -master="master0:9333,master1:9334,master2:9335" -port=17779 -ip=broker3'
+    depends_on:
+      - master0
+      - master1
+      - master2
+      - volume1
+      - volume2
+      - filer1
+      - filer2

docker/compose/local-filer-backup-compose.yml

@@ -0,0 +1,54 @@
+version: '3.9'
+
+services:
+  server-left:
+    image: chrislusf/seaweedfs:local
+    command: "-v=0 server -ip=server-left -filer -filer.maxMB 5 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
+      interval: 3s
+      start_period: 15s
+      timeout: 30s
+  server-right:
+    image: chrislusf/seaweedfs:local
+    command: "-v=0 server -ip=server-right -filer -filer.maxMB 64 -s3 -s3.config=/etc/seaweedfs/s3.json -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1"
+    volumes:
+      - ./s3.json:/etc/seaweedfs/s3.json
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
+      interval: 3s
+      start_period: 15s
+      timeout: 30s
+  filer-backup:
+    image: chrislusf/seaweedfs:local
+    command: "-v=0 filer.backup -debug -doDeleteFiles=False -filer server-left:8888"
+    volumes:
+      - ./replication.toml:/etc/seaweedfs/replication.toml
+    environment:
+      WEED_SINK_LOCAL_INCREMENTAL_ENABLED: "false"
+      WEED_SINK_S3_ENABLED: "true"
+      WEED_SINK_S3_BUCKET: "backup"
+      WEED_SINK_S3_ENDPOINT: "http://server-right:8333"
+      WEED_SINK_S3_DIRECTORY: "/"
+      WEED_SINK_S3_AWS_ACCESS_KEY_ID: "some_access_key1"
+      WEED_SINK_S3_AWS_SECRET_ACCESS_KEY: "some_secret_key1"
+      WEED_SINK_S3_S3_DISABLE_CONTENT_MD5_VALIDATION: "false"
+      WEED_SINK_S3_UPLOADER_PART_SIZE_MB: "5"
+      WEED_SINK_S3_KEEP_PART_SIZE: "false"
+    depends_on:
+      server-left:
+        condition: service_healthy
+      server-right:
+        condition: service_healthy
+  minio-warp:
+    image: minio/warp
+    command: 'mixed --duration 5s --obj.size=6mb --md5 --objects 10 --concurrent 2'
+    restart: on-failure
+    environment:
+      WARP_HOST: "server-left:8333"
+      WARP_ACCESS_KEY: "some_access_key1"
+      WARP_SECRET_KEY: "some_secret_key1"
+    depends_on:
+      - filer-backup

docker/compose/local-mq-test.yml

@@ -0,0 +1,27 @@
+version: '3.9'
+
+services:
+  server:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 9333:9333
+      - 19333:19333
+      - 8888:8888
+      - 18888:18888
+    command: "server -ip=server -filer -volume.max=0 -master.volumeSizeLimitMB=8 -volume.preStopSeconds=1"
+    healthcheck:
+      test: curl -f http://localhost:8888/healthz
+  mq_broker:
+    image: chrislusf/seaweedfs:local
+    ports:
+      - 17777:17777
+    command: "mq.broker -master=server:9333 -ip=mq_broker"
+    depends_on:
+      server:
+        condition: service_healthy
+  mq_client:
+    image: chrislusf/seaweedfs:local
+    # run a custom command instead of entrypoint
+    command: "ls -al"
+    depends_on:
+      - mq_broker

docker/compose/local-sync-mount-compose.yml

@@ -3,19 +3,54 @@ services:
   node1:
     image: chrislusf/seaweedfs:local
     command: "server -master -volume -filer"
+    ports:
+      - 8888:8888
+      - 18888:18888
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
+      interval: 1s
+      start_period: 10s
+      timeout: 30s
   mount1:
     image: chrislusf/seaweedfs:local
     privileged: true
     command: "mount -filer=node1:8888 -dir=/mnt -dirAutoCreate"
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://node1:8888/" ]
+      interval: 1s
+      start_period: 10s
+      timeout: 30s
+    depends_on:
+      node1:
+        condition: service_healthy
   node2:
     image: chrislusf/seaweedfs:local
     ports:
       - 7888:8888
+      - 17888:18888
     command: "server -master -volume -filer"
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://localhost:9333/cluster/healthz" ]
+      interval: 1s
+      start_period: 10s
+      timeout: 30s
   mount2:
     image: chrislusf/seaweedfs:local
     privileged: true
     command: "mount -filer=node2:8888 -dir=/mnt -dirAutoCreate"
+    healthcheck:
+      test: [ "CMD", "curl", "--fail", "-I", "http://node2:8888/" ]
+      interval: 1s
+      start_period: 10s
+      timeout: 30s
+    depends_on:
+      node2:
+        condition: service_healthy
   sync:
     image: chrislusf/seaweedfs:local
     command: "-v=4 filer.sync -a=node1:8888 -b=node2:8888 -a.debug -b.debug"
+    depends_on:
+      mount1:
+        condition: service_healthy
+      mount2:
+        condition: service_healthy

docker/compose/s3.json

@@ -40,7 +40,10 @@
         "List",
         "Tagging",
         "Write"
-      ]
+      ],
+      "account": {
+        "id": "testid"
+      }
     },
     {
       "name": "s3_tests_alt",
@@ -101,5 +104,12 @@
         "Write"
       ]
     }
-  ]
+  ],
+  "accounts": [
+      {
+        "id" : "testid",
+        "displayName": "M. Tester",
+        "emailAddress": "tester@ceph.com"
+      }
+    ]
 }

docker/compose/s3tests.conf

@@ -2,7 +2,7 @@
 ## this section is just used for host, port and bucket_prefix
 
 # host set for rgw in vstart.sh
-host = s3
+host = 127.0.0.1
 
 # port set for rgw in vstart.sh
 port = 8000
@@ -18,10 +18,10 @@ bucket prefix = yournamehere-{random}-
 
 [s3 main]
 # main display_name set in vstart.sh
-display_name = s3_tests
+display_name = M. Tester
 
 # main user_idname set in vstart.sh
-user_id = s3_tests
+user_id = testid
 
 # main email set in vstart.sh
 email = tester@ceph.com

docker/compose/test-etcd-filer.yml

@@ -30,6 +30,7 @@ services:
     environment:
       WEED_LEVELDB2_ENABLED: 'false'
       WEED_ETCD_ENABLED: 'true'
+      WEED_ETCD_KEY_PREFIX: 'seaweedfs.'
       WEED_ETCD_SERVERS: "http://etcd:2379"
     volumes:
       - ./s3.json:/etc/seaweedfs/s3.json

docker/compose/tls.env

@@ -12,5 +12,9 @@ WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,clie
 WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
 WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
 WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
+WEED_HTTPS_CLIENT_ENABLE=true
+WEED_HTTPS_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
+WEED_HTTPS_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
+WEED_HTTPS_VOLUME_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
 #GRPC_GO_LOG_SEVERITY_LEVEL=info
 #GRPC_GO_LOG_VERBOSITY_LEVEL=2

go.mod

@@ -1,24 +1,23 @@
 module github.com/seaweedfs/seaweedfs
 
-go 1.20
+go 1.22.0
 
 require (
-	cloud.google.com/go v0.110.7 // indirect
-	cloud.google.com/go/pubsub v1.33.0
-	cloud.google.com/go/storage v1.32.0
+	cloud.google.com/go v0.112.1 // indirect
+	cloud.google.com/go/pubsub v1.37.0
+	cloud.google.com/go/storage v1.40.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.44.314
+	github.com/aws/aws-sdk-go v1.51.30
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
-	github.com/cenkalti/backoff/v4 v4.2.1
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.1
 	github.com/eapache/go-resiliency v1.3.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
@@ -28,24 +27,22 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
-	github.com/fclairamb/ftpserverlib v0.22.0
-	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-errors/errors v1.1.1 // indirect
+	github.com/fclairamb/ftpserverlib v0.24.0
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-redis/redis/v8 v8.11.5
-	github.com/go-redsync/redsync/v4 v4.8.2
-	github.com/go-sql-driver/mysql v1.7.1
+	github.com/go-redsync/redsync/v4 v4.13.0
+	github.com/go-sql-driver/mysql v1.8.1
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.6.0
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3
+	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/btree v1.1.2
-	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.3.1
-	github.com/google/wire v0.5.0 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
-	github.com/gorilla/mux v1.8.0
+	github.com/google/uuid v1.6.0
+	github.com/google/wire v0.6.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.3 // indirect
+	github.com/gorilla/mux v1.8.1
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -57,264 +54,294 @@ require (
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12
 	github.com/karlseguin/ccache/v2 v2.0.8
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
-	github.com/klauspost/compress v1.16.7 // indirect
-	github.com/klauspost/reedsolomon v1.11.8
+	github.com/klauspost/compress v1.17.7 // indirect
+	github.com/klauspost/reedsolomon v1.12.1
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.9
-	github.com/linxGnu/grocksdb v1.8.0
+	github.com/linxGnu/grocksdb v1.8.14
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.11 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/olivere/elastic/v7 v7.0.32
 	github.com/peterh/liner v1.2.2
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/posener/complete v1.2.3
 	github.com/pquerna/cachecontrol v0.2.0
-	github.com/prometheus/client_golang v1.16.0
-	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.42.0 // indirect
-	github.com/prometheus/procfs v0.11.1
+	github.com/prometheus/client_golang v1.19.0
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
+	github.com/prometheus/procfs v0.13.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
 	github.com/seaweedfs/raft v1.1.3
-	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/spf13/afero v1.9.5 // indirect
-	github.com/spf13/cast v1.5.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/viper v1.16.0
-	github.com/stretchr/testify v1.8.4
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/viper v1.18.2
+	github.com/stretchr/testify v1.9.0
 	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tidwall/gjson v1.16.0
+	github.com/tidwall/gjson v1.17.1
 	github.com/tidwall/match v1.1.1
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
-	github.com/viant/assertly v0.5.4 // indirect
-	github.com/viant/ptrie v0.3.0
-	github.com/viant/toolbox v0.33.2 // indirect
+	github.com/viant/ptrie v1.0.1
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
-	go.etcd.io/etcd/client/v3 v3.5.9
-	go.mongodb.org/mongo-driver v1.12.1
+	go.etcd.io/etcd/client/v3 v3.5.13
+	go.mongodb.org/mongo-driver v1.15.0
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.34.0
-	gocloud.dev/pubsub/natspubsub v0.33.0
-	gocloud.dev/pubsub/rabbitpubsub v0.33.0
-	golang.org/x/crypto v0.13.0 // indirect
-	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
-	golang.org/x/image v0.11.0
-	golang.org/x/net v0.15.0
-	golang.org/x/oauth2 v0.12.0 // indirect
-	golang.org/x/sys v0.12.0
-	golang.org/x/text v0.13.0 // indirect
-	golang.org/x/tools v0.11.0
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.141.0
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
-	google.golang.org/grpc v1.58.0
-	google.golang.org/protobuf v1.31.0
+	gocloud.dev v0.37.0
+	gocloud.dev/pubsub/natspubsub v0.37.0
+	gocloud.dev/pubsub/rabbitpubsub v0.37.0
+	golang.org/x/crypto v0.22.0 // indirect
+	golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3
+	golang.org/x/image v0.15.0
+	golang.org/x/net v0.24.0
+	golang.org/x/oauth2 v0.19.0 // indirect
+	golang.org/x/sys v0.19.0
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/tools v0.19.0
+	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
+	google.golang.org/api v0.176.0
+	google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
+	google.golang.org/grpc v1.63.2
+	google.golang.org/protobuf v1.33.0
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.24.1 // indirect
-	modernc.org/mathutil v1.6.0 // indirect
-	modernc.org/memory v1.6.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.25.0
-	modernc.org/strutil v1.1.3
-	modernc.org/token v1.0.1 // indirect
+	modernc.org/libc v1.49.3 // indirect
+	modernc.org/mathutil v1.6.0
+	modernc.org/memory v1.8.0 // indirect
+	modernc.org/sqlite v1.29.8
+	modernc.org/strutil v1.2.0
+	modernc.org/token v1.1.0 // indirect
 )
 
 require (
-	github.com/Jille/raft-grpc-transport v1.4.0
-	github.com/arangodb/go-driver v1.6.0
+	github.com/Jille/raft-grpc-transport v1.5.0
+	github.com/arangodb/go-driver v1.6.2
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go-v2 v1.21.0
-	github.com/aws/aws-sdk-go-v2/config v1.18.39
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.37
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
+	github.com/aws/aws-sdk-go-v2 v1.26.1
+	github.com/aws/aws-sdk-go-v2/config v1.27.11
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1
+	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
-	github.com/golang-jwt/jwt/v5 v5.0.0
+	github.com/getsentry/sentry-go v0.27.0
+	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
-	github.com/hanwen/go-fuse/v2 v2.4.0
-	github.com/hashicorp/raft v1.5.0
-	github.com/hashicorp/raft-boltdb/v2 v2.2.2
+	github.com/hanwen/go-fuse/v2 v2.5.0
+	github.com/hashicorp/raft v1.6.1
+	github.com/hashicorp/raft-boltdb/v2 v2.3.0
 	github.com/orcaman/concurrent-map/v2 v2.0.1
-	github.com/puzpuzpuz/xsync/v2 v2.5.0
-	github.com/rabbitmq/amqp091-go v1.8.1
-	github.com/rclone/rclone v1.63.1
-	github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8
-	github.com/schollz/progressbar/v3 v3.13.1
-	github.com/shirou/gopsutil/v3 v3.23.5
+	github.com/parquet-go/parquet-go v0.20.1
+	github.com/rabbitmq/amqp091-go v1.9.0
+	github.com/rclone/rclone v1.66.0
+	github.com/rdleal/intervalst v1.4.0
+	github.com/schollz/progressbar/v3 v3.14.2
+	github.com/shirou/gopsutil/v3 v3.24.3
 	github.com/tikv/client-go/v2 v2.0.7
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.48.8
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0
+	go.etcd.io/etcd/client/pkg/v3 v3.5.13
+	go.uber.org/atomic v1.11.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
 require (
-	cloud.google.com/go/compute v1.23.0 // indirect
-	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.1 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 // indirect
+	cloud.google.com/go/auth v0.2.2 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.1 // indirect
+	cloud.google.com/go/compute/metadata v0.3.0 // indirect
+	cloud.google.com/go/iam v1.1.7 // indirect
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azfile v1.1.1 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
-	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
+	github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
+	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
+	github.com/ProtonMail/go-srp v0.0.7 // indirect
+	github.com/ProtonMail/gopenpgp/v2 v2.7.4 // indirect
+	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/Unknwon/goconfig v1.0.0 // indirect
 	github.com/abbot/go-http-auth v0.4.0 // indirect
+	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect
-	github.com/aws/smithy-go v1.14.2 // indirect
-	github.com/benbjohnson/clock v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
+	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/bradenaw/juniper v0.15.2 // indirect
 	github.com/buengese/sgzip v0.1.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
-	github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cloudsoda/go-smb2 v0.0.0-20231124195312-f3ec8ae2c891 // indirect
+	github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
+	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
 	github.com/elastic/gosigar v0.14.2 // indirect
+	github.com/emersion/go-message v0.18.0 // indirect
+	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
+	github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
 	github.com/fatih/color v1.14.1 // indirect
-	github.com/fclairamb/go-log v0.4.1 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
+	github.com/fclairamb/go-log v0.5.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/flynn/noise v1.0.1 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
-	github.com/go-chi/chi/v5 v5.0.8 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-chi/chi/v5 v5.0.11 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-resty/resty/v2 v2.11.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
-	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-hclog v1.6.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack v1.1.5 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
-	github.com/hirochachacha/go-smb2 v1.1.0 // indirect
-	github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
+	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
+	github.com/henrybear327/Proton-API-Bridge v1.0.0 // indirect
+	github.com/henrybear327/go-proton-api v1.0.0 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+	github.com/jlaffaye/ftp v0.2.0 // indirect
 	github.com/jonboulle/clockwork v0.3.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
+	github.com/jtolio/eventkit v0.0.0-20231019094657-5d77ebb407d9 // indirect
+	github.com/jtolio/noiseconn v0.0.0-20231127013910-f6d9ecbf1de7 // indirect
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20231016141302-07b5767bb0ed // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.14 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/montanaflynn/stats v0.7.0 // indirect
-	github.com/nats-io/nats.go v1.28.0 // indirect
-	github.com/nats-io/nkeys v0.4.4 // indirect
+	github.com/nats-io/nats.go v1.33.1 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
-	github.com/ncw/swift/v2 v2.0.1 // indirect
+	github.com/ncruces/go-strftime v0.1.9 // indirect
+	github.com/ncw/swift/v2 v2.0.2 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/onsi/gomega v1.27.10 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/oracle/oci-go-sdk/v65 v65.34.0 // indirect
+	github.com/oracle/oci-go-sdk/v65 v65.55.1 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
-	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
-	github.com/philhofer/fwd v1.1.1 // indirect
-	github.com/pierrec/lz4/v4 v4.1.17 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
 	github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
-	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
-	github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/pkg/sftp v1.13.6 // indirect
 	github.com/pkg/xattr v0.4.9 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
-	github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d // indirect
-	github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8 // indirect
+	github.com/relvacode/iso8601 v1.3.0 // indirect
 	github.com/rfjakob/eme v1.1.2 // indirect
-	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/segmentio/encoding v0.3.6 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/smartystreets/goconvey v1.8.1 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.22 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.4.2 // indirect
-	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/t3rm1n4l/go-mega v0.0.0-20240219080617-d494b6a8ace7 // indirect
 	github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
 	github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
-	github.com/tinylib/msgp v1.1.6 // indirect
-	github.com/tklauser/go-sysconf v0.3.11 // indirect
-	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/tinylib/msgp v1.1.8 // indirect
+	github.com/tklauser/go-sysconf v0.3.13 // indirect
+	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
-	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222 // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf // indirect
 	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
-	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
-	go.etcd.io/bbolt v1.3.7 // indirect
-	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
-	go.uber.org/atomic v1.11.0 // indirect
+	go.etcd.io/bbolt v1.3.8 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.13 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
-	golang.org/x/mod v0.12.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.12.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect
+	go.uber.org/zap v1.27.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/sync v0.6.0 // indirect
+	golang.org/x/term v0.19.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
+	gopkg.in/validator.v2 v2.0.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
-	storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
-	storj.io/drpc v0.0.32 // indirect
-	storj.io/uplink v1.10.0 // indirect
+	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
+	storj.io/common v0.0.0-20240111121419-ecae1362576c // indirect
+	storj.io/drpc v0.0.33 // indirect
+	storj.io/infectious v0.0.2 // indirect
+	storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
+	storj.io/uplink v1.12.2 // indirect
 )
 
 // replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft

k8s/charts/seaweedfs/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "3.57"
-version: 3.57.0
+appVersion: "3.66"
+version: 3.66.0

k8s/charts/seaweedfs/README.md

@@ -4,15 +4,21 @@
 
 ### Add the helm repo
 
-`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
+```bash
+helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm
+```
 
 ### Install the helm chart
 
-`helm install seaweedfs seaweedfs/seaweedfs`
+```bash
+helm install seaweedfs seaweedfs/seaweedfs
+```
 
 ### (Recommended) Provide `values.yaml`
 
-`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
+```bash
+helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs
+```
 
 ## Info:
 * master/filer/volume are stateful sets with anti-affinity on the hostname,
@@ -24,11 +30,11 @@ so your deployment will be spread/HA.
 ## Prerequisites
 ### Database
 
-leveldb is the default database this only supports one filer replica.
+leveldb is the default database, this supports multiple filer replicas that will [sync automatically](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication), with some [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation).
 
-To have multiple filers a external datastore is recommened.
+When the [limitations](https://github.com/seaweedfs/seaweedfs/wiki/Filer-Store-Replication#limitation) apply, or for a large number of filer replicas, an external datastore is recommended.
 
-Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`. 
+Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
 This database should be pre-configured and initialized by running:
 ```sql
 CREATE TABLE IF NOT EXISTS `filemeta` (
@@ -78,4 +84,63 @@ for your deployment.
 You can update the replicas count for each node type in values.yaml,
 need to add more nodes with the corresponding labels if applicable.
 
-Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
+Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
+
+## S3 configuration
+
+To enable an s3 endpoint for your filer with a default install add the following to your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+```
+
+### Enabling Authentication to S3
+
+To enable authentication for S3, you have two options:
+
+- let the helm chart create an admin user as well as a read only user
+- provide your own s3 config.json file via an existing Kubernetes Secret
+
+#### Use the default credentials for S3
+
+Example parameters for your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+    enableAuth: true
+```
+
+#### Provide your own credentials for S3
+
+Example parameters for your values.yaml:
+
+```yaml
+filer:
+  s3:
+    enabled: true
+    enableAuth: true
+    existingConfigSecret: my-s3-secret
+```
+
+Example existing secret with your s3 config to create an admin user and readonly user, both with credentials:
+
+```yaml
+---
+# Source: seaweedfs/templates/seaweedfs-s3-secret.yaml
+apiVersion: v1
+kind: Secret
+type: Opaque
+metadata:
+  name: my-s3-secret
+  namespace: seaweedfs
+  labels:
+    app.kubernetes.io/name: seaweedfs
+    app.kubernetes.io/component: s3
+stringData:
+  # this key must be an inline json config file
+  seaweedfs_s3_config: '{"identities":[{"name":"anvAdmin","credentials":[{"accessKey":"snu8yoP6QAlY0ne4","secretKey":"PNzBcmeLNEdR0oviwm04NQAicOrDH1Km"}],"actions":["Admin","Read","Write"]},{"name":"anvReadOnly","credentials":[{"accessKey":"SCigFee6c5lbi04A","secretKey":"kgFhbT38R8WUYVtiFQ1OiSVOrYr3NKku"}],"actions":["Read"]}]}'
+```

k8s/charts/seaweedfs/templates/_helpers.tpl

@@ -118,20 +118,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (or (eq .Values.volume.data.type "persistentVolumeClaim") (and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Volume HostPath exists */}}
-{{- define "volume.hostpath_exists" -}}
-{{- if or (or (eq .Values.volume.data.type "hostPath") (and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx )) (eq .Values.volume.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
-{{- end -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -140,16 +127,7 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Filer HostPath exists */}}
-{{- define "filer.hostpath_exists" -}}
-{{- if or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -158,29 +136,16 @@ Inject extra environment vars in the format key:value, if populated
 {{- if or (eq .Values.master.data.type "persistentVolumeClaim") (eq .Values.master.logs.type "persistentVolumeClaim") -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
-{{- end -}}
-{{- end -}}
-
-{{/* check if any Master HostPath exists */}}
-{{- define "master.hostpath_exists" -}}
-{{- if or (eq .Values.master.data.type "hostPath") (eq .Values.master.logs.type "hostPath") -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- if or .Values.global.enableSecurity .Values.volume.extraVolumes -}}
-{{- printf "true" -}}
-{{- else -}}
-{{- printf "false" -}}
-{{- end -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
 {{/* check if any InitContainers exist for Volumes */}}
 {{- define "volume.initContainers_exists" -}}
-{{- if or (not (empty .Values.volume.dir_idx )) (not (empty .Values.volume.initContainers )) -}}
+{{- if or (not (empty .Values.volume.idx )) (not (empty .Values.volume.initContainers )) -}}
 {{- printf "true" -}}
 {{- else -}}
-{{- printf "false" -}}
+{{- printf "" -}}
 {{- end -}}
 {{- end -}}
 
@@ -198,3 +163,22 @@ imagePullSecrets:
 {{- end }}
 {{- end }}
 {{- end -}}
+
+{{/*
+Renders a value that contains template perhaps with scope if the scope is present.
+Usage:
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }}
+{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }}
+*/}}
+{{- define "common.tplvalues.render" -}}
+{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }}
+{{- if contains "{{" (toJson .value) }}
+  {{- if .scope }}
+      {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }}
+  {{- else }}
+    {{- tpl $value .context }}
+  {{- end }}
+{{- else }}
+    {{- $value }}
+{{- end }}
+{{- end -}}

k8s/charts/seaweedfs/templates/ca-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/cert-caissuer.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Issuer
 metadata:

k8s/charts/seaweedfs/templates/cert-clusterissuer.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: ClusterIssuer
 metadata:

k8s/charts/seaweedfs/templates/client-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/filer-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/filer-ingress.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 {{- if .Values.filer.ingress.enabled }}
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
 apiVersion: networking.k8s.io/v1
@@ -10,9 +11,9 @@ kind: Ingress
 metadata:
   name: ingress-{{ template "seaweedfs.name" . }}-filer
   namespace: {{ .Release.Namespace }}
-  {{- if .Values.filer.ingress.annotations }}
+  {{- with .Values.filer.ingress.annotations }}
   annotations:
-    {{ tpl .Values.filer.ingress.annotations . | nindent 4 | trim }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
@@ -43,4 +44,5 @@ spec:
 {{- if .Values.filer.ingress.host }}
     host: {{ .Values.filer.ingress.host }}
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/filer-service-client.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -32,3 +33,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/component: filer
+{{- end }}

k8s/charts/seaweedfs/templates/filer-service.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -44,3 +45,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/component: filer
+{{- end }}

k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 {{- if .Values.filer.metricsPort }}
 {{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
@@ -21,4 +22,5 @@ spec:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
       app.kubernetes.io/component: filer
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -12,7 +12,7 @@ metadata:
     app.kubernetes.io/component: filer
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-filer
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.filer.podManagementPolicy }}
   replicas: {{ .Values.filer.replicas }}
   {{- if (gt (int .Values.filer.updatePartition) 0) }}
   updateStrategy:
@@ -23,7 +23,6 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: filer
   template:
@@ -33,6 +32,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: filer
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }}
       {{- if .Values.filer.affinity }}
@@ -45,16 +51,13 @@ spec:
       {{- end }}
       {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
       {{- if .Values.global.createClusterRole }}
-      serviceAccountName: seaweedfs-rw-sa #hack for delete pod master after migration
+      serviceAccountName: {{ .Values.filer.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       terminationGracePeriodSeconds: 60
       {{- if .Values.filer.priorityClassName }}
       priorityClassName: {{ .Values.filer.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.filer.serviceAccountName }}
-      serviceAccountName: {{ .Values.filer.serviceAccountName | quote }}
-      {{- end }}
       {{- if .Values.filer.initContainers }}
       initContainers:
         {{ tpl .Values.filer.initContainers . | nindent 8 | trim }}
@@ -102,7 +105,12 @@ spec:
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
             {{- if .Values.filer.secretExtraEnvironmentVars }}
@@ -110,11 +118,11 @@ spec:
             - name: {{ $key }}
               valueFrom: {{ toYaml $value | nindent 16 }}
             {{- end }}
-            {{- end }}             
+            {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
-            - | 
+            - |
               exec /usr/bin/weed \
               {{- if eq .Values.filer.logs.type "hostPath" }}
               -logdir=/logs \
@@ -182,15 +190,17 @@ spec:
               -s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
               {{- end }}
               {{- end }}
-              -master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+              -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
           volumeMounts:
             {{- if eq .Values.filer.logs.type "hostPath" }}
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
             {{- end }}
-            - mountPath: /etc/sw
-              name: config-users
+            {{- if .Values.filer.s3.enableAuth }}
+            - name: config-users
+              mountPath: /etc/sw
               readOnly: true
+            {{- end }}
             {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
             - name: data-filer
               mountPath: /data
@@ -252,6 +262,9 @@ spec:
           resources:
             {{ tpl .Values.filer.resources . | nindent 12 | trim }}
           {{- end }}
+        {{- if .Values.filer.sidecars }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }}
+        {{- end }}
       volumes:
         {{- if eq .Values.filer.logs.type "hostPath" }}
         - name: seaweedfs-filer-log-volume
@@ -259,19 +272,35 @@ spec:
             path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.filer.logs.type "existingClaim" }}
+        - name: seaweedfs-filer-log-volume
+          persistentVolumeClaim:
+            claimName: {{ .Values.filer.logs.claimName }}
+        {{- end }}
         {{- if eq .Values.filer.data.type "hostPath" }}
         - name: data-filer
           hostPath:
             path: {{ .Values.filer.data.hostPathPrefix }}/filer_store
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.filer.data.type "existingClaim" }}
+        - name: data-filer
+          persistentVolumeClaim:
+            claimName: {{ .Values.filer.data.claimName }}
+        {{- end }}
         - name: db-schema-config-volume
           configMap:
             name: seaweedfs-db-init-config
+        {{- if and .Values.filer.s3.enabled .Values.filer.s3.enableAuth }}
         - name: config-users
           secret:
             defaultMode: 420
+            {{- if .Values.filer.s3.existingConfigSecret }}
+            secretName: {{ .Values.filer.s3.existingConfigSecret }}
+            {{- else }}
             secretName: seaweedfs-s3-secret
+            {{- end }}
+        {{- end }}
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:
@@ -297,7 +326,7 @@ spec:
       nodeSelector:
         {{ tpl .Values.filer.nodeSelector . | indent 8 | trim }}
       {{- end }}
-  {{- if .Values.filer.enablePVC }}
+  {{- if and (.Values.filer.enablePVC) (eq .Values.filer.data.type "persistentVolumeClaim") }}
   # DEPRECATION: Deprecate in favor of filer.data section below
   volumeClaimTemplates:
   - metadata:
@@ -315,9 +344,13 @@ spec:
   {{- $pvc_exists := include "filer.pvc_exists" . -}}
   {{- if $pvc_exists }}
   volumeClaimTemplates:
-    {{- if eq .Values.filer.data.type "persistentVolumeClaim"}}
+    {{- if eq .Values.filer.data.type "persistentVolumeClaim" }}
     - metadata:
         name: data-filer
+        {{- with .Values.filer.data.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.filer.data.storageClass }}
@@ -325,9 +358,13 @@ spec:
           requests:
             storage: {{ .Values.filer.data.size }}
     {{- end }}
-    {{- if eq .Values.filer.logs.type "persistentVolumeClaim"}}
+    {{- if eq .Values.filer.logs.type "persistentVolumeClaim" }}
     - metadata:
         name: seaweedfs-filer-log-volume
+        {{- with .Values.filer.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.filer.logs.storageClass }}

k8s/charts/seaweedfs/templates/master-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/master-ingress.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.master.enabled }}
 {{- if .Values.master.ingress.enabled }}
 {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
 apiVersion: networking.k8s.io/v1
@@ -10,9 +11,9 @@ kind: Ingress
 metadata:
   name: ingress-{{ template "seaweedfs.name" . }}-master
   namespace: {{ .Release.Namespace }}
-  {{- if .Values.master.ingress.annotations }}
+  {{- with .Values.master.ingress.annotations }}
   annotations:
-    {{ tpl .Values.master.ingress.annotations . | nindent 4 | trim }}
+    {{- toYaml . | nindent 4 }}
   {{- end }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
@@ -22,6 +23,8 @@ metadata:
     app.kubernetes.io/component: master
 spec:
   ingressClassName: {{ .Values.master.ingress.className | quote }}
+  tls:
+    {{ .Values.master.ingress.tls | default list | toYaml | nindent 6 }}
   rules:
     - http:
         paths:
@@ -42,3 +45,4 @@ spec:
       host: {{ .Values.master.ingress.host }}
 {{- end }}
 {{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/master-service.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.master.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -9,7 +10,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
   annotations:
-    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"    
+    service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
 spec:
   clusterIP: None
   publishNotReadyAddresses: true
@@ -31,3 +32,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/component: master
+{{- end }}

k8s/charts/seaweedfs/templates/master-servicemonitor.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.master.enabled }}
 {{- if .Values.master.metricsPort }}
 {{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
@@ -18,7 +19,8 @@ spec:
       scrapeTimeout: 5s
   selector:
     matchLabels:
-      app: {{ template "seaweedfs.name" . }}
-      component: master
+      app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+      app.kubernetes.io/component: master
+{{- end }}
+{{- end }}
 {{- end }}
-{{- end }}

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -11,7 +11,7 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-master
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.master.podManagementPolicy }}
   replicas: {{ .Values.master.replicas }}
   {{- if (gt (int .Values.master.updatePartition) 0) }}
   updateStrategy:
@@ -22,7 +22,6 @@ spec:
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: master
   template:
@@ -32,6 +31,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: master
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }}
       {{- if .Values.master.affinity }}
@@ -48,8 +54,8 @@ spec:
       priorityClassName: {{ .Values.master.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.master.serviceAccountName }}
-      serviceAccountName: {{ .Values.master.serviceAccountName | quote }}
+      {{- if .Values.global.createClusterRole }}
+      serviceAccountName: {{ .Values.master.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       {{- if .Values.master.initContainers }}
       initContainers:
@@ -77,13 +83,23 @@ spec:
             {{- if .Values.master.extraEnvironmentVars }}
             {{- range $key, $value := .Values.master.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
           command:
@@ -119,10 +135,11 @@ spec:
               {{- if .Values.master.metricsIntervalSec }}
               -metrics.intervalSeconds={{ .Values.master.metricsIntervalSec }} \
               {{- end }}
-              {{- else if .Values.master.metricsPort }}
-              -metricsPort={{ .Values.master.metricsPort }} \
               {{- end }}
               {{- end }}
+              {{- if .Values.master.metricsPort }}
+              -metricsPort={{ .Values.master.metricsPort }} \
+              {{- end }}
               -volumeSizeLimitMB={{ .Values.master.volumeSizeLimitMB }} \
               {{- if .Values.master.disableHttp }}
               -disableHttp \
@@ -205,8 +222,9 @@ spec:
           resources:
             {{ tpl .Values.master.resources . | nindent 12 | trim }}
           {{- end }}
-      {{- $hostpath_exists := include "master.hostpath_exists" . -}}
-      {{- if $hostpath_exists }}
+        {{- if .Values.master.sidecars }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }}
+        {{- end }}
       volumes:
         {{- if eq .Values.master.logs.type "hostPath" }}
         - name: seaweedfs-master-log-volume
@@ -214,12 +232,22 @@ spec:
             path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.master.logs.type "existingClaim" }}
+        - name: seaweedfs-master-log-volume
+          persistentVolumeClaim:
+            claimName: {{ .Values.master.logs.claimName }} 
+        {{- end }}
         {{- if eq .Values.master.data.type "hostPath" }}
         - name: data-{{ .Release.Namespace }}
           hostPath:
             path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.master.data.type "existingClaim" }}
+        - name: data-{{ .Release.Namespace }}
+          persistentVolumeClaim:
+            claimName: {{ .Values.master.data.claimName }} 
+        {{- end }}
         - name: master-config
           configMap:
             name: {{ template "seaweedfs.name" . }}-master-config
@@ -244,7 +272,6 @@ spec:
             secretName: {{ template "seaweedfs.name" . }}-client-cert
         {{- end }}
         {{ tpl .Values.master.extraVolumes . | indent 8 | trim }}
-      {{- end }}
       {{- if .Values.master.nodeSelector }}
       nodeSelector:
         {{ tpl .Values.master.nodeSelector . | indent 8 | trim }}
@@ -255,6 +282,10 @@ spec:
     {{- if eq .Values.master.data.type "persistentVolumeClaim"}}
     - metadata:
         name: data-{{ .Release.Namespace }}
+        {{- with .Values.master.data.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.master.data.storageClass }}
@@ -265,6 +296,10 @@ spec:
     {{- if eq .Values.master.logs.type "persistentVolumeClaim"}}
     - metadata:
         name: seaweedfs-master-log-volume
+        {{- with .Values.master.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.master.logs.storageClass }}

k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml

@@ -0,0 +1,96 @@
+{{- if .Values.master.enabled }}
+{{- if .Values.filer.s3.enabled }}
+{{- if .Values.filer.s3.createBuckets }}
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{{ $.Release.Name }}-bucket-hook"
+  labels:
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+  annotations:
+    "helm.sh/hook": post-install
+    "helm.sh/hook-weight": "-5"
+    "helm.sh/hook-delete-policy": hook-succeeded
+spec:
+  template:
+    metadata:
+      name: "{{ .Release.Name }}"
+      labels:
+        app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+        app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: post-install-job
+        image: {{ template "master.image" . }}
+        env:
+          - name: WEED_CLUSTER_DEFAULT
+            value: "sw"
+          - name: WEED_CLUSTER_SW_MASTER
+            value: "{{ template "seaweedfs.name" . }}-master.{{ .Release.Namespace }}:9333"
+          - name: WEED_CLUSTER_SW_FILER
+            value: "{{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:8888"
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: SEAWEEDFS_FULLNAME
+            value: "{{ template "seaweedfs.name" . }}"
+        command:
+          - "/bin/sh"
+          - "-ec"
+          - |
+          {{- range $reg, $props := $.Values.filer.s3.createBuckets }}
+            exec /bin/echo \
+            "s3.bucket.create --name {{ $props.name }}" |\
+            /usr/bin/weed shell
+          {{- end }}
+          {{- range $reg, $props := $.Values.filer.s3.createBuckets }}
+          {{- if $props.anonymousRead }}
+            exec /bin/echo \
+            "s3.configure --user anonymous \
+                --buckets {{ $props.name }} \
+                --actions Read \
+                --apply true" |\
+                /usr/bin/weed shell
+          {{- end }}  
+          {{- end }}
+        {{- if .Values.filer.s3.enableAuth }}
+        volumeMounts:
+          - name: config-users
+            mountPath: /etc/sw
+            readOnly: true
+        {{- end }}
+        ports:
+          - containerPort: {{ .Values.master.port }}
+            name: swfs-master
+          {{- if and .Values.global.monitoring.enabled .Values.master.metricsPort }}
+          - containerPort: {{ .Values.master.metricsPort }}
+            name: metrics
+          {{- end }}
+          - containerPort: {{ .Values.master.grpcPort }}
+            #name: swfs-master-grpc
+    {{- if .Values.filer.s3.enableAuth }}
+      volumes:
+        - name: config-users
+          secret:
+            defaultMode: 420
+            {{- if not (empty .Values.filer.s3.existingConfigSecret) }}
+            secretName: {{ .Values.filer.s3.existingConfigSecret }}
+            {{- else }}
+            secretName: seaweedfs-s3-secret
+            {{- end }}
+    {{- end }}{{/** if .Values.filer.s3.enableAuth **/}}
+{{- end }}{{/** if .Values.master.enabled **/}}
+{{- end }}{{/** if .Values.filer.s3.enabled **/}}
+{{- end }}{{/** if .Values.filer.s3.createBuckets **/}}

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -24,6 +24,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: s3
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
       {{- if .Values.s3.tolerations }}
@@ -62,16 +69,32 @@ spec:
                   fieldPath: metadata.namespace
             - name: SEAWEEDFS_FULLNAME
               value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.s3.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.s3.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
           command:
             - "/bin/sh"
             - "-ec"
-            - | 
+            - |
               exec /usr/bin/weed \
                {{- if eq .Values.s3.logs.type "hostPath" }}
               -logdir=/logs \
@@ -114,9 +137,11 @@ spec:
             - name: logs
               mountPath: "/logs/"
             {{- end }}
+            {{- if .Values.s3.enableAuth }}
             - mountPath: /etc/sw
               name: config-users
               readOnly: true
+            {{- end }}
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -174,11 +199,20 @@ spec:
           resources:
             {{ tpl .Values.s3.resources . | nindent 12 | trim }}
           {{- end }}
+      {{- if .Values.s3.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }}
+      {{- end }}
       volumes:
+        {{- if .Values.s3.enableAuth }}
         - name: config-users
           secret:
             defaultMode: 420
+            {{- if .Values.s3.existingConfigSecret }}
+            secretName: {{ .Values.s3.existingConfigSecret }}
+            {{- else }}
             secretName: seaweedfs-s3-secret
+            {{- end }}
+        {{- end }}
         {{- if eq .Values.s3.logs.type "hostPath" }}
         - name: logs
           hostPath:

k8s/charts/seaweedfs/templates/s3-ingress.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.s3.ingress.enabled }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+{{- else }}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: ingress-{{ template "seaweedfs.name" . }}-s3
+  namespace: {{ .Release.Namespace }}
+  {{- with .Values.s3.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: s3
+spec:
+  ingressClassName: {{ .Values.s3.ingress.className | quote }}
+  tls:
+    {{ .Values.s3.ingress.tls | default list | toYaml | nindent 6}}
+  rules:
+  - http:
+      paths:
+      - path: /
+        pathType: ImplementationSpecific
+        backend:
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ template "seaweedfs.name" . }}-s3
+            port:
+              number: {{ .Values.s3.port }}
+              #name:
+{{- else }}
+          serviceName: {{ template "seaweedfs.name" . }}-s3
+          servicePort: {{ .Values.s3.port }}
+{{- end }}
+{{- if .Values.s3.ingress.host }}
+    host: {{ .Values.s3.ingress.host }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3-secret.yaml

@@ -1,4 +1,4 @@
-{{- if not (or .Values.filer.s3.skipAuthSecretCreation .Values.s3.skipAuthSecretCreation) }}
+{{- if or (and .Values.filer.s3.enabled .Values.filer.s3.enableAuth (not .Values.filer.s3.existingConfigSecret)) (and .Values.s3.enabled .Values.s3.enableAuth (not .Values.s3.existingConfigSecret)) }}
 {{- $access_key_admin := randAlphaNum 16 -}}
 {{- $secret_key_admin := randAlphaNum 32 -}}
 {{- $access_key_read := randAlphaNum 16 -}}
@@ -32,4 +32,4 @@ stringData:
   s3_auditLogConfig.json: |
     {{ toJson .Values.s3.auditLogConfig | nindent 4 }}
   {{- end }}
-{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3-service.yaml

@@ -1,3 +1,4 @@
+{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -9,6 +10,7 @@ metadata:
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
+  internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
   ports:
   - name: "swfs-s3"
     port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
@@ -29,3 +31,4 @@ spec:
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     app.kubernetes.io/component: {{ if .Values.s3.enabled }}s3{{ else }}filer{{ end }}
+{{- end }}

k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml

@@ -1,3 +1,4 @@
+{{- if or .Values.s3.enabled .Values.filer.s3.enabled }}
 {{- if .Values.s3.metricsPort }}
 {{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
@@ -21,4 +22,5 @@ spec:
       app: {{ template "seaweedfs.name" . }}
       component: s3
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/secret-seaweedfs-db.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.filer.enabled }}
 apiVersion: v1
 kind: Secret
 type: Opaque
@@ -17,3 +18,4 @@ stringData:
   password: "HardCodedPassword"
   # better to random generate and create in DB
   # password: {{ randAlphaNum 10 | sha256sum | b64enc | trunc 32 }}
+{{- end }}

k8s/charts/seaweedfs/templates/security-configmap.yaml

@@ -13,10 +13,38 @@ data:
   security.toml: |-
     # this file is read by master, volume server, and filer
 
+    {{- if .Values.global.securityConfig.jwtSigning.volumeWrite }}
     # the jwt signing key is read by master and volume server
     # a jwt expires in 10 seconds
     [jwt.signing]
     key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.volumeRead }}
+    # this jwt signing key is read by master and volume server, and it is used for read operations:
+    # - the Master server generates the JWT, which can be used to read a certain file on a volume server
+    # - the Volume server validates the JWT on reading
+    [jwt.signing.read]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.filerWrite }}
+    # If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
+    # - f.e. the S3 API Shim generates the JWT
+    # - the Filer server validates the JWT on writing
+    # the jwt defaults to expire after 10 seconds.
+    [jwt.filer_signing]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
+
+    {{- if .Values.global.securityConfig.jwtSigning.filerRead }}
+    # If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
+    # - f.e. the S3 API Shim generates the JWT
+    # - the Filer server validates the JWT on writing
+    # the jwt defaults to expire after 10 seconds.
+    [jwt.filer_signing.read]
+    key = "{{ randAlphaNum 10 | b64enc }}"
+    {{- end }}
 
     # all grpc tls authentications are mutual
     # the values for the following ca, cert, and key are paths to the PERM files.

k8s/charts/seaweedfs/templates/service-account.yaml

@@ -4,7 +4,7 @@
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: seaweedfs-rw-cr
+  name: {{ .Values.global.serviceAccountName }}-rw-cr
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -18,7 +18,7 @@ rules:
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: seaweedfs-rw-sa
+  name: {{ .Values.global.serviceAccountName }}
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
@@ -29,7 +29,7 @@ metadata:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: system:serviceaccount:seaweedfs-rw-sa:default
+  name: system:serviceaccount:{{ .Values.global.serviceAccountName }}:default
   labels:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
     helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
@@ -37,10 +37,10 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
 subjects:
 - kind: ServiceAccount
-  name: seaweedfs-rw-sa
+  name: {{ .Values.global.serviceAccountName }}
   namespace: {{ .Release.Namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
-  name: seaweedfs-rw-cr
+  name: {{ .Values.global.serviceAccountName }}-rw-cr
 {{- end }}

k8s/charts/seaweedfs/templates/volume-cert.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.global.enableSecurity }}
+{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
 apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
 kind: Certificate
 metadata:

k8s/charts/seaweedfs/templates/volume-service.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.volume.enabled }}
 apiVersion: v1
 kind: Service
 metadata:
@@ -10,6 +11,7 @@ metadata:
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 spec:
   clusterIP: None
+  internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
   ports:
   - name: "swfs-volume"
     port: {{ .Values.volume.port }}
@@ -27,4 +29,5 @@ spec:
 {{- end }}
   selector:
     app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-    app.kubernetes.io/component: volume
+    app.kubernetes.io/component: volume
+{{- end }}

k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.volume.enabled }}
 {{- if .Values.volume.metricsPort }}
 {{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
@@ -21,4 +22,5 @@ spec:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
       app.kubernetes.io/component: volume
 {{- end }}
-{{- end }}
+{{- end }}
+{{- end }}

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -12,11 +12,10 @@ metadata:
 spec:
   serviceName: {{ template "seaweedfs.name" . }}-volume
   replicas: {{ .Values.volume.replicas }}
-  podManagementPolicy: Parallel
+  podManagementPolicy: {{ .Values.volume.podManagementPolicy }}
   selector:
     matchLabels:
       app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
-      helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
       app.kubernetes.io/instance: {{ .Release.Name }}
       app.kubernetes.io/component: volume
   template:
@@ -26,6 +25,13 @@ spec:
         helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
         app.kubernetes.io/instance: {{ .Release.Name }}
         app.kubernetes.io/component: volume
+      {{ with .Values.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{ with .Values.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       {{- if .Values.volume.affinity }}
       affinity:
@@ -42,23 +48,25 @@ spec:
       priorityClassName: {{ .Values.volume.priorityClassName | quote }}
       {{- end }}
       enableServiceLinks: false
-      {{- if .Values.volume.serviceAccountName }}
-      serviceAccountName: {{ .Values.volume.serviceAccountName | quote }}
+      {{- if .Values.global.createClusterRole }}
+      serviceAccountName: {{ .Values.volume.serviceAccountName | default .Values.global.serviceAccountName | quote }} # for deleting statefulset pods after migration
       {{- end }}
       {{- $initContainers_exists := include "volume.initContainers_exists" . -}}
       {{- if $initContainers_exists }}
       initContainers:
-        {{- if .Values.volume.dir_idx }}
+        {{- if .Values.volume.idx }}
         - name: seaweedfs-vol-move-idx
           image: {{ template "volume.image" . }}
           imagePullPolicy: {{ .Values.global.imagePullPolicy | default "IfNotPresent" }}
           command: [ '/bin/sh', '-c' ]
-          args: [ 'if ls {{ .Values.volume.dir }}/*.idx >/dev/null 2>&1; then mv {{ .Values.volume.dir }}/*.idx {{ .Values.volume.dir_idx }}/; fi;' ]
+          args: [ '{{range $dir :=  .Values.volume.dataDirs }}if ls /{{$dir.name}}/*.idx  >/dev/null 2>&1; then mv /{{$dir.name}}/*.idx /idx/ ; fi; {{end}}' ]
           volumeMounts:
             - name: idx
-              mountPath: {{ .Values.volume.dir_idx }}
-            - name: data
-              mountPath: {{ .Values.volume.dir }}
+              mountPath: /idx
+          {{- range $dir :=  .Values.volume.dataDirs }}
+            - name: {{ $dir.name }}
+              mountPath: /{{ $dir.name }}
+          {{- end }}
         {{- end }}
         {{- if .Values.volume.initContainers }}
         {{ tpl .Values.volume.initContainers . | nindent 8 | trim }}
@@ -83,10 +91,26 @@ spec:
                   fieldPath: status.hostIP
             - name: SEAWEEDFS_FULLNAME
               value: "{{ template "seaweedfs.name" . }}"
+            {{- if .Values.volume.extraEnvironmentVars }}
+            {{- range $key, $value := .Values.volume.extraEnvironmentVars }}
+            - name: {{ $key }}
+            {{- if kindIs "string" $value }}
+              value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
+            {{- end }}
+            {{- end }}
             {{- if .Values.global.extraEnvironmentVars }}
             {{- range $key, $value := .Values.global.extraEnvironmentVars }}
             - name: {{ $key }}
+            {{- if kindIs "string" $value }}
               value: {{ $value | quote }}
+            {{- else }}
+              valueFrom:
+                {{ toYaml $value | nindent 16 | trim }}
+            {{- end -}}
             {{- end }}
             {{- end }}
           command:
@@ -94,7 +118,7 @@ spec:
             - "-ec"
             - |
               exec /usr/bin/weed \
-                {{- if eq .Values.volume.logs.type "hostPath" }}
+                {{- if .Values.volume.logs }}
                 -logdir=/logs \
                 {{- else }}
                 -logtostderr=true \
@@ -109,11 +133,11 @@ spec:
                 {{- if .Values.volume.metricsPort }}
                 -metricsPort={{ .Values.volume.metricsPort }} \
                 {{- end }}
-                -dir={{ .Values.volume.dir }} \
-                {{- if .Values.volume.dir_idx }}
-                -dir.idx={{ .Values.volume.dir_idx }} \
+                -dir {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}/{{$dir.name}}{{end}} \
+                {{- if .Values.volume.idx }}
+                -dir.idx=/idx \
                 {{- end }}
-                -max={{ .Values.volume.maxVolumes }} \
+                -max {{range $index, $dir :=  .Values.volume.dataDirs }}{{if ne $index 0}},{{end}}{{$dir.maxVolumes}}{{end}} \
                 {{- if .Values.volume.rack }}
                 -rack={{ .Values.volume.rack }} \
                 {{- end }}
@@ -140,18 +164,20 @@ spec:
                 -minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
                 -ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
                 -compactionMBps={{ .Values.volume.compactionMBps }} \
-                -mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
+                -mserver={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
           volumeMounts:
-            - name: data
-              mountPath: "{{ .Values.volume.dir }}/"
-            {{- if .Values.volume.dir_idx }}
-            - name: idx
-              mountPath: "{{ .Values.volume.dir_idx }}/"
+            {{- range $dir := .Values.volume.dataDirs }}
+            - name: {{ $dir.name }}
+              mountPath: "/{{ $dir.name }}/"
             {{- end }}
-            {{- if eq .Values.volume.logs.type "hostPath" }}
+            {{- if .Values.volume.logs }}
             - name: logs
               mountPath: "/logs/"
             {{- end }}
+            {{- if .Values.volume.idx }}
+            - name: idx
+              mountPath: "/idx/"
+            {{- end }}
             {{- if .Values.global.enableSecurity }}
             - name: security-config
               readOnly: true
@@ -178,7 +204,7 @@ spec:
             - containerPort: {{ .Values.volume.port }}
               name: swfs-vol
             {{- if .Values.volume.metricsPort }}
-            - containerPort: {{ .Values.filer.metricsPort }}
+            - containerPort: {{ .Values.volume.metricsPort }}
               name: metrics
             {{- end }}
             - containerPort: {{ .Values.volume.grpcPort }}
@@ -211,28 +237,55 @@ spec:
           resources:
             {{ tpl .Values.volume.resources . | nindent 12 | trim }}
           {{- end }}
-      {{- $hostpath_exists := include "volume.hostpath_exists" . -}}
-      {{- if $hostpath_exists }}
+      {{- if .Values.volume.sidecars }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }}
+      {{- end }}
       volumes:
-        {{- if eq .Values.volume.data.type "hostPath" }}
-        - name: data
+
+     {{- range $dir := .Values.volume.dataDirs }}
+
+      {{- if eq $dir.type "hostPath" }}
+        - name: {{ $dir.name }}
           hostPath:
-            path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
+            path: {{ $dir.hostPathPrefix }}/object_store/
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
+      {{- end }}
+      {{- if eq $dir.type "existingClaim" }}
+        - name: {{ $dir.name }}
+          persistentVolumeClaim:
+            claimName: {{ $dir.claimName }}
+      {{- end }}
+
+     {{- end }}
+
+     {{- if .Values.volume.idx }}
+       {{- if eq .Values.volume.idx.type "hostPath" }}
         - name: idx
           hostPath:
             path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if eq .Values.volume.logs.type "hostPath" }}
+       {{- end }}
+       {{- if eq .Values.volume.idx.type "existingClaim" }}
+        - name: idx
+          persistentVolumeClaim:
+            claimName: {{ .Values.volume.idx.claimName }}
+       {{- end }}
+     {{- end }}
+
+     {{- if .Values.volume.logs }}
+       {{- if eq .Values.volume.logs.type "hostPath" }}
         - name: logs
           hostPath:
             path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
             type: DirectoryOrCreate
-        {{- end }}
-        {{- if .Values.global.enableSecurity }}
+       {{- end }}
+       {{- if eq .Values.volume.logs.type "existingClaim" }}
+        - name: logs
+          persistentVolumeClaim:
+            claimName: {{ .Values.volume.logs.claimName }}
+       {{- end }}
+     {{- end }}
+     {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:
             name: {{ template "seaweedfs.name" . }}-security-config
@@ -251,31 +304,39 @@ spec:
         - name: client-cert
           secret:
             secretName: {{ template "seaweedfs.name" . }}-client-cert
-        {{- end }}
+      {{- end }}
       {{- if .Values.volume.extraVolumes }}
         {{ tpl .Values.volume.extraVolumes . | indent 8 | trim }}
       {{- end }}
-      {{- end }}
       {{- if .Values.volume.nodeSelector }}
       nodeSelector:
         {{ tpl .Values.volume.nodeSelector . | indent 8 | trim }}
       {{- end }}
-  {{- $pvc_exists := include "volume.pvc_exists" . -}}
-  {{- if $pvc_exists }}
   volumeClaimTemplates:
-    {{- if eq .Values.volume.data.type "persistentVolumeClaim"}}
+    {{- range $dir := .Values.volume.dataDirs }}
+    {{- if eq $dir.type "persistentVolumeClaim" }}
     - metadata:
-        name: data
+        name: {{ $dir.name }}
+        {{- with $dir.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
-        storageClassName: {{ .Values.volume.data.storageClass }}
+        storageClassName: {{ $dir.storageClass }}
         resources:
           requests:
-            storage: {{ .Values.volume.data.size }}
+            storage: {{ $dir.size }}
     {{- end }}
-    {{- if and (eq .Values.volume.idx.type "persistentVolumeClaim") .Values.volume.dir_idx }}
+    {{- end }}
+
+    {{- if and .Values.volume.idx (eq .Values.volume.idx.type "persistentVolumeClaim") }}
     - metadata:
         name: idx
+        {{- with .Values.volume.idx.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.volume.idx.storageClass }}
@@ -283,9 +344,13 @@ spec:
           requests:
             storage: {{ .Values.volume.idx.size }}
     {{- end }}
-    {{- if eq .Values.volume.logs.type "persistentVolumeClaim" }}
+    {{- if and .Values.volume.logs (eq .Values.volume.logs.type "persistentVolumeClaim") }}
     - metadata:
         name: logs
+        {{- with .Values.volume.logs.annotations }}
+        annotations:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
       spec:
         accessModes: [ "ReadWriteOnce" ]
         storageClassName: {{ .Values.volume.logs.storageClass }}
@@ -293,5 +358,4 @@ spec:
           requests:
             storage: {{ .Values.volume.logs.size }}
     {{- end }}
-      {{- end }}
-{{- end }}
+    {{- end }}

k8s/charts/seaweedfs/values.yaml

@@ -10,6 +10,15 @@ global:
   restartPolicy: Always
   loggingLevel: 1
   enableSecurity: false
+  masterServer: null
+  securityConfig:
+    jwtSigning:
+      volumeWrite: true
+      volumeRead: false
+      filerWrite: false
+      filerRead: false
+  # we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
+  serviceAccountName: "seaweedfs"
   certificates:
     alphacrds: false
   monitoring:
@@ -27,6 +36,10 @@ global:
     WEED_CLUSTER_DEFAULT: "sw"
     WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
     WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
+    # WEED_JWT_SIGNING_KEY:
+    #   secretKeyRef:
+    #     name: seaweedfs-signing-key
+    #     key: signingKey
 
 image:
   registry: ""
@@ -64,16 +77,23 @@ master:
 
   config: |-
     # Enter any extra configuration for master.toml here.
-    # It may be be a multi-line string.
+    # It may be a multi-line string.
 
-  # can use ANY storage-class , example with local-path-provisioner
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
   #  data:
   #    type: "persistentVolumeClaim"
   #    size: "24Ti"
   #    storageClass: "local-path-provisioner"
+  #    annotations:
+  #      "key": "value"
+  #
+  # You may also spacify an existing claim:
+  #  data:
+  #    type: "existingClaim"
+  #    claimName: "my-pvc"
   data:
     type: "hostPath"
-    size: ""
     storageClass: ""
     hostPathPrefix: /ssd
 
@@ -83,11 +103,25 @@ master:
     storageClass: ""
     hostPathPrefix: /storage
 
+  ## @param master.sidecars Add additional sidecar containers to the master pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Resource requests, limits, etc. for the master cluster placement. This
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request
@@ -120,7 +154,7 @@ master:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -137,7 +171,7 @@ master:
     className: "nginx"
     # host: false for "*" hostname
     host: "master.seaweedfs.local"
-    annotations: |
+    annotations:
       nginx.ingress.kubernetes.io/auth-type: "basic"
       nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
       nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Master'
@@ -153,12 +187,13 @@ master:
         sub_filter '=/' '=./';
         sub_filter '/seaweedfsstatic' './seaweedfsstatic';
         sub_filter_once off;
+    tls: []
 
   extraEnvironmentVars:
-    WEED_MASTER_VOLUME_GROWTH_COPY_1: 7
-    WEED_MASTER_VOLUME_GROWTH_COPY_2: 6
-    WEED_MASTER_VOLUME_GROWTH_COPY_3: 3
-    WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: 1
+    WEED_MASTER_VOLUME_GROWTH_COPY_1: '7'
+    WEED_MASTER_VOLUME_GROWTH_COPY_2: '6'
+    WEED_MASTER_VOLUME_GROWTH_COPY_3: '3'
+    WEED_MASTER_VOLUME_GROWTH_COPY_OTHER: '1'
 
   # used to configure livenessProbe on master-server containers
   #
@@ -208,40 +243,63 @@ volume:
   # minimum free disk space(in percents). If free disk space lower this value - all volumes marks as ReadOnly
   minFreeSpacePercent: 7
 
-# can use ANY storage-class , example with local-path-provisioner
-#  data:
-#    type: "persistentVolumeClaim"
-#    size: "24Ti"
-#    storageClass: "local-path-provisioner"
-  data:
-    type: "hostPath"
-    size: ""
-    storageClass: ""
-    hostPathPrefix: /storage
+  # For each data disk you may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
+  #  dataDirs:
+  #   - name: data:
+  #     type: "persistentVolumeClaim"
+  #     size: "24Ti"
+  #     storageClass: "local-path-provisioner"
+  #     annotations:
+  #      "key": "value"
+  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
+  #
+  # You may also spacify an existing claim:
+  #    - name: data
+  #      type: "existingClaim"
+  #      claimName: "my-pvc"
+  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
 
-  idx:
+  dataDirs:
+  - name: data1
     type: "hostPath"
-    size: ""
-    storageClass: ""
     hostPathPrefix: /ssd
+    maxVolumes: 0
 
-  logs:
-    type: "hostPath"
-    size: ""
-    storageClass: ""
-    hostPathPrefix: /storage
+   # - name: data2
+   #   type: "persistentVolumeClaim"
+   #   storageClass: "yourClassNameOfChoice"
+   #   size: "800Gi"
+   #   maxVolumes: 0
+
+  # idx can be defined by:
+  #
+  # idx:
+  #  type: "hostPath"
+  #  hostPathPrefix: /ssd
+  #
+  # or
+  #
+  # idx:
+  #  type: "persistentVolumeClaim"
+  #  size: "20Gi"
+  #  storageClass: "local-path-provisioner"
+  #
+  # or
+  #
+  # idx:
+  #   type: "existingClaim"
+  #   claimName: "myClaim"
+
+  # same applies to "logs"
+
+  idx: {}
+
+  logs: {}
 
   # limit background compaction or copying speed in mega bytes per second
   compactionMBps: "50"
 
-  # Directories to store data files. dir[,dir]... (default "/tmp")
-  dir: "/data"
-  # Directories to store index files. dir[,dir]... (default is the same as "dir")
-  dir_idx: null
-
-  # Maximum numbers of volumes, count[,count]...
-  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
-  maxVolumes: "0"
 
   # Volume server's rack name
   rack: null
@@ -258,11 +316,25 @@ volume:
   # Adjust jpg orientation when uploading.
   imagesFixOrientation: false
 
+  ## @param volume.sidecars Add additional sidecar containers to the volume pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Affinity Settings
   # Commenting out or setting as empty the affinity variable, will allow
   # deployment to single node services such as Minikube
@@ -291,7 +363,7 @@ volume:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-   beta.kubernetes.io/arch: amd64
+   kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-volume: "true"
 
@@ -303,6 +375,8 @@ volume:
   # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
   serviceAccountName: ""
 
+  extraEnvironmentVars:
+
   # used to configure livenessProbe on volume-server containers
   #
   livenessProbe:
@@ -373,12 +447,19 @@ filer:
   storage: 25Gi
   # storageClass is the class of storage which defaults to null (the Kube cluster will pick the default).
   storageClass: null
-
-  # can use ANY storage-class , example with local-path-provisioner
+  # You may use ANY storage-class, example with local-path-provisioner
+  # Annotations are optional.
   #  data:
   #    type: "persistentVolumeClaim"
   #    size: "24Ti"
   #    storageClass: "local-path-provisioner"
+  #    annotations:
+  #      "key": "value"
+  #
+  # You may also specify an existing claim:
+  #  data:
+  #    type: "existingClaim"
+  #    claimName: "my-pvc"
   data:
     type: "hostPath"
     size: ""
@@ -391,11 +472,25 @@ filer:
     storageClass: ""
     hostPathPrefix: /storage
 
+  ## @param filer.sidecars Add additional sidecar containers to the filer pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  ## Set podManagementPolicy
+  podManagementPolicy: Parallel
+
   # Affinity Settings
   # Commenting out or setting as empty the affinity variable, will allow
   # deployment to single node services such as Minikube
@@ -428,7 +523,7 @@ filer:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -445,7 +540,7 @@ filer:
     className: "nginx"
     # host: false for "*" hostname
     host: "seaweedfs.cluster.local"
-    annotations: |
+    annotations:
       nginx.ingress.kubernetes.io/backend-protocol: GRPC
       nginx.ingress.kubernetes.io/auth-type: "basic"
       nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
@@ -521,7 +616,7 @@ filer:
       #     key: password
 
   s3:
-    enabled: true
+    enabled: false
     port: 8333
     # add additional https port
     httpsPort: 0
@@ -531,8 +626,17 @@ filer:
     domainName: ""
     # enable user & permission to s3 (need to inject to all services)
     enableAuth: false
-    skipAuthSecretCreation: false
+    # set to the name of an existing kubernetes Secret with the s3 json config file
+    # should have a secret key called seaweedfs_s3_config with an inline json configure
+    existingConfigSecret: null
     auditLogConfig: {}
+    # You may specify buckets to be created during the install process.
+    # Buckets may be exposed publicly by setting `anonymousRead` to `true`
+    # createBuckets:
+    #   - name: bucket-a
+    #     anonymousRead: true
+    #   - name: bucket-b
+    #     anonymousRead: false
 
 s3:
   enabled: false
@@ -551,12 +655,25 @@ s3:
   allowEmptyFolder: true
   # enable user & permission to s3 (need to inject to all services)
   enableAuth: false
-  skipAuthSecretCreation: false
+  # set to the name of an existing kubernetes Secret with the s3 json config file
+  # should have a secret key called seaweedfs_s3_config with an inline json config
+  existingConfigSecret: null
   auditLogConfig: {}
 
   # Suffix of the host name, {bucket}.{domainName}
   domainName: ""
 
+  ## @param s3.sidecars Add additional sidecar containers to the s3 pod(s)
+  ## e.g:
+  ## sidecars:
+  ##   - name: your-image-name
+  ##     image: your-image
+  ##     imagePullPolicy: Always
+  ##     ports:
+  ##       - name: portname
+  ##         containerPort: 1234
+  ##
+  sidecars: []
   initContainers: ""
 
   extraVolumes: ""
@@ -577,7 +694,7 @@ s3:
   # ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
   # Example:
   nodeSelector: |
-    beta.kubernetes.io/arch: amd64
+    kubernetes.io/arch: amd64
   # nodeSelector: |
   #   sw-backend: "true"
 
@@ -595,6 +712,8 @@ s3:
     storageClass: ""
     hostPathPrefix: /storage
 
+  extraEnvironmentVars:
+
   # used to configure livenessProbe on s3 containers
   #
   livenessProbe:
@@ -621,6 +740,15 @@ s3:
     failureThreshold: 100
     timeoutSeconds: 10
 
+  ingress:
+    enabled: false
+    className: "nginx"
+    # host: false for "*" hostname
+    host: "seaweedfs.cluster.local"
+    # additional ingress annotations for the s3 endpoint
+    annotations: {}
+    tls: []
+
 certificates:
   commonName: "SeaweedFS CA"
   ipAddresses: []
@@ -628,3 +756,13 @@ certificates:
   keySize: 2048
   duration: 2160h  # 90d
   renewBefore: 360h  # 15d
+  externalCertificates:
+    # This will avoid the need to use cert-manager and will rely on providing your own external certificates and CA
+    # you will need to store your provided certificates in the secret read by the different services:
+    # seaweedfs-master-cert, seaweedfs-filer-cert, etc. Can see any statefulset definition to see secret names
+    enabled: false
+
+# Labels to be added to all the created pods
+podLabels: {}
+# Annotations to be added to all the created pods
+podAnnotations: {}

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -16,8 +16,8 @@
     <properties>
         <protobuf.version>3.16.3</protobuf.version>
         <!-- follow https://github.com/grpc/grpc-java -->
-        <grpc.version>1.23.0</grpc.version>
-        <guava.version>30.0-jre</guava.version>
+        <grpc.version>1.53.0</grpc.version>
+        <guava.version>32.0.0-jre</guava.version>
     </properties>
 
     <dependencies>
@@ -96,6 +96,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -91,6 +91,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>3.55</version>
+    <version>3.59</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -90,6 +90,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -13,12 +13,12 @@ public class FilerClient extends FilerGrpcClient {
 
     private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
 
-    public FilerClient(String host, int grpcPort) {
-        super(host, grpcPort-10000, grpcPort);
+    public FilerClient(String filerHost, int filerGrpcPort) {
+        super(filerHost, filerGrpcPort-10000, filerGrpcPort);
     }
 
-    public FilerClient(String host, int port, int grpcPort) {
-        super(host, port, grpcPort);
+    public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
+        super(filerHost, filerPort, filerGrpcPort);
     }
 
     public static String toFileId(FilerProto.FileId fid) {

other/java/client/src/main/java/seaweedfs/client/SeaweedRead.java

@@ -69,7 +69,7 @@ public class SeaweedRead {
             if (locations == null || locations.getLocationsCount() == 0) {
                 LOG.error("failed to locate {}", chunkView.fileId);
                 volumeIdCache.clearLocations(volumeId);
-                return 0;
+                throw new IOException("failed to locate fileId " + chunkView.fileId);
             }
 
             int len = readChunkView(filerClient, startOffset, buf, chunkView, locations);

other/java/client/src/main/proto/filer.proto

@@ -364,6 +364,7 @@ message LogEntry {
     int64 ts_ns = 1;
     int32 partition_key_hash = 2;
     bytes data = 3;
+    bytes key = 4;
 }
 
 message KeepConnectedRequest {
@@ -424,6 +425,7 @@ message FilerConf {
         string data_center = 9;
         string rack = 10;
         string data_node = 11;
+        uint32 max_file_name_length = 12;
     }
     repeated PathConf locations = 2;
 }
@@ -451,8 +453,9 @@ message LockRequest {
 }
 message LockResponse {
     string renew_token = 1;
-    string moved_to = 2;
-    string error = 3;
+    string lock_owner = 2;
+    string lock_host_moved_to = 3;
+    string error = 4;
 }
 message UnlockRequest {
     string name = 1;

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>3.55</version>
+            <version>3.59</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>3.55</version>
+            <version>3.59</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs-over-ftp/src/main/java/org/apache/hadoop/seaweed/ftp/service/HFtpService.java

@@ -16,7 +16,7 @@ import org.springframework.stereotype.Component;
 import java.io.File;
 
 /**
- * reference: https://github.com/iponweb/hdfs-over-ftp
+ * reference: https://github.com/AShiou/hof
  */
 @Component
 public class HFtpService {

other/java/hdfs2/dependency-reduced-pom.xml

@@ -17,6 +17,7 @@
         <configuration>
           <source>8</source>
           <target>8</target>
+          <release>8</release>
         </configuration>
       </plugin>
       <plugin>
@@ -317,7 +318,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.55</seaweedfs.client.version>
+    <seaweedfs.client.version>3.59</seaweedfs.client.version>
     <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.55</seaweedfs.client.version>
+        <seaweedfs.client.version>3.59</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
@@ -33,6 +33,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

other/java/hdfs3/dependency-reduced-pom.xml

@@ -17,6 +17,7 @@
         <configuration>
           <source>8</source>
           <target>8</target>
+          <release>8</release>
         </configuration>
       </plugin>
       <plugin>
@@ -317,7 +318,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>3.55</seaweedfs.client.version>
+    <seaweedfs.client.version>3.59</seaweedfs.client.version>
     <hadoop.version>3.2.4</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>3.55</seaweedfs.client.version>
+        <seaweedfs.client.version>3.59</seaweedfs.client.version>
         <hadoop.version>3.2.4</hadoop.version>
     </properties>
 
@@ -33,6 +33,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
             <plugin>

test/random_access/pom.xml

@@ -50,6 +50,7 @@
                 <configuration>
                     <source>8</source>
                     <target>8</target>
+                    <release>8</release>
                 </configuration>
             </plugin>
         </plugins>

test/s3/compatibility/.gitignore

@@ -1,2 +1,8 @@
+# Test run scratch data
 /s3-tests
 /tmp
+
+# Test run outputs
+weed.log
+compat.raw.txt
+compat.summary.txt

test/s3/compatibility/Dockerfile

@@ -1,11 +1,21 @@
 # the tests only support python 3.6, not newer
-FROM ubuntu:latest
+#FROM ubuntu:latest
+FROM python:3.6.15-slim-buster
 
-RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+# Installed required system deps
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
+
+# Install python deps
+RUN pip install virtualenv
+
+# Clone Ceph S3 tests
 RUN git clone https://github.com/ceph/s3-tests.git
+
 WORKDIR s3-tests
 
-# we pin a certain commit
+# Pin to a certain commit on ceph/s3-tests
+# https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
 RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
 
 RUN ./bootstrap

test/s3/compatibility/Makefile

@@ -0,0 +1,9 @@
+.PHONY: all build-image test
+
+all: build-image test
+
+build-image:
+	./prepare.sh
+
+test:
+	./run.sh

test/s3/compatibility/README.md

@@ -1,13 +1,245 @@
 # Running S3 Compatibility tests against SeaweedFS
 
-This is using [the tests from CephFS](https://github.com/ceph/s3-tests).
+This is using [the tests from CephFS][s3-tests].
+
+[s3-tests]: https://github.com/ceph/s3-tests
 
 ## Prerequisites
 
-- have Docker installed
+- have [Docker][docker] installed
 - this has been executed on Mac. On Linux, the hostname in `s3tests.conf`  needs to be adjusted.
 
+[docker]: https://docs.docker.com
+
 ## Running tests
 
-- `./prepare.sh` to build the docker image
-- `./run.sh` to execute all tests
+To build the docker image that is used for the tests:
+
+```console
+./prepare.sh
+```
+
+To execute all tests:
+
+```console
+./run.sh
+```
+To see debug output including all commands run by the script:
+
+```console
+DEBUG=y ./run.sh
+```
+
+> [!WARNING]
+>
+> If your output does *not* look like the content in [`results.summary.txt`](./results.summary.txt)
+> and it is full of HTTP level exceptions, there is likely an error contacting the `weed` server from
+> the container that is runnin the S3 compatibility tests.
+>
+> There are at least a couple ways to solve this:
+>
+> - Modify your `docker` setup to ensure `host.docker.internal` is connected to your host running `weed`
+> - Use `--net=host` and modify `host` in `s3tests.conf` to `localhost`
+>
+> The `--net=host` solution is potentially *unsafe*, as the container running [s3-tests][s3-tests] could
+> visit unexpected websites or use the host-passthrough internet access maliciously.
+>
+> If you are OK with the risk of allowing `--net=host`:
+>
+> - Set `host = localhost` in `s3tests.conf`
+> - Set `DOCKER_NET_HOST=y` when running `run.sh`
+
+## Most recent results
+
+See [`results.summary.txt`](./results.summary.txt) for the latest results of compatibility testing (with the caveat that `s3-tests` is pinned to [`ceph/s3-tests` @ 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e](https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e)).
+
+The file is reproduced below for ease of access:
+
+```
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------
+```

test/s3/compatibility/results.summary.txt

@@ -0,0 +1,186 @@
+/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
+  warnings.warn(warning, PythonDeprecationWarning)
+s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
+s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
+s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
+s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
+s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
+s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
+s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
+s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
+s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
+s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
+s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
+s3tests_boto3.functional.test_s3.test_object_write_file ... ok
+s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
+s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
+s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
+s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
+s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
+s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
+s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
+s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
+s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
+s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
+s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
+s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
+s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
+s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
+s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
+s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
+s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
+s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
+s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
+s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
+s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
+s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
+s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
+s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
+s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
+s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
+s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
+s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
+s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
+s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
+s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
+s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
+s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
+s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
+s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
+s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
+s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
+s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
+s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
+s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
+ERROR
+
+======================================================================
+ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
+----------------------------------------------------------------------
+Traceback (most recent call last):
+  File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
+    self.test(*self.arg)
+  File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
+    'DisplayName': acl_response['Owner']['DisplayName'],
+KeyError: 'Owner'
+-------------------- >> begin captured logging << --------------------

test/s3/compatibility/run.sh

@@ -1,24 +1,88 @@
 #!/usr/bin/env bash
 
-set -ex
+CONTAINER_NAME=${CONTAINER_NAME:-s3test-instance}
+CONF_FILE=${CONF_FILE:-s3tests.conf}
+WEED_BIN=${WEED_BIN:-../../../weed/weed}
+TEST_RAW_OUTPUT_FILE=${TEST_RAW_OUTPUT_FILE:-compat.raw.txt}
+TEST_PROCESSED_OUTPUT_FILE=${TEST_PROCESSED_OUTPUT_FILE:-compat.summary.txt}
 
+# Set up debugging for this bash script if DEBUG is set
+if [ -n "${DEBUG}" ]; then
+    echo -e "DEBUG set [${DEBUG}], enabling debugging output...";
+    set -ex
+fi
+
+# Reset from possible previous test run
 killall -9 weed || echo "already stopped"
 rm -Rf tmp
 mkdir tmp
-docker stop s3test-instance || echo "already stopped"
+docker stop $CONTAINER_NAME || echo "already stopped"
 
+# Ensure ulimit is set to reasonable value
 ulimit -n 10000
-../../../weed/weed server -filer -s3 -volume.max 0 -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log &
 
+# Start weed w/ filer + s3 in the background
+$WEED_BIN server \
+          -filer \
+          -s3 \
+          -volume.max 0 \
+          -master.volumeSizeLimitMB 5 \
+          -dir "$(pwd)/tmp" \
+          1>&2>weed.log &
+
+# Wait for master to start up
+echo -e "\n[info] waiting for master @ 9333...";
 until curl --output /dev/null --silent --head --fail http://127.0.0.1:9333; do
-    printf '.'
-    sleep 5
+    printf '.';
+    sleep 5;
 done
-sleep 3
+sleep 3;
 
-rm -Rf logs-full.txt logs-summary.txt
-# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v  -a 'resource=object,!bucket-policy,!versioning,!encryption'
-docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v "$(pwd)"/s3tests.conf:/s3-tests/s3tests.conf -it s3tests    ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v  -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt
+# Wait for s3 to start up
+echo -e "\n[info] waiting for S3 @ 8333...";
+until curl --output /dev/null --silent --fail http://127.0.0.1:8333; do
+    printf '.';
+    sleep 5;
+done
+sleep 3;
 
-docker stop s3test-instance || echo "already stopped"
-killall -9 weed
+# Determine whether docker net
+DOCKER_NET_HOST_ARGS=""
+if [ -n "${DOCKER_NET_HOST}" ]; then
+    DOCKER_NET_HOST_ARGS="--net=host"
+    echo -e "\n[info] setting docker to het nost"
+fi
+
+echo -e "\n[warn] You may have to run with UNFILTERED=y to disable output filtering, if you get the broken pipe error";
+echo -e "\n[info] running tests with unfiltered output...";
+docker run \
+       --name $CONTAINER_NAME \
+       --rm \
+       ${DOCKER_NET_HOST_ARGS} \
+       -e S3TEST_CONF=$CONF_FILE \
+       -v "$(pwd)"/$CONF_FILE:/s3-tests/s3tests.conf \
+       -it \
+       s3tests \
+       ./virtualenv/bin/nosetests \
+       s3tests_boto3/functional/test_s3.py \
+       -v \
+       -a 'resource=object,!bucket-policy,!versioning,!encryption' \
+    | tee ${TEST_RAW_OUTPUT_FILE}
+
+# If the summary logs are present, process them
+if [ -f "${TEST_RAW_OUTPUT_FILE}" ]; then
+    cat ${TEST_RAW_OUTPUT_FILE} | sed -n -e '/botocore.hooks/!p;//q' | tee ${TEST_PROCESSED_OUTPUT_FILE}
+    echo -e "\n[info] ✅ Successfully wrote processed output @ [${TEST_PROCESSED_OUTPUT_FILE}]";
+    if [ -z "${TEST_KEEP_RAW_OUTPUT}" ]; then
+        echo -e "\n[info] removing test raw output file @ [${TEST_RAW_OUTPUT_FILE}] (to disable this, set TEST_KEEP_RAW_OUTPUT=y)...";
+        rm -rf ${TEST_RAW_OUTPUT_FILE};
+    fi
+else
+    echo -e "\n[warn] failed to find raw output @ [${TEST_RAW_OUTPUT_FILE}]";
+fi
+
+echo -e "\n[info] stopping [${CONTAINER_NAME}] container...";
+docker stop $CONTAINER_NAME || echo "[info] already stopped";
+
+echo -e "\n[info] stopping seaweedfs processes (all, via kill -9)...";
+killall -9 weed;

test/s3/compatibility/s3tests.conf

@@ -4,6 +4,14 @@
 # host set for rgw in vstart.sh
 host = host.docker.internal
 
+## NOTE: if running docker --net=host (e.x. `DOCKER_NET_HOST=y ./run.sh`)
+## then use 'localhost', so the container can more easily access your *local* weed instance
+##
+## This is *unsafe* -- it is possible that this script will do malicious
+## things with host-level access. only run this if you trust the programmatic content of
+## ceph/s3-tests (https://github.com/ceph/s3-tests) which will be using this file
+#host = localhost
+
 # port set for rgw in vstart.sh
 port = 8333
 

unmaintained/see_dat/see_dat.go

@@ -33,8 +33,8 @@ func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
 
 func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
 	t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
-	glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v",
-		*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t)
+	glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v name %s",
+		*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t, n.Name)
 	return nil
 }
 

weed/cluster/lock_client.go

@@ -32,108 +32,104 @@ type LiveLock struct {
 	key            string
 	renewToken     string
 	expireAtNs     int64
-	filer          pb.ServerAddress
+	hostFiler      pb.ServerAddress
 	cancelCh       chan struct{}
 	grpcDialOption grpc.DialOption
 	isLocked       bool
-	owner          string
+	self           string
 	lc             *LockClient
+	owner          string
 }
 
-// NewLock creates a lock with a very long duration
-func (lc *LockClient) NewLock(key string, owner string) (lock *LiveLock) {
-	return lc.doNewLock(key, lock_manager.MaxDuration, owner)
-}
-
-// StartLock starts a goroutine to lock the key and returns immediately.
-func (lc *LockClient) StartLock(key string, owner string) (lock *LiveLock) {
+// NewShortLivedLock creates a lock with a 5-second duration
+func (lc *LockClient) NewShortLivedLock(key string, owner string) (lock *LiveLock) {
 	lock = &LiveLock{
 		key:            key,
-		filer:          lc.seedFiler,
+		hostFiler:      lc.seedFiler,
 		cancelCh:       make(chan struct{}),
-		expireAtNs:     time.Now().Add(lock_manager.MaxDuration).UnixNano(),
+		expireAtNs:     time.Now().Add(5 * time.Second).UnixNano(),
 		grpcDialOption: lc.grpcDialOption,
-		owner:          owner,
+		self:           owner,
+		lc:             lc,
+	}
+	lock.retryUntilLocked(5 * time.Second)
+	return
+}
+
+// StartLongLivedLock starts a goroutine to lock the key and returns immediately.
+func (lc *LockClient) StartLongLivedLock(key string, owner string, onLockOwnerChange func(newLockOwner string)) (lock *LiveLock) {
+	lock = &LiveLock{
+		key:            key,
+		hostFiler:      lc.seedFiler,
+		cancelCh:       make(chan struct{}),
+		expireAtNs:     time.Now().Add(lock_manager.LiveLockTTL).UnixNano(),
+		grpcDialOption: lc.grpcDialOption,
+		self:           owner,
 		lc:             lc,
 	}
 	go func() {
-		util.RetryForever("create lock:"+key, func() error {
-			errorMessage, err := lock.doLock(lock_manager.MaxDuration)
-			if err != nil {
-				glog.Infof("create lock %s: %s", key, err)
-				time.Sleep(time.Second)
-				return err
+		isLocked := false
+		lockOwner := ""
+		for {
+			if isLocked {
+				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err != nil {
+					glog.V(0).Infof("Lost lock %s: %v", key, err)
+					isLocked = false
+				}
+			} else {
+				if err := lock.AttemptToLock(lock_manager.LiveLockTTL); err == nil {
+					isLocked = true
+				}
 			}
-			if errorMessage != "" {
-				glog.Infof("create lock %s: %s", key, errorMessage)
-				time.Sleep(time.Second)
-				return fmt.Errorf("%v", errorMessage)
+			if lockOwner != lock.LockOwner() && lock.LockOwner() != "" {
+				glog.V(0).Infof("Lock owner changed from %s to %s", lockOwner, lock.LockOwner())
+				onLockOwnerChange(lock.LockOwner())
+				lockOwner = lock.LockOwner()
 			}
-			lock.isLocked = true
-			return nil
-		}, func(err error) (shouldContinue bool) {
-			if err != nil {
-				glog.Warningf("create lock %s: %s", key, err)
-				time.Sleep(time.Second)
+			select {
+			case <-lock.cancelCh:
+				return
+			default:
+				time.Sleep(lock_manager.RenewInterval)
 			}
-			return lock.renewToken == ""
-		})
-		lc.keepLock(lock)
+		}
 	}()
 	return
 }
 
-func (lc *LockClient) doNewLock(key string, lockDuration time.Duration, owner string) (lock *LiveLock) {
-	lock = &LiveLock{
-		key:            key,
-		filer:          lc.seedFiler,
-		cancelCh:       make(chan struct{}),
-		expireAtNs:     time.Now().Add(lockDuration).UnixNano(),
-		grpcDialOption: lc.grpcDialOption,
-		owner:          owner,
-		lc:             lc,
-	}
-	var needRenewal bool
-	if lockDuration > lc.maxLockDuration {
-		lockDuration = lc.maxLockDuration
-		needRenewal = true
-	}
-	util.RetryForever("create lock:"+key, func() error {
-		errorMessage, err := lock.doLock(lockDuration)
-		if err != nil {
-			time.Sleep(time.Second)
-			return err
-		}
-		if errorMessage != "" {
-			time.Sleep(time.Second)
-			return fmt.Errorf("%v", errorMessage)
-		}
-		lock.isLocked = true
-		return nil
+func (lock *LiveLock) retryUntilLocked(lockDuration time.Duration) {
+	util.RetryUntil("create lock:"+lock.key, func() error {
+		return lock.AttemptToLock(lockDuration)
 	}, func(err error) (shouldContinue bool) {
 		if err != nil {
-			glog.Warningf("create lock %s: %s", key, err)
+			glog.Warningf("create lock %s: %s", lock.key, err)
 		}
 		return lock.renewToken == ""
 	})
+}
 
-	if needRenewal {
-		go lc.keepLock(lock)
+func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
+	errorMessage, err := lock.doLock(lockDuration)
+	if err != nil {
+		time.Sleep(time.Second)
+		return err
 	}
-
-	return
+	if errorMessage != "" {
+		time.Sleep(time.Second)
+		return fmt.Errorf("%v", errorMessage)
+	}
+	lock.isLocked = true
+	return nil
 }
 
-func (lock *LiveLock) IsLocked() bool {
-	return lock.isLocked
-}
-
-func (lock *LiveLock) StopLock() error {
-	close(lock.cancelCh)
+func (lock *LiveLock) StopShortLivedLock() error {
 	if !lock.isLocked {
 		return nil
 	}
-	return pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	defer func() {
+		lock.isLocked = false
+	}()
+	return pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		_, err := client.DistributedUnlock(context.Background(), &filer_pb.UnlockRequest{
 			Name:       lock.key,
 			RenewToken: lock.renewToken,
@@ -142,69 +138,40 @@ func (lock *LiveLock) StopLock() error {
 	})
 }
 
-func (lc *LockClient) keepLock(lock *LiveLock) {
-	ticker := time.Tick(lc.sleepDuration)
-	for {
-		select {
-		case <-ticker:
-			// renew the lock if lock.expireAtNs is still greater than now
-			util.RetryForever("keep lock:"+lock.key, func() error {
-				lockDuration := time.Duration(lock.expireAtNs-time.Now().UnixNano()) * time.Nanosecond
-				if lockDuration > lc.maxLockDuration {
-					lockDuration = lc.maxLockDuration
-				}
-				if lockDuration <= 0 {
-					return nil
-				}
-
-				errorMessage, err := lock.doLock(lockDuration)
-				if err != nil {
-					lock.isLocked = false
-					time.Sleep(time.Second)
-					return err
-				}
-				if errorMessage != "" {
-					lock.isLocked = false
-					time.Sleep(time.Second)
-					return fmt.Errorf("%v", errorMessage)
-				}
-				return nil
-			}, func(err error) (shouldContinue bool) {
-				if err == nil {
-					return false
-				}
-				glog.Warningf("keep lock %s: %v", lock.key, err)
-				return true
-			})
-			if !lock.isLocked {
-				return
-			}
-		case <-lock.cancelCh:
-			return
-		}
-	}
-}
-
 func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, err error) {
-	err = pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithFilerClient(false, 0, lock.hostFiler, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.DistributedLock(context.Background(), &filer_pb.LockRequest{
 			Name:          lock.key,
 			SecondsToLock: int64(lockDuration.Seconds()),
 			RenewToken:    lock.renewToken,
 			IsMoved:       false,
-			Owner:         lock.owner,
+			Owner:         lock.self,
 		})
-		if err == nil {
+		if err == nil && resp != nil {
 			lock.renewToken = resp.RenewToken
+		} else {
+			//this can be retried. Need to remember the last valid renewToken
+			lock.renewToken = ""
 		}
 		if resp != nil {
 			errorMessage = resp.Error
-			if resp.MovedTo != "" {
-				lock.filer = pb.ServerAddress(resp.MovedTo)
-				lock.lc.seedFiler = lock.filer
+			if resp.LockHostMovedTo != "" {
+				lock.hostFiler = pb.ServerAddress(resp.LockHostMovedTo)
+				lock.lc.seedFiler = lock.hostFiler
+			}
+			if resp.LockOwner != "" {
+				lock.owner = resp.LockOwner
+				// fmt.Printf("lock %s owner: %s\n", lock.key, lock.owner)
+			} else {
+				// fmt.Printf("lock %s has no owner\n", lock.key)
+				lock.owner = ""
 			}
 		}
 		return err
 	})
 	return
 }
+
+func (lock *LiveLock) LockOwner() string {
+	return lock.owner
+}

weed/cluster/lock_manager/distributed_lock_manager.go

@@ -2,11 +2,13 @@ package lock_manager
 
 import (
 	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"time"
 )
 
-const MaxDuration = 1<<63 - 1
+const RenewInterval = time.Second * 3
+const LiveLockTTL = time.Second * 7
 
 var NoLockServerError = fmt.Errorf("no lock server found")
 
@@ -24,7 +26,7 @@ func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
 	}
 }
 
-func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (renewToken string, movedTo pb.ServerAddress, err error) {
+func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string, owner string) (lockOwner string, renewToken string, movedTo pb.ServerAddress, err error) {
 	movedTo, err = dlm.findLockOwningFiler(key)
 	if err != nil {
 		return
@@ -32,7 +34,7 @@ func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64
 	if movedTo != dlm.Host {
 		return
 	}
-	renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
+	lockOwner, renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token, owner)
 	return
 }
 
@@ -53,6 +55,8 @@ func (dlm *DistributedLockManager) FindLockOwner(key string) (owner string, move
 		return
 	}
 	if movedTo != dlm.Host {
+		servers := dlm.LockRing.GetSnapshot()
+		glog.V(0).Infof("lock %s not on current %s but on %s from %v", key, dlm.Host, movedTo, servers)
 		return
 	}
 	owner, err = dlm.lockManager.GetLockOwner(key)

weed/cluster/lock_manager/lock_manager.go

@@ -3,7 +3,8 @@ package lock_manager
 import (
 	"fmt"
 	"github.com/google/uuid"
-	"github.com/puzpuzpuz/xsync/v2"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"sync"
 	"time"
 )
 
@@ -11,10 +12,12 @@ var LockErrorNonEmptyTokenOnNewLock = fmt.Errorf("lock: non-empty token on a new
 var LockErrorNonEmptyTokenOnExpiredLock = fmt.Errorf("lock: non-empty token on an expired lock")
 var LockErrorTokenMismatch = fmt.Errorf("lock: token mismatch")
 var UnlockErrorTokenMismatch = fmt.Errorf("unlock: token mismatch")
+var LockNotFound = fmt.Errorf("lock not found")
 
 // LockManager local lock manager, used by distributed lock manager
 type LockManager struct {
-	locks *xsync.MapOf[string, *Lock]
+	locks      map[string]*Lock
+	accessLock sync.RWMutex
 }
 type Lock struct {
 	Token       string
@@ -25,126 +28,155 @@ type Lock struct {
 
 func NewLockManager() *LockManager {
 	t := &LockManager{
-		locks: xsync.NewMapOf[*Lock](),
+		locks: make(map[string]*Lock),
 	}
 	go t.CleanUp()
 	return t
 }
 
-func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (renewToken string, err error) {
-	lm.locks.Compute(path, func(oldValue *Lock, loaded bool) (newValue *Lock, delete bool) {
-		if oldValue != nil {
-			if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
-				// lock is expired, set to a new lock
-				if token != "" {
-					err = LockErrorNonEmptyTokenOnExpiredLock
-					return nil, false
-				} else {
-					// new lock
-					renewToken = uuid.New().String()
-					return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
-				}
-			}
-			// not expired
-			if oldValue.Token == token {
-				// token matches, renew the lock
-				renewToken = uuid.New().String()
-				return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
+func (lm *LockManager) Lock(path string, expiredAtNs int64, token string, owner string) (lockOwner, renewToken string, err error) {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	glog.V(4).Infof("lock %s %v %v %v", path, time.Unix(0, expiredAtNs), token, owner)
+
+	if oldValue, found := lm.locks[path]; found {
+		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
+			// lock is expired, set to a new lock
+			if token != "" {
+				glog.V(4).Infof("lock expired key %s non-empty token %v owner %v ts %s", path, token, owner, time.Unix(0, oldValue.ExpiredAtNs))
+				err = LockErrorNonEmptyTokenOnExpiredLock
+				return
 			} else {
-				err = LockErrorTokenMismatch
-				return oldValue, false
+				// new lock
+				renewToken = uuid.New().String()
+				glog.V(4).Infof("key %s new token %v owner %v", path, renewToken, owner)
+				lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+				return
 			}
+		}
+		// not expired
+		lockOwner = oldValue.Owner
+		if oldValue.Token == token {
+			// token matches, renew the lock
+			renewToken = uuid.New().String()
+			glog.V(4).Infof("key %s old token %v owner %v => %v owner %v", path, oldValue.Token, oldValue.Owner, renewToken, owner)
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+			return
 		} else {
 			if token == "" {
 				// new lock
-				renewToken = uuid.New().String()
-				return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}, false
-			} else {
-				err = LockErrorNonEmptyTokenOnNewLock
-				return nil, false
+				glog.V(4).Infof("key %s locked by %v", path, oldValue.Owner)
+				err = fmt.Errorf("lock already owned by %v", oldValue.Owner)
+				return
 			}
+			glog.V(4).Infof("key %s expected token %v owner %v received %v from %v", path, oldValue.Token, oldValue.Owner, token, owner)
+			err = fmt.Errorf("lock: token mismatch")
+			return
 		}
-	})
-	return
+	} else {
+		glog.V(4).Infof("key %s no lock owner %v", path, owner)
+		if token == "" {
+			// new lock
+			glog.V(4).Infof("key %s new token %v owner %v", path, token, owner)
+			renewToken = uuid.New().String()
+			lm.locks[path] = &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs, Owner: owner}
+			return
+		} else {
+			glog.V(4).Infof("key %s non-empty token %v owner %v", path, token, owner)
+			err = LockErrorNonEmptyTokenOnNewLock
+			return
+		}
+	}
 }
 
 func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err error) {
-	lm.locks.Compute(path, func(oldValue *Lock, loaded bool) (newValue *Lock, delete bool) {
-		if oldValue != nil {
-			now := time.Now()
-			if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
-				// lock is expired, delete it
-				isUnlocked = true
-				return nil, true
-			}
-			if oldValue.Token == token {
-				if oldValue.ExpiredAtNs <= now.UnixNano() {
-					isUnlocked = true
-					return nil, true
-				}
-				return oldValue, false
-			} else {
-				isUnlocked = false
-				err = UnlockErrorTokenMismatch
-				return oldValue, false
-			}
-		} else {
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	if oldValue, found := lm.locks[path]; found {
+		now := time.Now()
+		if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
+			// lock is expired, delete it
 			isUnlocked = true
-			return nil, true
+			glog.V(4).Infof("key %s expired at %v", path, time.Unix(0, oldValue.ExpiredAtNs))
+			delete(lm.locks, path)
+			return
 		}
-	})
+		if oldValue.Token == token {
+			isUnlocked = true
+			glog.V(4).Infof("key %s unlocked with %v", path, token)
+			delete(lm.locks, path)
+			return
+		} else {
+			isUnlocked = false
+			err = UnlockErrorTokenMismatch
+			return
+		}
+	}
+	err = LockNotFound
 	return
 }
 
 func (lm *LockManager) CleanUp() {
+
 	for {
 		time.Sleep(1 * time.Minute)
 		now := time.Now().UnixNano()
-		lm.locks.Range(func(key string, value *Lock) bool {
+
+		lm.accessLock.Lock()
+		for key, value := range lm.locks {
 			if value == nil {
-				return true
+				continue
 			}
 			if now > value.ExpiredAtNs {
-				lm.locks.Delete(key)
-				return true
+				glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, value.ExpiredAtNs))
+				delete(lm.locks, key)
 			}
-			return true
-		})
+		}
+		lm.accessLock.Unlock()
 	}
 }
 
 // SelectLocks takes out locks by key
 // if keyFn return true, the lock will be taken out
 func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Lock) {
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
+
 	now := time.Now().UnixNano()
-	lm.locks.Range(func(key string, lock *Lock) bool {
+
+	for key, lock := range lm.locks {
 		if now > lock.ExpiredAtNs {
-			lm.locks.Delete(key)
-			return true
+			glog.V(4).Infof("key %s expired at %v", key, time.Unix(0, lock.ExpiredAtNs))
+			delete(lm.locks, key)
+			continue
 		}
 		if selectFn(key) {
-			lm.locks.Delete(key)
+			glog.V(4).Infof("key %s selected and deleted", key)
+			delete(lm.locks, key)
 			lock.Key = key
 			locks = append(locks, lock)
 		}
-		return true
-	})
+	}
 	return
 }
 
 // InsertLock inserts a lock unconditionally
 func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string, owner string) {
-	lm.locks.Store(path, &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner})
+	lm.accessLock.Lock()
+	defer lm.accessLock.Unlock()
+
+	lm.locks[path] = &Lock{Token: token, ExpiredAtNs: expiredAtNs, Owner: owner}
 }
 
 func (lm *LockManager) GetLockOwner(key string) (owner string, err error) {
-	lm.locks.Range(func(k string, lock *Lock) bool {
-		if k == key && lock != nil {
-			owner = lock.Owner
-			return false
-		}
-		return true
-	})
-	return
+	lm.accessLock.RLock()
+	defer lm.accessLock.RUnlock()
 
+	if lock, found := lm.locks[key]; found {
+		return lock.Owner, nil
+	}
+	err = LockNotFound
+	return
 }

weed/cluster/lock_manager/lock_ring.go

@@ -1,6 +1,7 @@
 package lock_manager
 
 import (
+	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sort"
@@ -39,9 +40,11 @@ func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.Ser
 // AddServer adds a server to the ring
 // if the previous snapshot passed the snapshot interval, create a new snapshot
 func (r *LockRing) AddServer(server pb.ServerAddress) {
+	glog.V(0).Infof("add server %v", server)
 	r.Lock()
 
 	if _, found := r.candidateServers[server]; found {
+		glog.V(0).Infof("add server: already exists %v", server)
 		r.Unlock()
 		return
 	}
@@ -53,6 +56,8 @@ func (r *LockRing) AddServer(server pb.ServerAddress) {
 }
 
 func (r *LockRing) RemoveServer(server pb.ServerAddress) {
+	glog.V(0).Infof("remove server %v", server)
+
 	r.Lock()
 
 	if _, found := r.candidateServers[server]; !found {
@@ -72,7 +77,13 @@ func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
 		return servers[i] < servers[j]
 	})
 
+	r.Lock()
 	r.lastUpdateTime = time.Now()
+	// init candidateServers
+	for _, server := range servers {
+		r.candidateServers[server] = struct{}{}
+	}
+	r.Unlock()
 
 	r.addOneSnapshot(servers)
 

weed/command/compact.go

@@ -28,7 +28,7 @@ var (
 	compactVolumePath        = cmdCompact.Flag.String("dir", ".", "data directory to store files")
 	compactVolumeCollection  = cmdCompact.Flag.String("collection", "", "volume collection name")
 	compactVolumeId          = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
-	compactMethod            = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 or 1.")
+	compactMethod            = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 (default) or 1.")
 	compactVolumePreallocate = cmdCompact.Flag.Int64("preallocateMB", 0, "preallocate volume disk space")
 )
 

weed/command/filer.go

@@ -61,6 +61,8 @@ type FilerOptions struct {
 	showUIDirectoryDelete   *bool
 	downloadMaxMBps         *int
 	diskType                *string
+	allowedOrigins          *string
+	exposeDirectoryData     *bool
 }
 
 func init() {
@@ -91,6 +93,8 @@ func init() {
 	f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
 	f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
 	f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
+	f.allowedOrigins = cmdFiler.Flag.String("allowedOrigins", "*", "comma separated list of allowed origins")
+	f.exposeDirectoryData = cmdFiler.Flag.Bool("exposeDirectoryData", true, "whether to return directory metadata and content in Filer UI")
 
 	// start s3 on filer
 	filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -98,6 +102,7 @@ func init() {
 	filerS3Options.portHttps = cmdFiler.Flag.Int("s3.port.https", 0, "s3 server https listen port")
 	filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
 	filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
+	filerS3Options.allowedOrigins = cmdFiler.Flag.String("s3.allowedOrigins", "*", "comma separated list of allowed origins")
 	filerS3Options.dataCenter = cmdFiler.Flag.String("s3.dataCenter", "", "prefer to read and write to volumes in this data center")
 	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
@@ -169,7 +174,7 @@ func runFiler(cmd *Command, args []string) bool {
 
 	go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort)
 
-	filerAddress := util.JoinHostPort(*f.ip, *f.port)
+	filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
 	startDelay := time.Duration(2)
 	if *filerStartS3 {
 		filerS3Options.filer = &filerAddress
@@ -229,6 +234,9 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.bindIp == "" {
 		*fo.bindIp = *fo.ip
 	}
+	if *fo.allowedOrigins == "" {
+		*fo.allowedOrigins = "*"
+	}
 
 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
 
@@ -253,6 +261,7 @@ func (fo *FilerOptions) startFiler() {
 		ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
 		DownloadMaxBytesPs:    int64(*fo.downloadMaxMBps) * 1024 * 1024,
 		DiskType:              *fo.diskType,
+		AllowedOrigins:        strings.Split(*fo.allowedOrigins, ","),
 	})
 	if nfs_err != nil {
 		glog.Fatalf("Filer startup error: %v", nfs_err)

weed/command/filer_backup.go

@@ -8,6 +8,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"regexp"
 	"time"
 )
 
@@ -16,8 +17,10 @@ type FilerBackupOptions struct {
 	filer           *string
 	path            *string
 	excludePaths    *string
+	excludeFileName *string
 	debug           *bool
 	proxyByFiler    *bool
+	doDeleteFiles   *bool
 	timeAgo         *time.Duration
 	retentionDays   *int
 }
@@ -31,11 +34,12 @@ func init() {
 	filerBackupOptions.filer = cmdFilerBackup.Flag.String("filer", "localhost:8888", "filer of one SeaweedFS cluster")
 	filerBackupOptions.path = cmdFilerBackup.Flag.String("filerPath", "/", "directory to sync on filer")
 	filerBackupOptions.excludePaths = cmdFilerBackup.Flag.String("filerExcludePaths", "", "exclude directories to sync on filer")
+	filerBackupOptions.excludeFileName = cmdFilerBackup.Flag.String("filerExcludeFileName", "", "exclude file names that match the regexp to sync on filer")
 	filerBackupOptions.proxyByFiler = cmdFilerBackup.Flag.Bool("filerProxy", false, "read and write file chunks by filer instead of volume servers")
+	filerBackupOptions.doDeleteFiles = cmdFilerBackup.Flag.Bool("doDeleteFiles", false, "delete files on the destination")
 	filerBackupOptions.debug = cmdFilerBackup.Flag.Bool("debug", false, "debug mode to print out received files")
 	filerBackupOptions.timeAgo = cmdFilerBackup.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
 	filerBackupOptions.retentionDays = cmdFilerBackup.Flag.Int("retentionDays", 0, "incremental backup retention days")
-
 }
 
 var cmdFilerBackup = &Command{
@@ -81,8 +85,7 @@ const (
 func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, clientId int32, clientEpoch int32) error {
 
 	// find data sink
-	config := util.GetViper()
-	dataSink := findSink(config)
+	dataSink := findSink(util.GetViper())
 	if dataSink == nil {
 		return fmt.Errorf("no data sink configured in replication.toml")
 	}
@@ -90,6 +93,13 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 	sourceFiler := pb.ServerAddress(*backupOption.filer)
 	sourcePath := *backupOption.path
 	excludePaths := util.StringSplit(*backupOption.excludePaths, ",")
+	var reExcludeFileName *regexp.Regexp
+	if *backupOption.excludeFileName != "" {
+		var err error
+		if reExcludeFileName, err = regexp.Compile(*backupOption.excludeFileName); err != nil {
+			return fmt.Errorf("error compile regexp %v for exclude file name: %+v", *backupOption.excludeFileName, err)
+		}
+	}
 	timeAgo := *backupOption.timeAgo
 	targetPath := dataSink.GetSinkToDirectory()
 	debug := *backupOption.debug
@@ -119,7 +129,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		*backupOption.proxyByFiler)
 	dataSink.SetSourceFiler(filerSource)
 
-	processEventFn := genProcessFunction(sourcePath, targetPath, excludePaths, dataSink, debug)
+	processEventFn := genProcessFunction(sourcePath, targetPath, excludePaths, reExcludeFileName, dataSink, *backupOption.doDeleteFiles, debug)
 
 	processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
 		glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3))

weed/command/filer_copy.go

@@ -31,21 +31,22 @@ var (
 )
 
 type CopyOptions struct {
-	include          *string
-	replication      *string
-	collection       *string
-	ttl              *string
-	diskType         *string
-	maxMB            *int
-	masterClient     *wdclient.MasterClient
-	concurrentFiles  *int
-	concurrentChunks *int
-	grpcDialOption   grpc.DialOption
-	masters          []string
-	cipher           bool
-	ttlSec           int32
-	checkSize        *bool
-	verbose          *bool
+	include            *string
+	replication        *string
+	collection         *string
+	ttl                *string
+	diskType           *string
+	maxMB              *int
+	masterClient       *wdclient.MasterClient
+	concurrentFiles    *int
+	concurrentChunks   *int
+	grpcDialOption     grpc.DialOption
+	masters            []string
+	cipher             bool
+	ttlSec             int32
+	checkSize          *bool
+	verbose            *bool
+	volumeServerAccess *string
 }
 
 func init() {
@@ -61,6 +62,7 @@ func init() {
 	copy.concurrentChunks = cmdFilerCopy.Flag.Int("concurrentChunks", 8, "concurrent chunk copy goroutines for each file")
 	copy.checkSize = cmdFilerCopy.Flag.Bool("check.size", false, "copy when the target file size is different from the source file")
 	copy.verbose = cmdFilerCopy.Flag.Bool("verbose", false, "print out details during copying")
+	copy.volumeServerAccess = cmdFilerCopy.Flag.String("volumeServerAccess", "direct", "access volume servers by [direct|publicUrl]")
 }
 
 var cmdFilerCopy = &Command{
@@ -580,6 +582,9 @@ func (worker *FileCopyWorker) WithFilerClient(streamingMode bool, fn func(filer_
 }
 
 func (worker *FileCopyWorker) AdjustedUrl(location *filer_pb.Location) string {
+	if *worker.options.volumeServerAccess == "publicUrl" {
+		return location.PublicUrl
+	}
 	return location.Url
 }